Merge branch 'master' into BASEPATH

webapp/backend/cmd/scrutiny/scrutiny.go

@@ -27,8 +27,8 @@ func main() {
 	}
 
 	//we're going to load the config file manually, since we need to validate it.
-	err = config.ReadConfig("/scrutiny/config/scrutiny.yaml") // Find and read the config file
-	if _, ok := err.(errors.ConfigFileMissingError); ok {     // Handle errors reading the config file
+	err = config.ReadConfig("/opt/scrutiny/config/scrutiny.yaml") // Find and read the config file
+	if _, ok := err.(errors.ConfigFileMissingError); ok {         // Handle errors reading the config file
 		//ignore "could not find config file"
 	} else if err != nil {
 		os.Exit(1)

webapp/backend/pkg/config/config.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/viper"
 	"log"
 	"os"
+	"strings"
 )
 
 // When initializing this class the following methods must be called:
@@ -29,21 +30,29 @@ func (c *configuration) Init() error {
 	//set defaults
 	c.SetDefault("web.listen.port", "8080")
 	c.SetDefault("web.listen.host", "0.0.0.0")
-	c.SetDefault("web.src.frontend.path", "/scrutiny/web")
+	c.SetDefault("web.src.frontend.path", "/opt/scrutiny/web")
 	c.SetDefault("web.src.backend.basepath", "")
-	c.SetDefault("web.database.location", "/scrutiny/config/scrutiny.db")
+	c.SetDefault("web.database.location", "/opt/scrutiny/config/scrutiny.db")
 
 	c.SetDefault("log.level", "INFO")
 	c.SetDefault("log.file", "")
 
 	c.SetDefault("notify.urls", []string{})
 
+	c.SetDefault("web.influxdb.host", "0.0.0.0")
+	c.SetDefault("web.influxdb.port", "8086")
+	c.SetDefault("web.influxdb.org", "scrutiny")
+	c.SetDefault("web.influxdb.bucket", "metrics")
+	c.SetDefault("web.influxdb.init_username", "admin")
+	c.SetDefault("web.influxdb.init_password", "password12345")
+	c.SetDefault("web.influxdb.retention_policy", true)
+
 	//c.SetDefault("disks.include", []string{})
 	//c.SetDefault("disks.exclude", []string{})
 
-	//c.SetDefault("notify.metric.script", "/scrutiny/config/notify-metrics.sh")
-	//c.SetDefault("notify.long.script", "/scrutiny/config/notify-long-test.sh")
-	//c.SetDefault("notify.short.script", "/scrutiny/config/notify-short-test.sh")
+	//c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh")
+	//c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh")
+	//c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh")
 
 	//c.SetDefault("collect.metric.enable", true)
 	//c.SetDefault("collect.metric.command", "-a -o on -S on")
@@ -57,11 +66,19 @@ func (c *configuration) Init() error {
 	//c.SetConfigName("drawbridge")
 	//c.AddConfigPath("$HOME/")
 
+	//configure env variable parsing.
+	c.SetEnvPrefix("SCRUTINY")
+	c.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
+	c.AutomaticEnv()
+
 	//CLI options will be added via the `Set()` function
 	return nil
 }
 
 func (c *configuration) ReadConfig(configFilePath string) error {
+	//make sure that we specify that this is the correct config path (for eventual WriteConfig() calls)
+	c.SetConfigFile(configFilePath)
+
 	configFilePath, err := utils.ExpandPath(configFilePath)
 	if err != nil {
 		return err

webapp/backend/pkg/config/interface.go

@@ -9,6 +9,7 @@ import (
 type Interface interface {
 	Init() error
 	ReadConfig(configFilePath string) error
+	WriteConfig() error
 	Set(key string, value interface{})
 	SetDefault(key string, value interface{})
 

webapp/backend/pkg/config/mock/mock_config.go

@@ -5,87 +5,36 @@
 package mock_config
 
 import (
+	reflect "reflect"
+
 	gomock "github.com/golang/mock/gomock"
 	viper "github.com/spf13/viper"
-	reflect "reflect"
 )
 
-// MockInterface is a mock of Interface interface
+// MockInterface is a mock of Interface interface.
 type MockInterface struct {
 	ctrl     *gomock.Controller
 	recorder *MockInterfaceMockRecorder
 }
 
-// MockInterfaceMockRecorder is the mock recorder for MockInterface
+// MockInterfaceMockRecorder is the mock recorder for MockInterface.
 type MockInterfaceMockRecorder struct {
 	mock *MockInterface
 }
 
-// NewMockInterface creates a new mock instance
+// NewMockInterface creates a new mock instance.
 func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
 	mock := &MockInterface{ctrl: ctrl}
 	mock.recorder = &MockInterfaceMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use
+// EXPECT returns an object that allows the caller to indicate expected use.
 func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Init mocks base method
-func (m *MockInterface) Init() error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Init")
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Init indicates an expected call of Init
-func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
-}
-
-// ReadConfig mocks base method
-func (m *MockInterface) ReadConfig(configFilePath string) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// ReadConfig indicates an expected call of ReadConfig
-func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
-}
-
-// Set mocks base method
-func (m *MockInterface) Set(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Set", key, value)
-}
-
-// Set indicates an expected call of Set
-func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
-}
-
-// SetDefault mocks base method
-func (m *MockInterface) SetDefault(key string, value interface{}) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "SetDefault", key, value)
-}
-
-// SetDefault indicates an expected call of SetDefault
-func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
-}
-
-// AllSettings mocks base method
+// AllSettings mocks base method.
 func (m *MockInterface) AllSettings() map[string]interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "AllSettings")
@@ -93,27 +42,13 @@ func (m *MockInterface) AllSettings() map[string]interface{} {
 	return ret0
 }
 
-// AllSettings indicates an expected call of AllSettings
+// AllSettings indicates an expected call of AllSettings.
 func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings))
 }
 
-// IsSet mocks base method
-func (m *MockInterface) IsSet(key string) bool {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "IsSet", key)
-	ret0, _ := ret[0].(bool)
-	return ret0
-}
-
-// IsSet indicates an expected call of IsSet
-func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
-}
-
-// Get mocks base method
+// Get mocks base method.
 func (m *MockInterface) Get(key string) interface{} {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", key)
@@ -121,13 +56,13 @@ func (m *MockInterface) Get(key string) interface{} {
 	return ret0
 }
 
-// Get indicates an expected call of Get
+// Get indicates an expected call of Get.
 func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key)
 }
 
-// GetBool mocks base method
+// GetBool mocks base method.
 func (m *MockInterface) GetBool(key string) bool {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetBool", key)
@@ -135,13 +70,13 @@ func (m *MockInterface) GetBool(key string) bool {
 	return ret0
 }
 
-// GetBool indicates an expected call of GetBool
+// GetBool indicates an expected call of GetBool.
 func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key)
 }
 
-// GetInt mocks base method
+// GetInt mocks base method.
 func (m *MockInterface) GetInt(key string) int {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetInt", key)
@@ -149,13 +84,13 @@ func (m *MockInterface) GetInt(key string) int {
 	return ret0
 }
 
-// GetInt indicates an expected call of GetInt
+// GetInt indicates an expected call of GetInt.
 func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key)
 }
 
-// GetString mocks base method
+// GetString mocks base method.
 func (m *MockInterface) GetString(key string) string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetString", key)
@@ -163,13 +98,13 @@ func (m *MockInterface) GetString(key string) string {
 	return ret0
 }
 
-// GetString indicates an expected call of GetString
+// GetString indicates an expected call of GetString.
 func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key)
 }
 
-// GetStringSlice mocks base method
+// GetStringSlice mocks base method.
 func (m *MockInterface) GetStringSlice(key string) []string {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetStringSlice", key)
@@ -177,13 +112,79 @@ func (m *MockInterface) GetStringSlice(key string) []string {
 	return ret0
 }
 
-// GetStringSlice indicates an expected call of GetStringSlice
+// GetStringSlice indicates an expected call of GetStringSlice.
 func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key)
 }
 
-// UnmarshalKey mocks base method
+// Init mocks base method.
+func (m *MockInterface) Init() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Init")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Init indicates an expected call of Init.
+func (mr *MockInterfaceMockRecorder) Init() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init))
+}
+
+// IsSet mocks base method.
+func (m *MockInterface) IsSet(key string) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsSet", key)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsSet indicates an expected call of IsSet.
+func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key)
+}
+
+// ReadConfig mocks base method.
+func (m *MockInterface) ReadConfig(configFilePath string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReadConfig", configFilePath)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReadConfig indicates an expected call of ReadConfig.
+func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath)
+}
+
+// Set mocks base method.
+func (m *MockInterface) Set(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Set", key, value)
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value)
+}
+
+// SetDefault mocks base method.
+func (m *MockInterface) SetDefault(key string, value interface{}) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetDefault", key, value)
+}
+
+// SetDefault indicates an expected call of SetDefault.
+func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value)
+}
+
+// UnmarshalKey mocks base method.
 func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{key, rawVal}
@@ -195,9 +196,23 @@ func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts
 	return ret0
 }
 
-// UnmarshalKey indicates an expected call of UnmarshalKey
+// UnmarshalKey indicates an expected call of UnmarshalKey.
 func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{key, rawVal}, decoderOpts...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...)
 }
+
+// WriteConfig mocks base method.
+func (m *MockInterface) WriteConfig() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "WriteConfig")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// WriteConfig indicates an expected call of WriteConfig.
+func (mr *MockInterfaceMockRecorder) WriteConfig() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteConfig", reflect.TypeOf((*MockInterface)(nil).WriteConfig))
+}

webapp/backend/pkg/constants.go

@@ -0,0 +1,28 @@
+package pkg
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+const SmartAttributeStatusPassed = 0
+const SmartAttributeStatusFailed = 1
+const SmartAttributeStatusWarning = 2
+
+const SmartWhenFailedFailingNow = "FAILING_NOW"
+const SmartWhenFailedInThePast = "IN_THE_PAST"
+
+//const SmartStatusPassed = "passed"
+//const SmartStatusFailed = "failed"
+
+type DeviceStatus int
+
+const (
+	DeviceStatusPassed         DeviceStatus = 0
+	DeviceStatusFailedSmart    DeviceStatus = iota
+	DeviceStatusFailedScrutiny DeviceStatus = iota
+)
+
+func Set(b, flag DeviceStatus) DeviceStatus    { return b | flag }
+func Clear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
+func Toggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
+func Has(b, flag DeviceStatus) bool            { return b&flag != 0 }

webapp/backend/pkg/database/interface.go

@@ -0,0 +1,30 @@
+package database
+
+import (
+	"context"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+)
+
+type DeviceRepo interface {
+	Close() error
+
+	//GetSettings()
+	//SaveSetting()
+
+	RegisterDevice(ctx context.Context, dev models.Device) error
+	GetDevices(ctx context.Context) ([]models.Device, error)
+	UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
+	UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error)
+	GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error)
+
+	SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
+	GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error)
+
+	SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error
+
+	GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
+	GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
+}

webapp/backend/pkg/database/migrations/m20201107210306/device.go

@@ -0,0 +1,47 @@
+package m20201107210306
+
+import (
+	"time"
+)
+
+// Deprecated: m20201107210306.Device is deprecated, only used by db migrations
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN    string `json:"wwn" gorm:"primary_key"`
+	HostId string `json:"host_id"`
+
+	DeviceName     string  `json:"device_name"`
+	Manufacturer   string  `json:"manufacturer"`
+	ModelName      string  `json:"model_name"`
+	InterfaceType  string  `json:"interface_type"`
+	InterfaceSpeed string  `json:"interface_speed"`
+	SerialNumber   string  `json:"serial_number"`
+	Firmware       string  `json:"firmware"`
+	RotationSpeed  int     `json:"rotational_speed"`
+	Capacity       int64   `json:"capacity"`
+	FormFactor     string  `json:"form_factor"`
+	SmartSupport   bool    `json:"smart_support"`
+	DeviceProtocol string  `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string  `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+	SmartResults   []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
+}
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+func (dv *Device) IsAta() bool {
+	return dv.DeviceProtocol == DeviceProtocolAta
+}
+
+func (dv *Device) IsScsi() bool {
+	return dv.DeviceProtocol == DeviceProtocolScsi
+}
+
+func (dv *Device) IsNvme() bool {
+	return dv.DeviceProtocol == DeviceProtocolNvme
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart.go

@@ -0,0 +1,26 @@
+package m20201107210306
+
+import (
+	"gorm.io/gorm"
+	"time"
+)
+
+// Deprecated: m20201107210306.Smart is deprecated, only used by db migrations
+type Smart struct {
+	gorm.Model
+
+	DeviceWWN string `json:"device_wwn"`
+	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
+
+	TestDate    time.Time `json:"date"`
+	SmartStatus string    `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed
+
+	//Metrics
+	Temp            int64 `json:"temp"`
+	PowerOnHours    int64 `json:"power_on_hours"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+
+	AtaAttributes  []SmartAtaAttribute  `json:"ata_attributes" gorm:"foreignkey:SmartId"`
+	NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"`
+	ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go

@@ -0,0 +1,26 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartAtaAttribute is deprecated, only used by db migrations
+type SmartAtaAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId int    `json:"attribute_id"`
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Worst       int    `json:"worst"`
+	Threshold   int    `json:"thresh"`
+	RawValue    int64  `json:"raw_value"`
+	RawString   string `json:"raw_string"`
+	WhenFailed  string `json:"when_failed"`
+
+	TransformedValue int64               `json:"transformed_value"`
+	Status           string              `gorm:"-" json:"status,omitempty"`
+	StatusReason     string              `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64             `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartAtaAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go

@@ -0,0 +1,22 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartNvmeAttribute is deprecated, only used by db migrations
+type SmartNvmeAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Threshold   int    `json:"thresh"`
+
+	TransformedValue int64                `json:"transformed_value"`
+	Status           string               `gorm:"-" json:"status,omitempty"`
+	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go

@@ -0,0 +1,22 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartScsiAttribute is deprecated, only used by db migrations
+type SmartScsiAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Threshold   int    `json:"thresh"`
+
+	TransformedValue int64                `json:"transformed_value"`
+	Status           string               `gorm:"-" json:"status,omitempty"`
+	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartScsiAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/scrutiny_repository.go

@@ -0,0 +1,407 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+	"github.com/influxdata/influxdb-client-go/v2/domain"
+	"github.com/sirupsen/logrus"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"time"
+)
+
+const (
+	// 60seconds * 60minutes * 24hours * 15 days
+	RETENTION_PERIOD_15_DAYS_IN_SECONDS = 1_296_000
+
+	// 60seconds * 60minutes * 24hours * 7 days * 9 weeks
+	RETENTION_PERIOD_9_WEEKS_IN_SECONDS = 5_443_200
+
+	// 60seconds * 60minutes * 24hours * 7 days * (52 + 52 + 4)weeks
+	RETENTION_PERIOD_25_MONTHS_IN_SECONDS = 65_318_400
+
+	DURATION_KEY_WEEK    = "week"
+	DURATION_KEY_MONTH   = "month"
+	DURATION_KEY_YEAR    = "year"
+	DURATION_KEY_FOREVER = "forever"
+)
+
+//// GormLogger is a custom logger for Gorm, making it use logrus.
+//type GormLogger struct{ Logger logrus.FieldLogger }
+//
+//// Print handles log events from Gorm for the custom logger.
+//func (gl *GormLogger) Print(v ...interface{}) {
+//	switch v[0] {
+//	case "sql":
+//		gl.Logger.WithFields(
+//			logrus.Fields{
+//				"module":  "gorm",
+//				"type":    "sql",
+//				"rows":    v[5],
+//				"src_ref": v[1],
+//				"values":  v[4],
+//			},
+//		).Debug(v[3])
+//	case "log":
+//		gl.Logger.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2])
+//	}
+//}
+
+func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.FieldLogger) (DeviceRepo, error) {
+	backgroundContext := context.Background()
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// Gorm/SQLite setup
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
+	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
+		//TODO: figure out how to log database queries again.
+		//Logger: logger
+		DisableForeignKeyConstraintWhenMigrating: true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Failed to connect to database! - %v", err)
+	}
+	globalLogger.Infof("Successfully connected to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
+
+	//database.SetLogger()
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// InfluxDB setup
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+	// Create a new client using an InfluxDB server base URL and an authentication token
+	influxdbUrl := fmt.Sprintf("http://%s:%s", appConfig.GetString("web.influxdb.host"), appConfig.GetString("web.influxdb.port"))
+	globalLogger.Debugf("InfluxDB url: %s", influxdbUrl)
+
+	client := influxdb2.NewClient(influxdbUrl, appConfig.GetString("web.influxdb.token"))
+
+	if !appConfig.IsSet("web.influxdb.token") {
+		globalLogger.Debugf("No influxdb token found, running first-time setup...")
+
+		// if no token is provided, but we have a valid server, we're going to assume this is the first setup of our server.
+		// we will initialize with a predetermined username & password, that you should change.
+
+		// metrics bucket will have a retention period of 8 days (since it will be down-sampled once a week)
+		// in seconds (60seconds * 60minutes * 24hours * 15 days) = 1_296_000 (see EnsureBucket() function)
+		onboardingResponse, err := client.Setup(
+			backgroundContext,
+			appConfig.GetString("web.influxdb.init_username"),
+			appConfig.GetString("web.influxdb.init_password"),
+			appConfig.GetString("web.influxdb.org"),
+			appConfig.GetString("web.influxdb.bucket"),
+			0)
+		if err != nil {
+			return nil, err
+		}
+
+		appConfig.Set("web.influxdb.token", *onboardingResponse.Auth.Token)
+		// we should write the config file out here. Ignore failures.
+		err = appConfig.WriteConfig()
+		if err != nil {
+			globalLogger.Infof("ignoring error while writing influxdb info to config: %v", err)
+		}
+	}
+
+	// Use blocking write client for writes to desired bucket
+	writeAPI := client.WriteAPIBlocking(appConfig.GetString("web.influxdb.org"), appConfig.GetString("web.influxdb.bucket"))
+
+	// Get query client
+	queryAPI := client.QueryAPI(appConfig.GetString("web.influxdb.org"))
+
+	// Get task client
+	taskAPI := client.TasksAPI()
+
+	if writeAPI == nil || queryAPI == nil || taskAPI == nil {
+		return nil, fmt.Errorf("Failed to connect to influxdb!")
+	}
+
+	deviceRepo := scrutinyRepository{
+		appConfig:      appConfig,
+		logger:         globalLogger,
+		influxClient:   client,
+		influxWriteApi: writeAPI,
+		influxQueryApi: queryAPI,
+		influxTaskApi:  taskAPI,
+		gormClient:     database,
+	}
+
+	orgInfo, err := client.OrganizationsAPI().FindOrganizationByName(backgroundContext, appConfig.GetString("web.influxdb.org"))
+	if err != nil {
+		return nil, err
+	}
+
+	// Initialize Buckets (if necessary)
+	err = deviceRepo.EnsureBuckets(backgroundContext, orgInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	// Initialize Background Tasks
+	err = deviceRepo.EnsureTasks(backgroundContext, *orgInfo.Id)
+	if err != nil {
+		return nil, err
+	}
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// InfluxDB & SQLite migrations
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	//database.AutoMigrate(&models.Device{})
+	err = deviceRepo.Migrate(backgroundContext)
+	if err != nil {
+		return nil, err
+	}
+
+	return &deviceRepo, nil
+}
+
+type scrutinyRepository struct {
+	appConfig config.Interface
+	logger    logrus.FieldLogger
+
+	influxWriteApi api.WriteAPIBlocking
+	influxQueryApi api.QueryAPI
+	influxTaskApi  api.TasksAPI
+	influxClient   influxdb2.Client
+
+	gormClient *gorm.DB
+}
+
+func (sr *scrutinyRepository) Close() error {
+	sr.influxClient.Close()
+	return nil
+}
+
+func (sr *scrutinyRepository) EnsureBuckets(ctx context.Context, org *domain.Organization) error {
+
+	var mainBucketRetentionRule domain.RetentionRule
+	var weeklyBucketRetentionRule domain.RetentionRule
+	var monthlyBucketRetentionRule domain.RetentionRule
+	if sr.appConfig.GetBool("web.influxdb.retention_policy") {
+
+		// in tests, we may not want to set a retention policy. If "false", we can set data with old timestamps,
+		// then manually run the down sampling scripts. This should be true for production environments.
+		mainBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_15_DAYS_IN_SECONDS}
+		weeklyBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_9_WEEKS_IN_SECONDS}
+		monthlyBucketRetentionRule = domain.RetentionRule{EverySeconds: RETENTION_PERIOD_25_MONTHS_IN_SECONDS}
+	}
+
+	mainBucket := sr.appConfig.GetString("web.influxdb.bucket")
+	if foundMainBucket, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, mainBucket); foundErr != nil {
+		// metrics bucket will have a retention period of 15 days (since it will be down-sampled once a week)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, mainBucket, mainBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	} else if sr.appConfig.GetBool("web.influxdb.retention_policy") {
+		//correctly set the retention period for the main bucket (cant do it during setup/creation)
+		foundMainBucket.RetentionRules = domain.RetentionRules{mainBucketRetentionRule}
+		sr.influxClient.BucketsAPI().UpdateBucket(ctx, foundMainBucket)
+	}
+
+	//create buckets (used for downsampling)
+	weeklyBucket := fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, weeklyBucket); foundErr != nil {
+		// metrics_weekly bucket will have a retention period of 8+1 weeks (since it will be down-sampled once a month)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, weeklyBucket, weeklyBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	}
+
+	monthlyBucket := fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, monthlyBucket); foundErr != nil {
+		// metrics_monthly bucket will have a retention period of 24+1 months (since it will be down-sampled once a year)
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, monthlyBucket, monthlyBucketRetentionRule)
+		if err != nil {
+			return err
+		}
+	}
+
+	yearlyBucket := fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
+	if _, foundErr := sr.influxClient.BucketsAPI().FindBucketByName(ctx, yearlyBucket); foundErr != nil {
+		// metrics_yearly bucket will have an infinite retention period
+		_, err := sr.influxClient.BucketsAPI().CreateBucketWithName(ctx, org, yearlyBucket)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// DeviceSummary
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// get a map of all devices and associated SMART data
+func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) {
+	devices, err := sr.GetDevices(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	summaries := map[string]*models.DeviceSummary{}
+
+	for _, device := range devices {
+		summaries[device.WWN] = &models.DeviceSummary{Device: device}
+	}
+
+	// Get parser flux query result
+	//appConfig.GetString("web.influxdb.bucket")
+	queryStr := fmt.Sprintf(`
+  	import "influxdata/influxdb/schema"
+  	bucketBaseName = "%s"
+
+	dailyData = from(bucket: bucketBaseName)
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	weeklyData = from(bucket: bucketBaseName + "_weekly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	monthlyData = from(bucket: bucketBaseName + "_monthly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	yearlyData = from(bucket: bucketBaseName + "_yearly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+	
+	union(tables: [dailyData, weeklyData, monthlyData, yearlyData])
+	|> sort(columns: ["_time"], desc: false)
+	|> group(columns: ["device_wwn"])
+	|> last(column: "device_wwn")
+	|> yield(name: "last")
+		`,
+		sr.appConfig.GetString("web.influxdb.bucket"),
+	)
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+			// Observe when there is new grouping key producing new table
+			if result.TableChanged() {
+				//fmt.Printf("table: %s\n", result.TableMetadata().String())
+			}
+			// read result
+
+			//get summary data from Influxdb.
+			//result.Record().Values()
+			if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
+
+				//ensure summaries is intialized for this wwn
+				if _, exists := summaries[deviceWWN.(string)]; !exists {
+					summaries[deviceWWN.(string)] = &models.DeviceSummary{}
+				}
+
+				summaries[deviceWWN.(string)].SmartResults = &models.SmartSummary{
+					Temp:          result.Record().Values()["temp"].(int64),
+					PowerOnHours:  result.Record().Values()["power_on_hours"].(int64),
+					CollectorDate: result.Record().Values()["_time"].(time.Time),
+				}
+			}
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+
+	deviceTempHistory, err := sr.GetSmartTemperatureHistory(ctx, DURATION_KEY_FOREVER)
+	if err != nil {
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("========================>>>>>>>>======================")
+		sr.logger.Printf("Error: %v", err)
+	}
+	for wwn, tempHistory := range deviceTempHistory {
+		summaries[wwn].TempHistory = tempHistory
+	}
+
+	return summaries, nil
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper Methods
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+func (sr *scrutinyRepository) lookupBucketName(durationKey string) string {
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//data stored in the last week
+		return sr.appConfig.GetString("web.influxdb.bucket")
+	case DURATION_KEY_MONTH:
+		// data stored in the last month (after the first week)
+		return fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
+	}
+	return sr.appConfig.GetString("web.influxdb.bucket")
+}
+
+func (sr *scrutinyRepository) lookupDuration(durationKey string) []string {
+
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//data stored in the last week
+		return []string{"-1w", "now()"}
+	case DURATION_KEY_MONTH:
+		// data stored in the last month (after the first week)
+		return []string{"-1mo", "-1w"}
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return []string{"-1y", "-1mo"}
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return []string{"-10y", "-1y"}
+	}
+	return []string{"-1w", "now()"}
+}
+
+func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []string {
+	switch durationKey {
+	case DURATION_KEY_WEEK:
+		//all data is stored in a single bucket
+		return []string{DURATION_KEY_WEEK}
+	case DURATION_KEY_MONTH:
+		//data is stored in the week bucket and the month bucket
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH}
+	case DURATION_KEY_YEAR:
+		// data stored in the last year (after the first month)
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR}
+	case DURATION_KEY_FOREVER:
+		//data stored before the last year
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR, DURATION_KEY_FOREVER}
+	}
+	return []string{DURATION_KEY_WEEK}
+}

webapp/backend/pkg/database/scrutiny_repository_device.go

@@ -0,0 +1,74 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"gorm.io/gorm/clause"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Device
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+//insert device into DB (and update specified columns if device is already registered)
+// update device fields that may change: (DeviceType, HostID)
+func (sr *scrutinyRepository) RegisterDevice(ctx context.Context, dev models.Device) error {
+	if err := sr.gormClient.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "wwn"}},
+		DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
+	}).Create(&dev).Error; err != nil {
+		return err
+	}
+	return nil
+}
+
+// get a list of all devices (only device metadata, no SMART data)
+func (sr *scrutinyRepository) GetDevices(ctx context.Context) ([]models.Device, error) {
+	//Get a list of all the active devices.
+	devices := []models.Device{}
+	if err := sr.gormClient.WithContext(ctx).Find(&devices).Error; err != nil {
+		return nil, fmt.Errorf("Could not get device summary from DB: %v", err)
+	}
+	return devices, nil
+}
+
+// update device (only metadata) from collector
+func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
+	var device models.Device
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return device, fmt.Errorf("Could not get device from DB: %v", err)
+	}
+
+	//TODO catch GormClient err
+	err := device.UpdateFromCollectorSmartInfo(collectorSmartData)
+	if err != nil {
+		return device, err
+	}
+	return device, sr.gormClient.Model(&device).Updates(device).Error
+}
+
+//Update Device Status
+func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
+	var device models.Device
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return device, fmt.Errorf("Could not get device from DB: %v", err)
+	}
+
+	device.DeviceStatus = pkg.Set(device.DeviceStatus, status)
+	return device, sr.gormClient.Model(&device).Updates(device).Error
+}
+
+func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
+	var device models.Device
+
+	fmt.Println("GetDeviceDetails from GORM")
+
+	if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
+		return models.Device{}, err
+	}
+
+	return device, nil
+}

webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go

@@ -0,0 +1,169 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+	log "github.com/sirupsen/logrus"
+	"strings"
+	"time"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// SMART
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
+	deviceSmartData := measurements.Smart{}
+	err := deviceSmartData.FromCollectorSmartInfo(wwn, collectorSmartData)
+	if err != nil {
+		sr.logger.Errorln("Could not process SMART metrics", err)
+		return measurements.Smart{}, err
+	}
+
+	tags, fields := deviceSmartData.Flatten()
+
+	// write point immediately
+	return deviceSmartData, sr.saveDatapoint(sr.influxWriteApi, "smart", tags, fields, deviceSmartData.Date, ctx)
+}
+
+func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) {
+	// Get SMartResults from InfluxDB
+
+	//TODO: change the filter startrange to a real number.
+
+	// Get parser flux query result
+	//appConfig.GetString("web.influxdb.bucket")
+	queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey)
+	log.Infoln(queryStr)
+
+	smartResults := []measurements.Smart{}
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+			// Observe when there is new grouping key producing new table
+			if result.TableChanged() {
+				//fmt.Printf("table: %s\n", result.TableMetadata().String())
+			}
+
+			smartData, err := measurements.NewSmartFromInfluxDB(result.Record().Values())
+			if err != nil {
+				return nil, err
+			}
+			smartResults = append(smartResults, *smartData)
+
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+
+	return smartResults, nil
+
+	//if err := device.SquashHistory(); err != nil {
+	//	logger.Errorln("An error occurred while squashing device history", err)
+	//	c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+	//	return
+	//}
+	//
+	//if err := device.ApplyMetadataRules(); err != nil {
+	//	logger.Errorln("An error occurred while applying scrutiny thresholds & rules", err)
+	//	c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+	//	return
+	//}
+
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper Methods
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error {
+	//sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields))
+	p := influxdb2.NewPoint(measurement,
+		tags,
+		fields,
+		date)
+
+	// write point immediately
+	return influxWriteApi.WritePoint(ctx, p)
+}
+
+func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string {
+
+	/*
+
+		import "influxdata/influxdb/schema"
+		weekData = from(bucket: "metrics")
+		|> range(start: -1w, stop: now())
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		monthData = from(bucket: "metrics_weekly")
+		|> range(start: -1mo, stop: -1w)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		yearData = from(bucket: "metrics_monthly")
+		|> range(start: -1y, stop: -1mo)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		foreverData = from(bucket: "metrics_yearly")
+		|> range(start: -10y, stop: -1y)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		union(tables: [weekData, monthData, yearData, foreverData])
+		|> sort(columns: ["_time"], desc: false)
+		|> yield(name: "last")
+
+	*/
+
+	partialQueryStr := []string{
+		`import "influxdata/influxdb/schema"`,
+	}
+
+	nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey)
+
+	subQueryNames := []string{}
+	for _, nestedDurationKey := range nestedDurationKeys {
+		bucketName := sr.lookupBucketName(nestedDurationKey)
+		durationRange := sr.lookupDuration(nestedDurationKey)
+
+		subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey))
+		partialQueryStr = append(partialQueryStr, []string{
+			fmt.Sprintf(`%sData = from(bucket: "%s")`, nestedDurationKey, bucketName),
+			fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
+			`|> filter(fn: (r) => r["_measurement"] == "smart" )`,
+			fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn),
+			"|> schema.fieldsAsCols()",
+		}...)
+	}
+
+	if len(subQueryNames) == 1 {
+		//there's only one bucket being queried, no need to union, just aggregate the dataset and return
+		partialQueryStr = append(partialQueryStr, []string{
+			subQueryNames[0],
+			`|> yield()`,
+		}...)
+	} else {
+		partialQueryStr = append(partialQueryStr, []string{
+			fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
+			`|> sort(columns: ["_time"], desc: false)`,
+			`|> yield(name: "last")`,
+		}...)
+	}
+
+	return strings.Join(partialQueryStr, "\n")
+}

webapp/backend/pkg/database/scrutiny_repository_migrations.go

@@ -0,0 +1,451 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"github.com/go-gormigrate/gormigrate/v2"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"gorm.io/gorm"
+	"strconv"
+	"time"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// SQLite migrations
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+//database.AutoMigrate(&models.Device{})
+
+func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
+
+	sr.logger.Infoln("Database migration starting")
+
+	m := gormigrate.New(sr.gormClient, gormigrate.DefaultOptions, []*gormigrate.Migration{
+		{
+			ID: "20201107210306", // v0.3.13 (pre-influxdb schema). 9fac3c6308dc6cb6cd5bbc43a68cd93e8fb20b87
+			Migrate: func(tx *gorm.DB) error {
+				// it's a good practice to copy the struct inside the function,
+
+				return tx.AutoMigrate(
+					&m20201107210306.Device{},
+					&m20201107210306.Smart{},
+					&m20201107210306.SmartAtaAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+				)
+			},
+			Rollback: func(tx *gorm.DB) error {
+				return tx.Migrator().DropTable(
+					&m20201107210306.Device{},
+					&m20201107210306.Smart{},
+					&m20201107210306.SmartAtaAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+					"self_tests",
+				)
+			},
+		},
+		{
+			ID: "20220503113100", // backwards compatible - influxdb schema
+			Migrate: func(tx *gorm.DB) error {
+				// delete unnecessary table.
+				err := tx.Migrator().DropTable("self_tests")
+				if err != nil {
+					return err
+				}
+
+				//add columns to the Device schema, so we can start adding data to the database & influxdb
+				err = tx.Migrator().AddColumn(&models.Device{}, "Label") //Label  string `json:"label"`
+				if err != nil {
+					return err
+				}
+				err = tx.Migrator().AddColumn(&models.Device{}, "DeviceStatus") //DeviceStatus pkg.DeviceStatus `json:"device_status"`
+				if err != nil {
+					return err
+				}
+
+				//TODO: migrate the data from GORM to influxdb.
+				//get a list of all devices:
+				//	get a list of all smart scans in the last 2 weeks:
+				//		get a list of associated smart attribute data:
+				//			translate to a measurements.Smart{} object
+				//			call CUSTOM INFLUXDB SAVE FUNCTION (taking bucket as parameter)
+				//	get a list of all smart scans in the last 9 weeks:
+				//		do same as above (select 1 scan per week)
+				//	get a list of all smart scans in the last 25 months:
+				//		do same as above (select 1 scan per month)
+				//	get a list of all smart scans:
+				//		do same as above (select 1 scan per year)
+
+				preDevices := []m20201107210306.Device{} //pre-migration device information
+				if err = tx.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
+					return db.Order("smarts.created_at ASC") //OLD: .Limit(devicesCount)
+				}).Find(&preDevices).Error; err != nil {
+					sr.logger.Errorln("Could not get device summary from DB", err)
+					return err
+				}
+
+				//calculate bucket oldest dates
+				today := time.Now()
+				dailyBucketMax := today.Add(-RETENTION_PERIOD_15_DAYS_IN_SECONDS * time.Second)     //15 days
+				weeklyBucketMax := today.Add(-RETENTION_PERIOD_9_WEEKS_IN_SECONDS * time.Second)    //9 weeks
+				monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks
+
+				for _, preDevice := range preDevices {
+					sr.logger.Debugf("====================================")
+					sr.logger.Infof("begin processing device: %s", preDevice.WWN)
+
+					//weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary.
+					weeklyLookup := map[string]bool{}
+					monthlyLookup := map[string]bool{}
+					yearlyLookup := map[string]bool{}
+					for _, preSmartResult := range preDevice.SmartResults { //pre-migration smart results
+
+						//we're looping in ASC mode, so from oldest entry to most current.
+
+						err, postSmartResults := m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(tx, preDevice, preSmartResult)
+						if err != nil {
+							return err
+						}
+						smartTags, smartFields := postSmartResults.Flatten()
+
+						err, postSmartTemp := m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice, preSmartResult)
+						if err != nil {
+							return err
+						}
+						tempTags, tempFields := postSmartTemp.Flatten()
+						tempTags["device_wwn"] = preDevice.WWN
+
+						year, week := postSmartResults.Date.ISOWeek()
+						month := postSmartResults.Date.Month()
+
+						yearStr := strconv.Itoa(year)
+						yearMonthStr := fmt.Sprintf("%d-%d", year, month)
+						yearWeekStr := fmt.Sprintf("%d-%d", year, week)
+
+						//write data to daily bucket if in the last 15 days
+						if postSmartResults.Date.After(dailyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: daily", preDevice.WWN)
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						//write data to the weekly bucket if in the last 9 weeks, and week has not been processed yet
+						if _, weekExists := weeklyLookup[yearWeekStr]; !weekExists && postSmartResults.Date.After(weeklyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: weekly", preDevice.WWN)
+
+							//this week/year pair has not been processed
+							weeklyLookup[yearWeekStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						//write data to the monthly bucket if in the last 9 weeks, and week has not been processed yet
+						if _, monthExists := monthlyLookup[yearMonthStr]; !monthExists && postSmartResults.Date.After(monthlyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: monthly", preDevice.WWN)
+
+							//this month/year pair has not been processed
+							monthlyLookup[yearMonthStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						if _, yearExists := yearlyLookup[yearStr]; !yearExists && year != today.Year() {
+							sr.logger.Debugf("device (%s) smart data added to bucket: yearly", preDevice.WWN)
+
+							//this year has not been processed
+							yearlyLookup[yearStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+					}
+					sr.logger.Infof("finished processing device %s. weekly: %d, monthly: %d, yearly: %d", preDevice.WWN, len(weeklyLookup), len(monthlyLookup), len(yearlyLookup))
+
+				}
+
+				return nil
+			},
+		},
+		{
+			ID: "20220503120000", // cleanup - v0.4.0 - influxdb schema
+			Migrate: func(tx *gorm.DB) error {
+				// delete unnecessary tables.
+				err := tx.Migrator().DropTable(
+					&m20201107210306.Smart{},
+					&m20201107210306.SmartAtaAttribute{},
+					&m20201107210306.SmartNvmeAttribute{},
+					&m20201107210306.SmartScsiAttribute{},
+				)
+				if err != nil {
+					return err
+				}
+
+				//migrate the device database to the final version
+				return tx.AutoMigrate(models.Device{})
+			},
+		},
+	})
+
+	if err := m.Migrate(); err != nil {
+		sr.logger.Errorf("Database migration failed with error: %w", err)
+		return err
+	}
+	sr.logger.Infoln("Database migration completed successfully")
+	return nil
+}
+
+// Deprecated
+func m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.SmartTemperature) {
+	//extract temperature data for every datapoint
+	postSmartTemp := measurements.SmartTemperature{
+		Date: preSmartResult.TestDate,
+		Temp: preSmartResult.Temp,
+	}
+
+	return nil, postSmartTemp
+}
+
+// Deprecated
+func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(database *gorm.DB, preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.Smart) {
+	//create a measurements.Smart object (which we will then push to the InfluxDB)
+	postDeviceSmartData := measurements.Smart{
+		Date:            preSmartResult.TestDate,
+		DeviceWWN:       preDevice.WWN,
+		DeviceProtocol:  preDevice.DeviceProtocol,
+		Temp:            preSmartResult.Temp,
+		PowerOnHours:    preSmartResult.PowerOnHours,
+		PowerCycleCount: preSmartResult.PowerCycleCount,
+
+		// this needs to be populated using measurements.Smart.ProcessAtaSmartInfo, ProcessScsiSmartInfo or ProcessNvmeSmartInfo
+		// because those functions will take into account thresholds (which we didn't consider correctly previously)
+		Attributes: map[string]measurements.SmartAttribute{},
+	}
+
+	result := database.Preload("AtaAttributes").Preload("NvmeAttributes").Preload("ScsiAttributes").Find(&preSmartResult)
+	if result.Error != nil {
+		return result.Error, postDeviceSmartData
+	}
+
+	if preDevice.IsAta() {
+		preAtaSmartAttributesTable := []collector.AtaSmartAttributesTableItem{}
+		for _, preAtaAttribute := range preSmartResult.AtaAttributes {
+			preAtaSmartAttributesTable = append(preAtaSmartAttributesTable, collector.AtaSmartAttributesTableItem{
+				ID:         preAtaAttribute.AttributeId,
+				Name:       preAtaAttribute.Name,
+				Value:      int64(preAtaAttribute.Value),
+				Worst:      int64(preAtaAttribute.Worst),
+				Thresh:     int64(preAtaAttribute.Threshold),
+				WhenFailed: preAtaAttribute.WhenFailed,
+				Flags: struct {
+					Value         int    `json:"value"`
+					String        string `json:"string"`
+					Prefailure    bool   `json:"prefailure"`
+					UpdatedOnline bool   `json:"updated_online"`
+					Performance   bool   `json:"performance"`
+					ErrorRate     bool   `json:"error_rate"`
+					EventCount    bool   `json:"event_count"`
+					AutoKeep      bool   `json:"auto_keep"`
+				}{
+					Value:         0,
+					String:        "",
+					Prefailure:    false,
+					UpdatedOnline: false,
+					Performance:   false,
+					ErrorRate:     false,
+					EventCount:    false,
+					AutoKeep:      false,
+				},
+				Raw: struct {
+					Value  int64  `json:"value"`
+					String string `json:"string"`
+				}{
+					Value:  preAtaAttribute.RawValue,
+					String: preAtaAttribute.RawString,
+				},
+			})
+		}
+
+		postDeviceSmartData.ProcessAtaSmartInfo(preAtaSmartAttributesTable)
+
+	} else if preDevice.IsNvme() {
+		//info collector.SmartInfo
+		postNvmeSmartHealthInformation := collector.NvmeSmartHealthInformationLog{}
+
+		for _, preNvmeAttribute := range preSmartResult.NvmeAttributes {
+			switch preNvmeAttribute.AttributeId {
+			case "critical_warning":
+				postNvmeSmartHealthInformation.CriticalWarning = int64(preNvmeAttribute.Value)
+			case "temperature":
+				postNvmeSmartHealthInformation.Temperature = int64(preNvmeAttribute.Value)
+			case "available_spare":
+				postNvmeSmartHealthInformation.AvailableSpare = int64(preNvmeAttribute.Value)
+			case "available_spare_threshold":
+				postNvmeSmartHealthInformation.AvailableSpareThreshold = int64(preNvmeAttribute.Value)
+			case "percentage_used":
+				postNvmeSmartHealthInformation.PercentageUsed = int64(preNvmeAttribute.Value)
+			case "data_units_read":
+				postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value)
+			case "data_units_written":
+				postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value)
+			case "host_reads":
+				postNvmeSmartHealthInformation.HostReads = int64(preNvmeAttribute.Value)
+			case "host_writes":
+				postNvmeSmartHealthInformation.HostWrites = int64(preNvmeAttribute.Value)
+			case "controller_busy_time":
+				postNvmeSmartHealthInformation.ControllerBusyTime = int64(preNvmeAttribute.Value)
+			case "power_cycles":
+				postNvmeSmartHealthInformation.PowerCycles = int64(preNvmeAttribute.Value)
+			case "power_on_hours":
+				postNvmeSmartHealthInformation.PowerOnHours = int64(preNvmeAttribute.Value)
+			case "unsafe_shutdowns":
+				postNvmeSmartHealthInformation.UnsafeShutdowns = int64(preNvmeAttribute.Value)
+			case "media_errors":
+				postNvmeSmartHealthInformation.MediaErrors = int64(preNvmeAttribute.Value)
+			case "num_err_log_entries":
+				postNvmeSmartHealthInformation.NumErrLogEntries = int64(preNvmeAttribute.Value)
+			case "warning_temp_time":
+				postNvmeSmartHealthInformation.WarningTempTime = int64(preNvmeAttribute.Value)
+			case "critical_comp_time":
+				postNvmeSmartHealthInformation.CriticalCompTime = int64(preNvmeAttribute.Value)
+			}
+		}
+
+		postDeviceSmartData.ProcessNvmeSmartInfo(postNvmeSmartHealthInformation)
+
+	} else if preDevice.IsScsi() {
+		//info collector.SmartInfo
+		var postScsiGrownDefectList int64
+		postScsiErrorCounterLog := collector.ScsiErrorCounterLog{
+			Read: struct {
+				ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+				ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+				ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+				TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+				CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+				GigabytesProcessed               string `json:"gigabytes_processed"`
+				TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+			}{},
+			Write: struct {
+				ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+				ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+				ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+				TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+				CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+				GigabytesProcessed               string `json:"gigabytes_processed"`
+				TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+			}{},
+		}
+
+		for _, preScsiAttribute := range preSmartResult.ScsiAttributes {
+			switch preScsiAttribute.AttributeId {
+			case "scsi_grown_defect_list":
+				postScsiGrownDefectList = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_eccfast":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_eccdelayed":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_rereads_rewrites":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value)
+			case "read.total_errors_corrected":
+				postScsiErrorCounterLog.Read.TotalErrorsCorrected = int64(preScsiAttribute.Value)
+			case "read.correction_algorithm_invocations":
+				postScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value)
+			case "read.total_uncorrected_errors":
+				postScsiErrorCounterLog.Read.TotalUncorrectedErrors = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_eccfast":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_eccdelayed":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_rereads_rewrites":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value)
+			case "write.total_errors_corrected":
+				postScsiErrorCounterLog.Write.TotalErrorsCorrected = int64(preScsiAttribute.Value)
+			case "write.correction_algorithm_invocations":
+				postScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value)
+			case "write.total_uncorrected_errors":
+				postScsiErrorCounterLog.Write.TotalUncorrectedErrors = int64(preScsiAttribute.Value)
+			}
+		}
+		postDeviceSmartData.ProcessScsiSmartInfo(postScsiGrownDefectList, postScsiErrorCounterLog)
+	} else {
+		return fmt.Errorf("Unknown device protocol: %s", preDevice.DeviceProtocol), postDeviceSmartData
+	}
+
+	return nil, postDeviceSmartData
+}

webapp/backend/pkg/database/scrutiny_repository_tasks.go

@@ -0,0 +1,122 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Tasks
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) error {
+	weeklyTaskName := "tsk-weekly-aggr"
+	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: weeklyTaskName}); findErr == nil && len(found) == 0 {
+		//weekly on Sunday at 1:00am
+		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, weeklyTaskName, sr.DownsampleScript("weekly"), "0 1 * * 0", orgID)
+		if err != nil {
+			return err
+		}
+	}
+
+	monthlyTaskName := "tsk-monthly-aggr"
+	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: monthlyTaskName}); findErr == nil && len(found) == 0 {
+		//monthly on first day of the month at 1:30am
+		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, monthlyTaskName, sr.DownsampleScript("monthly"), "30 1 1 * *", orgID)
+		if err != nil {
+			return err
+		}
+	}
+
+	yearlyTaskName := "tsk-yearly-aggr"
+	if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: yearlyTaskName}); findErr == nil && len(found) == 0 {
+		//yearly on the first day of the year at 2:00am
+		_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, yearlyTaskName, sr.DownsampleScript("yearly"), "0 2 1 1 *", orgID)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
+	var sourceBucket string // the source of the data
+	var destBucket string   // the destination for the aggregated data
+	var rangeStart string
+	var rangeEnd string
+	var aggWindow string
+	switch aggregationType {
+	case "weekly":
+		sourceBucket = sr.appConfig.GetString("web.influxdb.bucket")
+		destBucket = fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+		rangeStart = "-2w"
+		rangeEnd = "-1w"
+		aggWindow = "1w"
+	case "monthly":
+		sourceBucket = fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
+		destBucket = fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+		rangeStart = "-2mo"
+		rangeEnd = "-1mo"
+		aggWindow = "1mo"
+	case "yearly":
+		sourceBucket = fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
+		destBucket = fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
+		rangeStart = "-2y"
+		rangeEnd = "-1y"
+		aggWindow = "1y"
+	}
+
+	// TODO: using "last" function for aggregation. This should eventually be replaced with a more accurate represenation
+	/*
+	  import "types"
+	  smart_data = from(bucket: sourceBucket)
+	  |> range(start: rangeStart, stop: rangeEnd)
+	  |> filter(fn: (r) => r["_measurement"] == "smart" )
+	  |> group(columns: ["device_wwn", "_field"])
+
+	  non_numeric_smart_data = smart_data
+	    |> filter(fn: (r) => types.isType(v: r._value, type: "string") or types.isType(v: r._value, type: "bool"))
+	    |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+
+	  numeric_smart_data = smart_data
+	    |> filter(fn: (r) => types.isType(v: r._value, type: "int") or types.isType(v: r._value, type: "float"))
+	    |> aggregateWindow(every: aggWindow, fn: mean, createEmpty: false)
+
+	  union(tables: [non_numeric_smart_data, numeric_smart_data])
+	  |> to(bucket: destBucket, org: destOrg)
+
+	*/
+
+	return fmt.Sprintf(`
+  sourceBucket = "%s"
+  rangeStart = %s
+  rangeEnd = %s
+  aggWindow = %s
+  destBucket = "%s"
+  destOrg = "%s"
+
+  from(bucket: sourceBucket)
+  |> range(start: rangeStart, stop: rangeEnd)
+  |> filter(fn: (r) => r["_measurement"] == "smart" )
+  |> group(columns: ["device_wwn", "_field"])
+  |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+  |> to(bucket: destBucket, org: destOrg)
+
+  temp_data = from(bucket: sourceBucket)
+  |> range(start: rangeStart, stop: rangeEnd)
+  |> filter(fn: (r) => r["_measurement"] == "temp")
+  |> group(columns: ["device_wwn"])
+  |> toInt()
+
+  temp_data
+  |> aggregateWindow(fn: mean, every: aggWindow)
+  |> to(bucket: destBucket, org: destOrg)
+		`,
+		sourceBucket,
+		rangeStart,
+		rangeEnd,
+		aggWindow,
+		destBucket,
+		sr.appConfig.GetString("web.influxdb.org"),
+	)
+}

webapp/backend/pkg/database/scrutiny_repository_temperature.go

@@ -0,0 +1,167 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"strings"
+	"time"
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Temperature Data
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error {
+	if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 {
+
+		for ndx, temp := range collectorSmartData.AtaSctTemperatureHistory.Table {
+
+			minutesOffset := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * int64(ndx) * 60
+			smartTemp := measurements.SmartTemperature{
+				Date: time.Unix(collectorSmartData.LocalTime.TimeT-minutesOffset, 0),
+				Temp: temp,
+			}
+
+			tags, fields := smartTemp.Flatten()
+			tags["device_wwn"] = wwn
+			p := influxdb2.NewPoint("temp",
+				tags,
+				fields,
+				smartTemp.Date)
+			err := sr.influxWriteApi.WritePoint(ctx, p)
+			if err != nil {
+				return err
+			}
+		}
+		// also add the current temperature.
+	} else {
+
+		smartTemp := measurements.SmartTemperature{
+			Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0),
+			Temp: collectorSmartData.Temperature.Current,
+		}
+
+		tags, fields := smartTemp.Flatten()
+		tags["device_wwn"] = wwn
+		p := influxdb2.NewPoint("temp",
+			tags,
+			fields,
+			smartTemp.Date)
+		return sr.influxWriteApi.WritePoint(ctx, p)
+	}
+	return nil
+}
+
+func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) {
+	//we can get temp history for "week", "month", DURATION_KEY_YEAR, "forever"
+
+	deviceTempHistory := map[string][]measurements.SmartTemperature{}
+
+	//TODO: change the query range to a variable.
+	queryStr := sr.aggregateTempQuery(durationKey)
+
+	result, err := sr.influxQueryApi.Query(ctx, queryStr)
+	if err == nil {
+		// Use Next() to iterate over query result lines
+		for result.Next() {
+
+			if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
+
+				//check if deviceWWN has been seen and initialized already
+				if _, ok := deviceTempHistory[deviceWWN.(string)]; !ok {
+					deviceTempHistory[deviceWWN.(string)] = []measurements.SmartTemperature{}
+				}
+
+				currentTempHistory := deviceTempHistory[deviceWWN.(string)]
+				smartTemp := measurements.SmartTemperature{}
+
+				for key, val := range result.Record().Values() {
+					smartTemp.Inflate(key, val)
+				}
+				smartTemp.Date = result.Record().Values()["_time"].(time.Time)
+				currentTempHistory = append(currentTempHistory, smartTemp)
+				deviceTempHistory[deviceWWN.(string)] = currentTempHistory
+			}
+		}
+		if result.Err() != nil {
+			fmt.Printf("Query error: %s\n", result.Err().Error())
+		}
+	} else {
+		return nil, err
+	}
+	return deviceTempHistory, nil
+
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper Methods
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string {
+
+	/*
+		import "influxdata/influxdb/schema"
+		weekData = from(bucket: "metrics")
+		  |> range(start: -1w, stop: now())
+		  |> filter(fn: (r) => r["_measurement"] == "temp" )
+		  |> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+		  |> group(columns: ["device_wwn"])
+		  |> toInt()
+
+		monthData = from(bucket: "metrics_weekly")
+		  |> range(start: -1mo, stop: now())
+		  |> filter(fn: (r) => r["_measurement"] == "temp" )
+		  |> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
+		  |> group(columns: ["device_wwn"])
+		  |> toInt()
+
+		union(tables: [weekData, monthData])
+		  |> group(columns: ["device_wwn"])
+		  |> sort(columns: ["_time"], desc: false)
+		  |> schema.fieldsAsCols()
+
+	*/
+
+	partialQueryStr := []string{
+		`import "influxdata/influxdb/schema"`,
+	}
+
+	nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey)
+
+	subQueryNames := []string{}
+	for _, nestedDurationKey := range nestedDurationKeys {
+		bucketName := sr.lookupBucketName(nestedDurationKey)
+		durationRange := sr.lookupDuration(nestedDurationKey)
+
+		subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey))
+		partialQueryStr = append(partialQueryStr, []string{
+			fmt.Sprintf(`%sData = from(bucket: "%s")`, nestedDurationKey, bucketName),
+			fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
+			`|> filter(fn: (r) => r["_measurement"] == "temp" )`,
+			`|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)`,
+			`|> group(columns: ["device_wwn"])`,
+			`|> toInt()`,
+			"",
+		}...)
+	}
+
+	if len(subQueryNames) == 1 {
+		//there's only one bucket being queried, no need to union, just aggregate the dataset and return
+		partialQueryStr = append(partialQueryStr, []string{
+			subQueryNames[0],
+			"|> schema.fieldsAsCols()",
+			"|> yield()",
+		}...)
+	} else {
+		partialQueryStr = append(partialQueryStr, []string{
+			fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
+			`|> group(columns: ["device_wwn"])`,
+			`|> sort(columns: ["_time"], desc: false)`,
+			"|> schema.fieldsAsCols()",
+		}...)
+	}
+
+	return strings.Join(partialQueryStr, "\n")
+}

webapp/backend/pkg/models/collector/smart.go

@@ -119,30 +119,23 @@ type SmartInfo struct {
 		FeatureControlSupported       bool `json:"feature_control_supported"`
 		DataTableSupported            bool `json:"data_table_supported"`
 	} `json:"ata_sct_capabilities"`
+	AtaSctTemperatureHistory struct {
+		Version                int   `json:"version"`
+		SamplingPeriodMinutes  int64 `json:"sampling_period_minutes"`
+		LoggingIntervalMinutes int64 `json:"logging_interval_minutes"`
+		Temperature            struct {
+			OpLimitMin int `json:"op_limit_min"`
+			OpLimitMax int `json:"op_limit_max"`
+			LimitMin   int `json:"limit_min"`
+			LimitMax   int `json:"limit_max"`
+		} `json:"temperature"`
+		Size  int     `json:"size"`
+		Index int     `json:"index"`
+		Table []int64 `json:"table"`
+	} `json:"ata_sct_temperature_history"`
 	AtaSmartAttributes struct {
-		Revision int `json:"revision"`
-		Table    []struct {
-			ID         int    `json:"id"`
-			Name       string `json:"name"`
-			Value      int    `json:"value"`
-			Worst      int    `json:"worst"`
-			Thresh     int    `json:"thresh"`
-			WhenFailed string `json:"when_failed"`
-			Flags      struct {
-				Value         int    `json:"value"`
-				String        string `json:"string"`
-				Prefailure    bool   `json:"prefailure"`
-				UpdatedOnline bool   `json:"updated_online"`
-				Performance   bool   `json:"performance"`
-				ErrorRate     bool   `json:"error_rate"`
-				EventCount    bool   `json:"event_count"`
-				AutoKeep      bool   `json:"auto_keep"`
-			} `json:"flags"`
-			Raw struct {
-				Value  int64  `json:"value"`
-				String string `json:"string"`
-			} `json:"raw"`
-		} `json:"table"`
+		Revision int                           `json:"revision"`
+		Table    []AtaSmartAttributesTableItem `json:"table"`
 	} `json:"ata_smart_attributes"`
 	AtaSmartErrorLog struct {
 		Summary struct {
@@ -236,49 +229,77 @@ type SmartInfo struct {
 		} `json:"utilization"`
 		FormattedLbaSize int `json:"formatted_lba_size"`
 	} `json:"nvme_namespaces"`
-	NvmeSmartHealthInformationLog struct {
-		CriticalWarning         int `json:"critical_warning"`
-		Temperature             int `json:"temperature"`
-		AvailableSpare          int `json:"available_spare"`
-		AvailableSpareThreshold int `json:"available_spare_threshold"`
-		PercentageUsed          int `json:"percentage_used"`
-		DataUnitsRead           int `json:"data_units_read"`
-		DataUnitsWritten        int `json:"data_units_written"`
-		HostReads               int `json:"host_reads"`
-		HostWrites              int `json:"host_writes"`
-		ControllerBusyTime      int `json:"controller_busy_time"`
-		PowerCycles             int `json:"power_cycles"`
-		PowerOnHours            int `json:"power_on_hours"`
-		UnsafeShutdowns         int `json:"unsafe_shutdowns"`
-		MediaErrors             int `json:"media_errors"`
-		NumErrLogEntries        int `json:"num_err_log_entries"`
-		WarningTempTime         int `json:"warning_temp_time"`
-		CriticalCompTime        int `json:"critical_comp_time"`
-	} `json:"nvme_smart_health_information_log"`
+	NvmeSmartHealthInformationLog NvmeSmartHealthInformationLog `json:"nvme_smart_health_information_log"`
 
 	// SCSI Protocol Specific Fields
-	Vendor              string `json:"vendor"`
-	Product             string `json:"product"`
-	ScsiVersion         string `json:"scsi_version"`
-	ScsiGrownDefectList int    `json:"scsi_grown_defect_list"`
-	ScsiErrorCounterLog struct {
-		Read struct {
-			ErrorsCorrectedByEccfast         int    `json:"errors_corrected_by_eccfast"`
-			ErrorsCorrectedByEccdelayed      int    `json:"errors_corrected_by_eccdelayed"`
-			ErrorsCorrectedByRereadsRewrites int    `json:"errors_corrected_by_rereads_rewrites"`
-			TotalErrorsCorrected             int    `json:"total_errors_corrected"`
-			CorrectionAlgorithmInvocations   int    `json:"correction_algorithm_invocations"`
-			GigabytesProcessed               string `json:"gigabytes_processed"`
-			TotalUncorrectedErrors           int    `json:"total_uncorrected_errors"`
-		} `json:"read"`
-		Write struct {
-			ErrorsCorrectedByEccfast         int    `json:"errors_corrected_by_eccfast"`
-			ErrorsCorrectedByEccdelayed      int    `json:"errors_corrected_by_eccdelayed"`
-			ErrorsCorrectedByRereadsRewrites int    `json:"errors_corrected_by_rereads_rewrites"`
-			TotalErrorsCorrected             int    `json:"total_errors_corrected"`
-			CorrectionAlgorithmInvocations   int    `json:"correction_algorithm_invocations"`
-			GigabytesProcessed               string `json:"gigabytes_processed"`
-			TotalUncorrectedErrors           int    `json:"total_uncorrected_errors"`
-		} `json:"write"`
-	} `json:"scsi_error_counter_log"`
+	Vendor              string              `json:"vendor"`
+	Product             string              `json:"product"`
+	ScsiVersion         string              `json:"scsi_version"`
+	ScsiGrownDefectList int64               `json:"scsi_grown_defect_list"`
+	ScsiErrorCounterLog ScsiErrorCounterLog `json:"scsi_error_counter_log"`
+}
+
+//Primary Attribute Structs
+type AtaSmartAttributesTableItem struct {
+	ID         int    `json:"id"`
+	Name       string `json:"name"`
+	Value      int64  `json:"value"`
+	Worst      int64  `json:"worst"`
+	Thresh     int64  `json:"thresh"`
+	WhenFailed string `json:"when_failed"`
+	Flags      struct {
+		Value         int    `json:"value"`
+		String        string `json:"string"`
+		Prefailure    bool   `json:"prefailure"`
+		UpdatedOnline bool   `json:"updated_online"`
+		Performance   bool   `json:"performance"`
+		ErrorRate     bool   `json:"error_rate"`
+		EventCount    bool   `json:"event_count"`
+		AutoKeep      bool   `json:"auto_keep"`
+	} `json:"flags"`
+	Raw struct {
+		Value  int64  `json:"value"`
+		String string `json:"string"`
+	} `json:"raw"`
+}
+
+type NvmeSmartHealthInformationLog struct {
+	CriticalWarning         int64 `json:"critical_warning"`
+	Temperature             int64 `json:"temperature"`
+	AvailableSpare          int64 `json:"available_spare"`
+	AvailableSpareThreshold int64 `json:"available_spare_threshold"`
+	PercentageUsed          int64 `json:"percentage_used"`
+	DataUnitsRead           int64 `json:"data_units_read"`
+	DataUnitsWritten        int64 `json:"data_units_written"`
+	HostReads               int64 `json:"host_reads"`
+	HostWrites              int64 `json:"host_writes"`
+	ControllerBusyTime      int64 `json:"controller_busy_time"`
+	PowerCycles             int64 `json:"power_cycles"`
+	PowerOnHours            int64 `json:"power_on_hours"`
+	UnsafeShutdowns         int64 `json:"unsafe_shutdowns"`
+	MediaErrors             int64 `json:"media_errors"`
+	NumErrLogEntries        int64 `json:"num_err_log_entries"`
+	WarningTempTime         int64 `json:"warning_temp_time"`
+	CriticalCompTime        int64 `json:"critical_comp_time"`
+}
+
+type ScsiErrorCounterLog struct {
+	Read struct {
+		ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+		ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+		ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+		TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+		CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+		GigabytesProcessed               string `json:"gigabytes_processed"`
+		TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+	} `json:"read"`
+	Write struct {
+		ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+		ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+		ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+		TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+		CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+		GigabytesProcessed               string `json:"gigabytes_processed"`
+		TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+	} `json:"write"`
 }

webapp/backend/pkg/models/db/device.go

@@ -1,160 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"time"
-)
-
-type DeviceWrapper struct {
-	Success bool     `json:"success"`
-	Errors  []error  `json:"errors"`
-	Data    []Device `json:"data"`
-}
-
-const DeviceProtocolAta = "ATA"
-const DeviceProtocolScsi = "SCSI"
-const DeviceProtocolNvme = "NVMe"
-
-type Device struct {
-	//GORM attributes, see: http://gorm.io/docs/conventions.html
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt *time.Time
-
-	WWN    string `json:"wwn" gorm:"primary_key"`
-	HostId string `json:"host_id"`
-
-	DeviceName     string  `json:"device_name"`
-	Manufacturer   string  `json:"manufacturer"`
-	ModelName      string  `json:"model_name"`
-	InterfaceType  string  `json:"interface_type"`
-	InterfaceSpeed string  `json:"interface_speed"`
-	SerialNumber   string  `json:"serial_number"`
-	Firmware       string  `json:"firmware"`
-	RotationSpeed  int     `json:"rotational_speed"`
-	Capacity       int64   `json:"capacity"`
-	FormFactor     string  `json:"form_factor"`
-	SmartSupport   bool    `json:"smart_support"`
-	DeviceProtocol string  `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
-	DeviceType     string  `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
-	SmartResults   []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
-}
-
-func (dv *Device) IsAta() bool {
-	return dv.DeviceProtocol == DeviceProtocolAta
-}
-
-func (dv *Device) IsScsi() bool {
-	return dv.DeviceProtocol == DeviceProtocolScsi
-}
-
-func (dv *Device) IsNvme() bool {
-	return dv.DeviceProtocol == DeviceProtocolNvme
-}
-
-//This method requires a device with an array of SmartResults.
-//It will remove all SmartResults other than the first (the latest one)
-//All removed SmartResults, will be processed, grouping SmartAtaAttribute by attribute_id
-// and adding theme to an array called History.
-func (dv *Device) SquashHistory() error {
-	if len(dv.SmartResults) <= 1 {
-		return nil //no ataHistory found. ignore
-	}
-
-	latestSmartResultSlice := dv.SmartResults[0:1]
-	historicalSmartResultSlice := dv.SmartResults[1:]
-
-	//re-assign the latest slice to the SmartResults field
-	dv.SmartResults = latestSmartResultSlice
-
-	//process the historical slice for ATA data
-	if len(dv.SmartResults[0].AtaAttributes) > 0 {
-		ataHistory := map[int][]SmartAtaAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.AtaAttributes {
-				if _, ok := ataHistory[smartAttribute.AttributeId]; !ok {
-					ataHistory[smartAttribute.AttributeId] = []SmartAtaAttribute{}
-				}
-				ataHistory[smartAttribute.AttributeId] = append(ataHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].AtaAttributes {
-			if attributeHistory, ok := ataHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].AtaAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-
-	//process the historical slice for Nvme data
-	if len(dv.SmartResults[0].NvmeAttributes) > 0 {
-		nvmeHistory := map[string][]SmartNvmeAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.NvmeAttributes {
-				if _, ok := nvmeHistory[smartAttribute.AttributeId]; !ok {
-					nvmeHistory[smartAttribute.AttributeId] = []SmartNvmeAttribute{}
-				}
-				nvmeHistory[smartAttribute.AttributeId] = append(nvmeHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].NvmeAttributes {
-			if attributeHistory, ok := nvmeHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].NvmeAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-	//process the historical slice for Scsi data
-	if len(dv.SmartResults[0].ScsiAttributes) > 0 {
-		scsiHistory := map[string][]SmartScsiAttribute{}
-		for _, smartResult := range historicalSmartResultSlice {
-			for _, smartAttribute := range smartResult.ScsiAttributes {
-				if _, ok := scsiHistory[smartAttribute.AttributeId]; !ok {
-					scsiHistory[smartAttribute.AttributeId] = []SmartScsiAttribute{}
-				}
-				scsiHistory[smartAttribute.AttributeId] = append(scsiHistory[smartAttribute.AttributeId], smartAttribute)
-			}
-		}
-
-		//now assign the historical slices to the AtaAttributes in the latest SmartResults
-		for sandx, smartAttribute := range dv.SmartResults[0].ScsiAttributes {
-			if attributeHistory, ok := scsiHistory[smartAttribute.AttributeId]; ok {
-				dv.SmartResults[0].ScsiAttributes[sandx].History = attributeHistory
-			}
-		}
-	}
-	return nil
-}
-
-func (dv *Device) ApplyMetadataRules() error {
-
-	//embed metadata in the latest smart attributes object
-	if len(dv.SmartResults) > 0 {
-		for ndx, attr := range dv.SmartResults[0].AtaAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].AtaAttributes[ndx] = attr
-		}
-
-		for ndx, attr := range dv.SmartResults[0].NvmeAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].NvmeAttributes[ndx] = attr
-
-		}
-
-		for ndx, attr := range dv.SmartResults[0].ScsiAttributes {
-			attr.PopulateAttributeStatus()
-			dv.SmartResults[0].ScsiAttributes[ndx] = attr
-
-		}
-	}
-	return nil
-}
-
-// This function is called every time the collector sends SMART data to the API.
-// It can be used to update device data that can change over time.
-func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
-	dv.Firmware = info.FirmwareVersion
-	return nil
-}

webapp/backend/pkg/models/db/selftest.go

@@ -1,15 +0,0 @@
-package db
-
-import "time"
-
-type SelfTest struct {
-	//GORM attributes, see: http://gorm.io/docs/conventions.html
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt *time.Time
-
-	DeviceWWN string
-	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
-
-	Date time.Time
-}

webapp/backend/pkg/models/db/smart.go

@@ -1,127 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"gorm.io/gorm"
-	"time"
-)
-
-const SmartWhenFailedFailingNow = "FAILING_NOW"
-const SmartWhenFailedInThePast = "IN_THE_PAST"
-
-const SmartStatusPassed = "passed"
-const SmartStatusFailed = "failed"
-
-type Smart struct {
-	gorm.Model
-
-	DeviceWWN string `json:"device_wwn"`
-	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
-
-	TestDate    time.Time `json:"date"`
-	SmartStatus string    `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed
-
-	//Metrics
-	Temp            int64 `json:"temp"`
-	PowerOnHours    int64 `json:"power_on_hours"`
-	PowerCycleCount int64 `json:"power_cycle_count"`
-
-	AtaAttributes  []SmartAtaAttribute  `json:"ata_attributes" gorm:"foreignkey:SmartId"`
-	NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"`
-	ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"`
-}
-
-//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries)
-func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error {
-	sm.DeviceWWN = wwn
-	sm.TestDate = time.Unix(info.LocalTime.TimeT, 0)
-
-	//smart metrics
-	sm.Temp = info.Temperature.Current
-	sm.PowerCycleCount = info.PowerCycleCount
-	sm.PowerOnHours = info.PowerOnTime.Hours
-
-	// process ATA/NVME/SCSI protocol data
-	if info.Device.Protocol == DeviceProtocolAta {
-		sm.ProcessAtaSmartInfo(info)
-	} else if info.Device.Protocol == DeviceProtocolNvme {
-		sm.ProcessNvmeSmartInfo(info)
-	} else if info.Device.Protocol == DeviceProtocolScsi {
-		sm.ProcessScsiSmartInfo(info)
-	}
-
-	if info.SmartStatus.Passed {
-		sm.SmartStatus = SmartStatusPassed
-	} else {
-		sm.SmartStatus = SmartStatusFailed
-	}
-	return nil
-}
-
-//generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) {
-	sm.AtaAttributes = []SmartAtaAttribute{}
-	for _, collectorAttr := range info.AtaSmartAttributes.Table {
-		attrModel := SmartAtaAttribute{
-			AttributeId: collectorAttr.ID,
-			Name:        collectorAttr.Name,
-			Value:       collectorAttr.Value,
-			Worst:       collectorAttr.Worst,
-			Threshold:   collectorAttr.Thresh,
-			RawValue:    collectorAttr.Raw.Value,
-			RawString:   collectorAttr.Raw.String,
-			WhenFailed:  collectorAttr.WhenFailed,
-		}
-
-		//now that we've parsed the data from the smartctl response, lets match it against our metadata rules and add additional Scrutiny specific data.
-		if smartMetadata, ok := metadata.AtaMetadata[collectorAttr.ID]; ok {
-			attrModel.Name = smartMetadata.DisplayName
-			if smartMetadata.Transform != nil {
-				attrModel.TransformedValue = smartMetadata.Transform(attrModel.Value, attrModel.RawValue, attrModel.RawString)
-			}
-		}
-		sm.AtaAttributes = append(sm.AtaAttributes, attrModel)
-	}
-}
-
-//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) {
-	sm.NvmeAttributes = []SmartNvmeAttribute{
-		{AttributeId: "critical_warning", Name: "Critical Warning", Value: info.NvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0},
-		{AttributeId: "temperature", Name: "Temperature", Value: info.NvmeSmartHealthInformationLog.Temperature, Threshold: -1},
-		{AttributeId: "available_spare", Name: "Available Spare", Value: info.NvmeSmartHealthInformationLog.AvailableSpare, Threshold: info.NvmeSmartHealthInformationLog.AvailableSpareThreshold},
-		{AttributeId: "percentage_used", Name: "Percentage Used", Value: info.NvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100},
-		{AttributeId: "data_units_read", Name: "Data Units Read", Value: info.NvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1},
-		{AttributeId: "data_units_written", Name: "Data Units Written", Value: info.NvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1},
-		{AttributeId: "host_reads", Name: "Host Reads", Value: info.NvmeSmartHealthInformationLog.HostReads, Threshold: -1},
-		{AttributeId: "host_writes", Name: "Host Writes", Value: info.NvmeSmartHealthInformationLog.HostWrites, Threshold: -1},
-		{AttributeId: "controller_busy_time", Name: "Controller Busy Time", Value: info.NvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1},
-		{AttributeId: "power_cycles", Name: "Power Cycles", Value: info.NvmeSmartHealthInformationLog.PowerCycles, Threshold: -1},
-		{AttributeId: "power_on_hours", Name: "Power on Hours", Value: info.NvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1},
-		{AttributeId: "unsafe_shutdowns", Name: "Unsafe Shutdowns", Value: info.NvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1},
-		{AttributeId: "media_errors", Name: "Media Errors", Value: info.NvmeSmartHealthInformationLog.MediaErrors, Threshold: 0},
-		{AttributeId: "num_err_log_entries", Name: "Numb Err Log Entries", Value: info.NvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0},
-		{AttributeId: "warning_temp_time", Name: "Warning Temp Time", Value: info.NvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1},
-		{AttributeId: "critical_comp_time", Name: "Critical CompTime", Value: info.NvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1},
-	}
-}
-
-//generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessScsiSmartInfo(info collector.SmartInfo) {
-	sm.ScsiAttributes = []SmartScsiAttribute{
-		{AttributeId: "scsi_grown_defect_list", Name: "Grown Defect List", Value: info.ScsiGrownDefectList, Threshold: 0},
-		{AttributeId: "read.errors_corrected_by_eccfast", Name: "Read Errors Corrected by ECC Fast", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1},
-		{AttributeId: "read.errors_corrected_by_eccdelayed", Name: "Read Errors Corrected by ECC Delayed", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1},
-		{AttributeId: "read.errors_corrected_by_rereads_rewrites", Name: "Read Errors Corrected by ReReads/ReWrites", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0},
-		{AttributeId: "read.total_errors_corrected", Name: "Read Total Errors Corrected", Value: info.ScsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1},
-		{AttributeId: "read.correction_algorithm_invocations", Name: "Read Correction Algorithm Invocations", Value: info.ScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1},
-		{AttributeId: "read.total_uncorrected_errors", Name: "Read Total Uncorrected Errors", Value: info.ScsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0},
-		{AttributeId: "write.errors_corrected_by_eccfast", Name: "Write Errors Corrected by ECC Fast", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1},
-		{AttributeId: "write.errors_corrected_by_eccdelayed", Name: "Write Errors Corrected by ECC Delayed", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1},
-		{AttributeId: "write.errors_corrected_by_rereads_rewrites", Name: "Write Errors Corrected by ReReads/ReWrites", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0},
-		{AttributeId: "write.total_errors_corrected", Name: "Write Total Errors Corrected", Value: info.ScsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1},
-		{AttributeId: "write.correction_algorithm_invocations", Name: "Write Correction Algorithm Invocations", Value: info.ScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1},
-		{AttributeId: "write.total_uncorrected_errors", Name: "Write Total Uncorrected Errors", Value: info.ScsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0},
-	}
-}

webapp/backend/pkg/models/db/smart_ata_attribute.go

@@ -1,111 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-	"strings"
-)
-
-const SmartAttributeStatusPassed = "passed"
-const SmartAttributeStatusFailed = "failed"
-const SmartAttributeStatusWarning = "warn"
-
-type SmartAtaAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId int    `json:"attribute_id"`
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Worst       int    `json:"worst"`
-	Threshold   int    `json:"thresh"`
-	RawValue    int64  `json:"raw_value"`
-	RawString   string `json:"raw_string"`
-	WhenFailed  string `json:"when_failed"`
-
-	TransformedValue int64               `json:"transformed_value"`
-	Status           string              `gorm:"-" json:"status,omitempty"`
-	StatusReason     string              `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64             `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartAtaAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartAtaAttribute) PopulateAttributeStatus() {
-	if strings.ToUpper(sa.WhenFailed) == SmartWhenFailedFailingNow {
-		//this attribute has previously failed
-		sa.Status = SmartAttributeStatusFailed
-		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
-
-	} else if strings.ToUpper(sa.WhenFailed) == SmartWhenFailedInThePast {
-		sa.Status = SmartAttributeStatusWarning
-		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
-	}
-
-	if smartMetadata, ok := metadata.AtaMetadata[sa.AttributeId]; ok {
-		sa.MetadataObservedThresholdStatus(smartMetadata)
-	}
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}
-
-// compare the attribute (raw, normalized, transformed) value to observed thresholds, and update status if necessary
-func (sa *SmartAtaAttribute) MetadataObservedThresholdStatus(smartMetadata metadata.AtaAttributeMetadata) {
-	//TODO: multiple rules
-	// try to predict the failure rates for observed thresholds that have 0 failure rate and error bars.
-	// - if the attribute is critical
-	//		- the failure rate is over 10 - set to failed
-	//		- the attribute does not match any threshold, set to warn
-	// - if the attribute is not critical
-	//		- if failure rate is above 20 - set to failed
-	// 		- if failure rate is above 10 but below 20 - set to warn
-
-	//update the smart attribute status based on Observed thresholds.
-	var value int64
-	if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeNormalized {
-		value = int64(sa.Value)
-	} else if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeTransformed {
-		value = sa.TransformedValue
-	} else {
-		value = sa.RawValue
-	}
-
-	for _, obsThresh := range smartMetadata.ObservedThresholds {
-
-		//check if "value" is in this bucket
-		if ((obsThresh.Low == obsThresh.High) && value == obsThresh.Low) ||
-			(obsThresh.Low < value && value <= obsThresh.High) {
-			sa.FailureRate = obsThresh.AnnualFailureRate
-
-			if smartMetadata.Critical {
-				if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = SmartAttributeStatusFailed
-					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
-				}
-			} else {
-				if obsThresh.AnnualFailureRate >= 0.20 {
-					sa.Status = SmartAttributeStatusFailed
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
-				} else if obsThresh.AnnualFailureRate >= 0.10 {
-					sa.Status = SmartAttributeStatusWarning
-					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
-				}
-			}
-
-			//we've found the correct bucket, we can drop out of this loop
-			return
-		}
-	}
-	// no bucket found
-	if smartMetadata.Critical {
-		sa.Status = SmartAttributeStatusWarning
-		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
-	}
-
-	return
-}

webapp/backend/pkg/models/db/smart_nvme_attribute.go

@@ -1,46 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-)
-
-type SmartNvmeAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId string `json:"attribute_id"` //json string from smartctl
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Threshold   int    `json:"thresh"`
-
-	TransformedValue int64                `json:"transformed_value"`
-	Status           string               `gorm:"-" json:"status,omitempty"`
-	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartNvmeAttribute) PopulateAttributeStatus() {
-
-	//-1 is a special number meaning no threshold.
-	if sa.Threshold != -1 {
-		if smartMetadata, ok := metadata.NmveMetadata[sa.AttributeId]; ok {
-			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
-			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
-				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = SmartAttributeStatusFailed
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
-			}
-		}
-	}
-	//TODO: eventually figure out the critical_warning bits and determine correct error messages here.
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}

webapp/backend/pkg/models/db/smart_scsci_attribute.go

@@ -1,45 +0,0 @@
-package db
-
-import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	"gorm.io/gorm"
-)
-
-type SmartScsiAttribute struct {
-	gorm.Model
-
-	SmartId int    `json:"smart_id"`
-	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
-
-	AttributeId string `json:"attribute_id"` //json string from smartctl
-	Name        string `json:"name"`
-	Value       int    `json:"value"`
-	Threshold   int    `json:"thresh"`
-
-	TransformedValue int64                `json:"transformed_value"`
-	Status           string               `gorm:"-" json:"status,omitempty"`
-	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
-	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
-	History          []SmartScsiAttribute `gorm:"-" json:"history,omitempty"`
-}
-
-//populate attribute status, using SMART Thresholds & Observed Metadata
-func (sa *SmartScsiAttribute) PopulateAttributeStatus() {
-
-	//-1 is a special number meaning no threshold.
-	if sa.Threshold != -1 {
-		if smartMetadata, ok := metadata.NmveMetadata[sa.AttributeId]; ok {
-			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
-			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
-				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
-				sa.Status = SmartAttributeStatusFailed
-				sa.StatusReason = "Attribute is failing recommended SMART threshold"
-			}
-		}
-	}
-
-	//check if status is blank, set to "passed"
-	if len(sa.Status) == 0 {
-		sa.Status = SmartAttributeStatusPassed
-	}
-}

webapp/backend/pkg/models/db/smart_test.go

@@ -1,155 +0,0 @@
-package db_test
-
-import (
-	"encoding/json"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
-	"github.com/stretchr/testify/require"
-	"io/ioutil"
-	"os"
-	"testing"
-)
-
-func TestFromCollectorSmartInfo(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-ata.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 18, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-
-	//check that temperature was correctly parsed
-	for _, attr := range smartMdl.AtaAttributes {
-		if attr.AttributeId == 194 {
-			require.Equal(t, int64(163210330144), attr.RawValue)
-			require.Equal(t, int64(32), attr.TransformedValue)
-		}
-	}
-}
-
-func TestFromCollectorSmartInfo_Fail(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-fail.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "failed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-}
-
-func TestFromCollectorSmartInfo_Fail2(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-fail2.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "failed", smartMdl.SmartStatus)
-	require.Equal(t, 17, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-}
-
-func TestFromCollectorSmartInfo_Nvme(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-nvme.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 16, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 0, len(smartMdl.ScsiAttributes))
-
-	require.Equal(t, 111303174, smartMdl.NvmeAttributes[6].Value)
-	require.Equal(t, 83170961, smartMdl.NvmeAttributes[7].Value)
-}
-
-func TestFromCollectorSmartInfo_Scsi(t *testing.T) {
-	//setup
-	smartDataFile, err := os.Open("../testdata/smart-scsi.json")
-	require.NoError(t, err)
-	defer smartDataFile.Close()
-
-	var smartJson collector.SmartInfo
-
-	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
-	require.NoError(t, err)
-	err = json.Unmarshal(smartDataBytes, &smartJson)
-	require.NoError(t, err)
-
-	//test
-	smartMdl := db.Smart{}
-	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
-
-	//assert
-	require.NoError(t, err)
-	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
-	require.Equal(t, "passed", smartMdl.SmartStatus)
-	require.Equal(t, 0, len(smartMdl.AtaAttributes))
-	require.Equal(t, 0, len(smartMdl.NvmeAttributes))
-	require.Equal(t, 13, len(smartMdl.ScsiAttributes))
-
-	require.Equal(t, 56, smartMdl.ScsiAttributes[0].Value)
-	require.Equal(t, 300357663, smartMdl.ScsiAttributes[4].Value) //total_errors_corrected
-}

webapp/backend/pkg/models/device.go

@@ -0,0 +1,169 @@
+package models
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"time"
+)
+
+type DeviceWrapper struct {
+	Success bool     `json:"success"`
+	Errors  []error  `json:"errors"`
+	Data    []Device `json:"data"`
+}
+
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN string `json:"wwn" gorm:"primary_key"`
+
+	DeviceName     string `json:"device_name"`
+	Manufacturer   string `json:"manufacturer"`
+	ModelName      string `json:"model_name"`
+	InterfaceType  string `json:"interface_type"`
+	InterfaceSpeed string `json:"interface_speed"`
+	SerialNumber   string `json:"serial_number"`
+	Firmware       string `json:"firmware"`
+	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
+	DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string `json:"label"`
+	HostId string `json:"host_id"`
+
+	// Data set by Scrutiny
+	DeviceStatus pkg.DeviceStatus `json:"device_status"`
+}
+
+func (dv *Device) IsAta() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolAta
+}
+
+func (dv *Device) IsScsi() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolScsi
+}
+
+func (dv *Device) IsNvme() bool {
+	return dv.DeviceProtocol == pkg.DeviceProtocolNvme
+}
+
+//
+////This method requires a device with an array of SmartResults.
+////It will remove all SmartResults other than the first (the latest one)
+////All removed SmartResults, will be processed, grouping SmartAtaAttribute by attribute_id
+//// and adding theme to an array called History.
+//func (dv *Device) SquashHistory() error {
+//	if len(dv.SmartResults) <= 1 {
+//		return nil //no ataHistory found. ignore
+//	}
+//
+//	latestSmartResultSlice := dv.SmartResults[0:1]
+//	historicalSmartResultSlice := dv.SmartResults[1:]
+//
+//	//re-assign the latest slice to the SmartResults field
+//	dv.SmartResults = latestSmartResultSlice
+//
+//	//process the historical slice for ATA data
+//	if len(dv.SmartResults[0].AtaAttributes) > 0 {
+//		ataHistory := map[int][]SmartAtaAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.AtaAttributes {
+//				if _, ok := ataHistory[smartAttribute.AttributeId]; !ok {
+//					ataHistory[smartAttribute.AttributeId] = []SmartAtaAttribute{}
+//				}
+//				ataHistory[smartAttribute.AttributeId] = append(ataHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].AtaAttributes {
+//			if attributeHistory, ok := ataHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].AtaAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//
+//	//process the historical slice for Nvme data
+//	if len(dv.SmartResults[0].NvmeAttributes) > 0 {
+//		nvmeHistory := map[string][]SmartNvmeAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.NvmeAttributes {
+//				if _, ok := nvmeHistory[smartAttribute.AttributeId]; !ok {
+//					nvmeHistory[smartAttribute.AttributeId] = []SmartNvmeAttribute{}
+//				}
+//				nvmeHistory[smartAttribute.AttributeId] = append(nvmeHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].NvmeAttributes {
+//			if attributeHistory, ok := nvmeHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].NvmeAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//	//process the historical slice for Scsi data
+//	if len(dv.SmartResults[0].ScsiAttributes) > 0 {
+//		scsiHistory := map[string][]SmartScsiAttribute{}
+//		for _, smartResult := range historicalSmartResultSlice {
+//			for _, smartAttribute := range smartResult.ScsiAttributes {
+//				if _, ok := scsiHistory[smartAttribute.AttributeId]; !ok {
+//					scsiHistory[smartAttribute.AttributeId] = []SmartScsiAttribute{}
+//				}
+//				scsiHistory[smartAttribute.AttributeId] = append(scsiHistory[smartAttribute.AttributeId], smartAttribute)
+//			}
+//		}
+//
+//		//now assign the historical slices to the AtaAttributes in the latest SmartResults
+//		for sandx, smartAttribute := range dv.SmartResults[0].ScsiAttributes {
+//			if attributeHistory, ok := scsiHistory[smartAttribute.AttributeId]; ok {
+//				dv.SmartResults[0].ScsiAttributes[sandx].History = attributeHistory
+//			}
+//		}
+//	}
+//	return nil
+//}
+//
+//func (dv *Device) ApplyMetadataRules() error {
+//
+//	//embed metadata in the latest smart attributes object
+//	if len(dv.SmartResults) > 0 {
+//		for ndx, attr := range dv.SmartResults[0].AtaAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].AtaAttributes[ndx] = attr
+//		}
+//
+//		for ndx, attr := range dv.SmartResults[0].NvmeAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].NvmeAttributes[ndx] = attr
+//
+//		}
+//
+//		for ndx, attr := range dv.SmartResults[0].ScsiAttributes {
+//			attr.PopulateAttributeStatus()
+//			dv.SmartResults[0].ScsiAttributes[ndx] = attr
+//
+//		}
+//	}
+//	return nil
+//}
+
+// This function is called every time the collector sends SMART data to the API.
+// It can be used to update device data that can change over time.
+func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
+	dv.Firmware = info.FirmwareVersion
+	dv.DeviceProtocol = info.Device.Protocol
+
+	if !info.SmartStatus.Passed {
+		dv.DeviceStatus = pkg.Set(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
+	}
+
+	return nil
+}

webapp/backend/pkg/models/device_summary.go

@@ -0,0 +1,27 @@
+package models
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"time"
+)
+
+type DeviceSummaryWrapper struct {
+	Success bool    `json:"success"`
+	Errors  []error `json:"errors"`
+	Data    struct {
+		Summary map[string]*DeviceSummary `json:"summary"`
+	} `json:"data"`
+}
+
+type DeviceSummary struct {
+	Device Device `json:"device"`
+
+	SmartResults *SmartSummary                   `json:"smart,omitempty"`
+	TempHistory  []measurements.SmartTemperature `json:"temp_history,omitempty"`
+}
+type SmartSummary struct {
+	// Collector Summary Data
+	CollectorDate time.Time `json:"collector_date,omitempty"`
+	Temp          int64     `json:"temp,omitempty"`
+	PowerOnHours  int64     `json:"power_on_hours,omitempty"`
+}

webapp/backend/pkg/models/measurements/smart.go

@@ -0,0 +1,211 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"log"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Smart struct {
+	Date           time.Time `json:"date"`
+	DeviceWWN      string    `json:"device_wwn"` //(tag)
+	DeviceProtocol string    `json:"device_protocol"`
+
+	//Metrics (fields)
+	Temp            int64 `json:"temp"`
+	PowerOnHours    int64 `json:"power_on_hours"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+
+	//Attributes (fields)
+	Attributes map[string]SmartAttribute `json:"attrs"`
+
+	//status
+	Status pkg.DeviceStatus
+}
+
+func (sm *Smart) Flatten() (tags map[string]string, fields map[string]interface{}) {
+	tags = map[string]string{
+		"device_wwn":      sm.DeviceWWN,
+		"device_protocol": sm.DeviceProtocol,
+	}
+
+	fields = map[string]interface{}{
+		"temp":              sm.Temp,
+		"power_on_hours":    sm.PowerOnHours,
+		"power_cycle_count": sm.PowerCycleCount,
+	}
+
+	for _, attr := range sm.Attributes {
+		for attrKey, attrVal := range attr.Flatten() {
+			fields[attrKey] = attrVal
+		}
+	}
+
+	return tags, fields
+}
+
+func NewSmartFromInfluxDB(attrs map[string]interface{}) (*Smart, error) {
+	//go though the massive map returned from influxdb. If a key is associated with the Smart struct, assign it. If it starts with "attr.*" group it by attributeId, and pass to attribute inflate.
+
+	sm := Smart{
+		//required fields
+		Date:           attrs["_time"].(time.Time),
+		DeviceWWN:      attrs["device_wwn"].(string),
+		DeviceProtocol: attrs["device_protocol"].(string),
+
+		Attributes: map[string]SmartAttribute{},
+	}
+
+	for key, val := range attrs {
+		switch key {
+		case "temp":
+			sm.Temp = val.(int64)
+		case "power_on_hours":
+			sm.PowerOnHours = val.(int64)
+		case "power_cycle_count":
+			sm.PowerCycleCount = val.(int64)
+		default:
+			// this key is unknown.
+			if !strings.HasPrefix(key, "attr.") {
+				continue
+			}
+			//this is a attribute, lets group it with its related "siblings", populating a SmartAttribute object
+			keyParts := strings.Split(key, ".")
+			attributeId := keyParts[1]
+			if _, ok := sm.Attributes[attributeId]; !ok {
+				// init the attribute group
+				if sm.DeviceProtocol == pkg.DeviceProtocolAta {
+					sm.Attributes[attributeId] = &SmartAtaAttribute{}
+				} else if sm.DeviceProtocol == pkg.DeviceProtocolNvme {
+					sm.Attributes[attributeId] = &SmartNvmeAttribute{}
+				} else if sm.DeviceProtocol == pkg.DeviceProtocolScsi {
+					sm.Attributes[attributeId] = &SmartScsiAttribute{}
+				} else {
+					return nil, fmt.Errorf("Unknown Device Protocol: %s", sm.DeviceProtocol)
+				}
+			}
+
+			sm.Attributes[attributeId].Inflate(key, val)
+		}
+
+	}
+
+	log.Printf("Found Smart Device (%s) Attributes (%v)", sm.DeviceWWN, len(sm.Attributes))
+
+	return &sm, nil
+}
+
+//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries)
+func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error {
+	sm.DeviceWWN = wwn
+	sm.Date = time.Unix(info.LocalTime.TimeT, 0)
+
+	//smart metrics
+	sm.Temp = info.Temperature.Current
+	sm.PowerCycleCount = info.PowerCycleCount
+	sm.PowerOnHours = info.PowerOnTime.Hours
+	if !info.SmartStatus.Passed {
+		sm.Status = pkg.DeviceStatusFailedSmart
+	}
+
+	sm.DeviceProtocol = info.Device.Protocol
+	// process ATA/NVME/SCSI protocol data
+	sm.Attributes = map[string]SmartAttribute{}
+	if sm.DeviceProtocol == pkg.DeviceProtocolAta {
+		sm.ProcessAtaSmartInfo(info.AtaSmartAttributes.Table)
+	} else if sm.DeviceProtocol == pkg.DeviceProtocolNvme {
+		sm.ProcessNvmeSmartInfo(info.NvmeSmartHealthInformationLog)
+	} else if sm.DeviceProtocol == pkg.DeviceProtocolScsi {
+		sm.ProcessScsiSmartInfo(info.ScsiGrownDefectList, info.ScsiErrorCounterLog)
+	}
+
+	return nil
+}
+
+//generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTableItem) {
+	for _, collectorAttr := range tableItems {
+		attrModel := SmartAtaAttribute{
+			AttributeId: collectorAttr.ID,
+			Value:       collectorAttr.Value,
+			Worst:       collectorAttr.Worst,
+			Threshold:   collectorAttr.Thresh,
+			RawValue:    collectorAttr.Raw.Value,
+			RawString:   collectorAttr.Raw.String,
+			WhenFailed:  collectorAttr.WhenFailed,
+		}
+
+		//now that we've parsed the data from the smartctl response, lets match it against our metadata rules and add additional Scrutiny specific data.
+		if smartMetadata, ok := thresholds.AtaMetadata[collectorAttr.ID]; ok {
+			if smartMetadata.Transform != nil {
+				attrModel.TransformedValue = smartMetadata.Transform(attrModel.Value, attrModel.RawValue, attrModel.RawString)
+			}
+		}
+		attrModel.PopulateAttributeStatus()
+		sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel
+		if attrModel.Status == pkg.SmartAttributeStatusFailed {
+			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		}
+	}
+}
+
+//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.NvmeSmartHealthInformationLog) {
+
+	sm.Attributes = map[string]SmartAttribute{
+		"critical_warning":     (&SmartNvmeAttribute{AttributeId: "critical_warning", Value: nvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0}).PopulateAttributeStatus(),
+		"temperature":          (&SmartNvmeAttribute{AttributeId: "temperature", Value: nvmeSmartHealthInformationLog.Temperature, Threshold: -1}).PopulateAttributeStatus(),
+		"available_spare":      (&SmartNvmeAttribute{AttributeId: "available_spare", Value: nvmeSmartHealthInformationLog.AvailableSpare, Threshold: nvmeSmartHealthInformationLog.AvailableSpareThreshold}).PopulateAttributeStatus(),
+		"percentage_used":      (&SmartNvmeAttribute{AttributeId: "percentage_used", Value: nvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100}).PopulateAttributeStatus(),
+		"data_units_read":      (&SmartNvmeAttribute{AttributeId: "data_units_read", Value: nvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1}).PopulateAttributeStatus(),
+		"data_units_written":   (&SmartNvmeAttribute{AttributeId: "data_units_written", Value: nvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1}).PopulateAttributeStatus(),
+		"host_reads":           (&SmartNvmeAttribute{AttributeId: "host_reads", Value: nvmeSmartHealthInformationLog.HostReads, Threshold: -1}).PopulateAttributeStatus(),
+		"host_writes":          (&SmartNvmeAttribute{AttributeId: "host_writes", Value: nvmeSmartHealthInformationLog.HostWrites, Threshold: -1}).PopulateAttributeStatus(),
+		"controller_busy_time": (&SmartNvmeAttribute{AttributeId: "controller_busy_time", Value: nvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1}).PopulateAttributeStatus(),
+		"power_cycles":         (&SmartNvmeAttribute{AttributeId: "power_cycles", Value: nvmeSmartHealthInformationLog.PowerCycles, Threshold: -1}).PopulateAttributeStatus(),
+		"power_on_hours":       (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: nvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(),
+		"unsafe_shutdowns":     (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: nvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(),
+		"media_errors":         (&SmartNvmeAttribute{AttributeId: "media_errors", Value: nvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(),
+		"num_err_log_entries":  (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: nvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(),
+		"warning_temp_time":    (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: nvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(),
+		"critical_comp_time":   (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: nvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(),
+	}
+
+	//find analyzed attribute status
+	for _, val := range sm.Attributes {
+		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
+			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		}
+	}
+}
+
+//generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog) {
+	sm.Attributes = map[string]SmartAttribute{
+		"scsi_grown_defect_list":                     (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: defectGrownList, Threshold: 0}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_eccfast":           (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_eccdelayed":        (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_rereads_rewrites":  (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(),
+		"read_total_errors_corrected":                (&SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Value: scsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(),
+		"read_correction_algorithm_invocations":      (&SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Value: scsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(),
+		"read_total_uncorrected_errors":              (&SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Value: scsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_eccfast":          (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_eccdelayed":       (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(),
+		"write_total_errors_corrected":               (&SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Value: scsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(),
+		"write_correction_algorithm_invocations":     (&SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Value: scsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(),
+		"write_total_uncorrected_errors":             (&SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Value: scsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(),
+	}
+
+	//find analyzed attribute status
+	for _, val := range sm.Attributes {
+		if val.GetStatus() == pkg.SmartAttributeStatusFailed {
+			sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
+		}
+	}
+}

webapp/backend/pkg/models/measurements/smart_ata_attribute.go

@@ -0,0 +1,163 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"strconv"
+	"strings"
+)
+
+type SmartAtaAttribute struct {
+	AttributeId int    `json:"attribute_id"`
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+	Worst       int64  `json:"worst"`
+	RawValue    int64  `json:"raw_value"`
+	RawString   string `json:"raw_string"`
+	WhenFailed  string `json:"when_failed"`
+
+	//Generated data
+	TransformedValue int64   `json:"transformed_value"`
+	Status           int64   `json:"status"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartAtaAttribute) GetStatus() int64 {
+	return sa.Status
+}
+
+func (sa *SmartAtaAttribute) Flatten() map[string]interface{} {
+
+	idString := strconv.Itoa(sa.AttributeId)
+
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", idString): idString,
+		fmt.Sprintf("attr.%s.value", idString):        sa.Value,
+		fmt.Sprintf("attr.%s.worst", idString):        sa.Worst,
+		fmt.Sprintf("attr.%s.thresh", idString):       sa.Threshold,
+		fmt.Sprintf("attr.%s.raw_value", idString):    sa.RawValue,
+		fmt.Sprintf("attr.%s.raw_string", idString):   sa.RawString,
+		fmt.Sprintf("attr.%s.when_failed", idString):  sa.WhenFailed,
+
+		//Generated Data
+		fmt.Sprintf("attr.%s.transformed_value", idString): sa.TransformedValue,
+		fmt.Sprintf("attr.%s.status", idString):            sa.Status,
+		fmt.Sprintf("attr.%s.status_reason", idString):     sa.StatusReason,
+		fmt.Sprintf("attr.%s.failure_rate", idString):      sa.FailureRate,
+	}
+}
+func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		attrId, err := strconv.Atoi(val.(string))
+		if err == nil {
+			sa.AttributeId = attrId
+		}
+	case "value":
+		sa.Value = val.(int64)
+	case "worst":
+		sa.Worst = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+	case "raw_value":
+		sa.RawValue = val.(int64)
+	case "raw_string":
+		sa.RawString = val.(string)
+	case "when_failed":
+		sa.WhenFailed = val.(string)
+
+	//generated
+	case "transformed_value":
+		sa.TransformedValue = val.(int64)
+	case "status":
+		sa.Status = val.(int64)
+	case "status_reason":
+		sa.StatusReason = val.(string)
+	case "failure_rate":
+		sa.FailureRate = val.(float64)
+
+	}
+}
+
+//populate attribute status, using SMART Thresholds & Observed Metadata
+// Chainable
+func (sa *SmartAtaAttribute) PopulateAttributeStatus() *SmartAtaAttribute {
+	if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedFailingNow {
+		//this attribute has previously failed
+		sa.Status = pkg.SmartAttributeStatusFailed
+		sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
+
+	} else if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedInThePast {
+		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
+	}
+
+	if smartMetadata, ok := thresholds.AtaMetadata[sa.AttributeId]; ok {
+		sa.ValidateThreshold(smartMetadata)
+	}
+
+	return sa
+}
+
+// compare the attribute (raw, normalized, transformed) value to observed thresholds, and update status if necessary
+func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttributeMetadata) {
+	//TODO: multiple rules
+	// try to predict the failure rates for observed thresholds that have 0 failure rate and error bars.
+	// - if the attribute is critical
+	//		- the failure rate is over 10 - set to failed
+	//		- the attribute does not match any threshold, set to warn
+	// - if the attribute is not critical
+	//		- if failure rate is above 20 - set to failed
+	// 		- if failure rate is above 10 but below 20 - set to warn
+
+	//update the smart attribute status based on Observed thresholds.
+	var value int64
+	if smartMetadata.DisplayType == thresholds.AtaSmartAttributeDisplayTypeNormalized {
+		value = int64(sa.Value)
+	} else if smartMetadata.DisplayType == thresholds.AtaSmartAttributeDisplayTypeTransformed {
+		value = sa.TransformedValue
+	} else {
+		value = sa.RawValue
+	}
+
+	for _, obsThresh := range smartMetadata.ObservedThresholds {
+
+		//check if "value" is in this bucket
+		if ((obsThresh.Low == obsThresh.High) && value == obsThresh.Low) ||
+			(obsThresh.Low < value && value <= obsThresh.High) {
+			sa.FailureRate = obsThresh.AnnualFailureRate
+
+			if smartMetadata.Critical {
+				if obsThresh.AnnualFailureRate >= 0.10 {
+					sa.Status = pkg.SmartAttributeStatusFailed
+					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
+				}
+			} else {
+				if obsThresh.AnnualFailureRate >= 0.20 {
+					sa.Status = pkg.SmartAttributeStatusFailed
+					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
+				} else if obsThresh.AnnualFailureRate >= 0.10 {
+					sa.Status = pkg.SmartAttributeStatusWarning
+					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
+				}
+			}
+
+			//we've found the correct bucket, we can drop out of this loop
+			return
+		}
+	}
+	// no bucket found
+	if smartMetadata.Critical {
+		sa.Status = pkg.SmartAttributeStatusWarning
+		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
+	}
+
+	return
+}

webapp/backend/pkg/models/measurements/smart_attribute.go

@@ -0,0 +1,7 @@
+package measurements
+
+type SmartAttribute interface {
+	Flatten() (fields map[string]interface{})
+	Inflate(key string, val interface{})
+	GetStatus() int64
+}

webapp/backend/pkg/models/measurements/smart_nvme_attribute.go

@@ -0,0 +1,83 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"strings"
+)
+
+type SmartNvmeAttribute struct {
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+
+	TransformedValue int64   `json:"transformed_value"`
+	Status           int64   `json:"status"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartNvmeAttribute) GetStatus() int64 {
+	return sa.Status
+}
+
+func (sa *SmartNvmeAttribute) Flatten() map[string]interface{} {
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", sa.AttributeId): sa.AttributeId,
+		fmt.Sprintf("attr.%s.value", sa.AttributeId):        sa.Value,
+		fmt.Sprintf("attr.%s.thresh", sa.AttributeId):       sa.Threshold,
+
+		//Generated Data
+		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
+		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
+	}
+}
+func (sa *SmartNvmeAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		sa.AttributeId = val.(string)
+	case "value":
+		sa.Value = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+
+	//generated
+	case "transformed_value":
+		sa.TransformedValue = val.(int64)
+	case "status":
+		sa.Status = val.(int64)
+	case "status_reason":
+		sa.StatusReason = val.(string)
+	case "failure_rate":
+		sa.FailureRate = val.(float64)
+	}
+}
+
+//populate attribute status, using SMART Thresholds & Observed Metadata
+// Chainable
+func (sa *SmartNvmeAttribute) PopulateAttributeStatus() *SmartNvmeAttribute {
+
+	//-1 is a special number meaning no threshold.
+	if sa.Threshold != -1 {
+		if smartMetadata, ok := thresholds.NmveMetadata[sa.AttributeId]; ok {
+			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
+			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
+				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
+				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+			}
+		}
+	}
+	//TODO: eventually figure out the critical_warning bits and determine correct error messages here.
+
+	return sa
+}

webapp/backend/pkg/models/measurements/smart_scsci_attribute.go

@@ -0,0 +1,83 @@
+package measurements
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
+	"strings"
+)
+
+type SmartScsiAttribute struct {
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Value       int64  `json:"value"`
+	Threshold   int64  `json:"thresh"`
+
+	TransformedValue int64   `json:"transformed_value"`
+	Status           int64   `json:"status"`
+	StatusReason     string  `json:"status_reason,omitempty"`
+	FailureRate      float64 `json:"failure_rate,omitempty"`
+}
+
+func (sa *SmartScsiAttribute) GetStatus() int64 {
+	return sa.Status
+}
+
+func (sa *SmartScsiAttribute) Flatten() map[string]interface{} {
+	return map[string]interface{}{
+		fmt.Sprintf("attr.%s.attribute_id", sa.AttributeId): sa.AttributeId,
+		fmt.Sprintf("attr.%s.value", sa.AttributeId):        sa.Value,
+		fmt.Sprintf("attr.%s.thresh", sa.AttributeId):       sa.Threshold,
+
+		//Generated Data
+		fmt.Sprintf("attr.%s.transformed_value", sa.AttributeId): sa.TransformedValue,
+		fmt.Sprintf("attr.%s.status", sa.AttributeId):            sa.Status,
+		fmt.Sprintf("attr.%s.status_reason", sa.AttributeId):     sa.StatusReason,
+		fmt.Sprintf("attr.%s.failure_rate", sa.AttributeId):      sa.FailureRate,
+	}
+}
+func (sa *SmartScsiAttribute) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	keyParts := strings.Split(key, ".")
+
+	switch keyParts[2] {
+	case "attribute_id":
+		sa.AttributeId = val.(string)
+	case "value":
+		sa.Value = val.(int64)
+	case "thresh":
+		sa.Threshold = val.(int64)
+
+	//generated
+	case "transformed_value":
+		sa.TransformedValue = val.(int64)
+	case "status":
+		sa.Status = val.(int64)
+	case "status_reason":
+		sa.StatusReason = val.(string)
+	case "failure_rate":
+		sa.FailureRate = val.(float64)
+	}
+}
+
+//
+//populate attribute status, using SMART Thresholds & Observed Metadata
+//Chainable
+func (sa *SmartScsiAttribute) PopulateAttributeStatus() *SmartScsiAttribute {
+
+	//-1 is a special number meaning no threshold.
+	if sa.Threshold != -1 {
+		if smartMetadata, ok := thresholds.NmveMetadata[sa.AttributeId]; ok {
+			//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
+			if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
+				(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
+				sa.Status = pkg.SmartAttributeStatusFailed
+				sa.StatusReason = "Attribute is failing recommended SMART threshold"
+			}
+		}
+	}
+
+	return sa
+}

webapp/backend/pkg/models/measurements/smart_temperature.go

@@ -0,0 +1,34 @@
+package measurements
+
+import (
+	"time"
+)
+
+type SmartTemperature struct {
+	Date time.Time `json:"date"`
+	Temp int64     `json:"temp"`
+}
+
+func (st *SmartTemperature) Flatten() (tags map[string]string, fields map[string]interface{}) {
+	fields = map[string]interface{}{
+		"temp": st.Temp,
+	}
+	tags = map[string]string{}
+
+	return tags, fields
+}
+
+func (st *SmartTemperature) Inflate(key string, val interface{}) {
+	if val == nil {
+		return
+	}
+
+	if key == "temp" {
+		switch t := val.(type) {
+		case int64:
+			st.Temp = t
+		case float64:
+			st.Temp = int64(t)
+		}
+	}
+}

webapp/backend/pkg/models/measurements/smart_test.go

@@ -0,0 +1,436 @@
+package measurements_test
+
+import (
+	"encoding/json"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
+	"github.com/stretchr/testify/require"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestSmart_Flatten(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smart := measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  pkg.DeviceProtocolAta,
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Attributes:      nil,
+		Status:          0,
+	}
+
+	//test
+	tags, fields := smart.Flatten()
+
+	//assert
+	require.Equal(t, map[string]string{"device_protocol": "ATA", "device_wwn": "test-wwn"}, tags)
+	require.Equal(t, map[string]interface{}{"power_cycle_count": int64(10), "power_on_hours": int64(10), "temp": int64(50)}, fields)
+}
+
+func TestSmart_Flatten_ATA(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smart := measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  pkg.DeviceProtocolAta,
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Status:          0,
+		Attributes: map[string]measurements.SmartAttribute{
+			"1": &measurements.SmartAtaAttribute{
+				AttributeId: 1,
+				Value:       100,
+				Threshold:   1,
+				Worst:       100,
+				RawValue:    0,
+				RawString:   "0",
+				WhenFailed:  "",
+			},
+			"2": &measurements.SmartAtaAttribute{
+				AttributeId: 2,
+				Value:       135,
+				Threshold:   54,
+				Worst:       135,
+				RawValue:    108,
+				RawString:   "108",
+				WhenFailed:  "",
+			},
+		},
+	}
+
+	//test
+	tags, fields := smart.Flatten()
+
+	//assert
+	require.Equal(t, map[string]string{"device_protocol": "ATA", "device_wwn": "test-wwn"}, tags)
+	require.Equal(t, map[string]interface{}{
+		"attr.1.attribute_id":      "1",
+		"attr.1.failure_rate":      float64(0),
+		"attr.1.raw_string":        "0",
+		"attr.1.raw_value":         int64(0),
+		"attr.1.status":            int64(0),
+		"attr.1.status_reason":     "",
+		"attr.1.thresh":            int64(1),
+		"attr.1.transformed_value": int64(0),
+		"attr.1.value":             int64(100),
+		"attr.1.when_failed":       "",
+		"attr.1.worst":             int64(100),
+
+		"attr.2.attribute_id":      "2",
+		"attr.2.failure_rate":      float64(0),
+		"attr.2.raw_string":        "108",
+		"attr.2.raw_value":         int64(108),
+		"attr.2.status":            int64(0),
+		"attr.2.status_reason":     "",
+		"attr.2.thresh":            int64(54),
+		"attr.2.transformed_value": int64(0),
+		"attr.2.value":             int64(135),
+		"attr.2.when_failed":       "",
+		"attr.2.worst":             int64(135),
+
+		"power_cycle_count": int64(10),
+		"power_on_hours":    int64(10),
+		"temp":              int64(50),
+	}, fields)
+}
+
+func TestSmart_Flatten_SCSI(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smart := measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  pkg.DeviceProtocolScsi,
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Status:          0,
+		Attributes: map[string]measurements.SmartAttribute{
+			"read_errors_corrected_by_eccfast": &measurements.SmartScsiAttribute{
+				AttributeId: "read_errors_corrected_by_eccfast",
+				Value:       int64(300357663),
+			},
+		},
+	}
+
+	//test
+	tags, fields := smart.Flatten()
+
+	//assert
+	require.Equal(t, map[string]string{"device_protocol": "SCSI", "device_wwn": "test-wwn"}, tags)
+	require.Equal(t, map[string]interface{}{
+		"attr.read_errors_corrected_by_eccfast.attribute_id":      "read_errors_corrected_by_eccfast",
+		"attr.read_errors_corrected_by_eccfast.failure_rate":      float64(0),
+		"attr.read_errors_corrected_by_eccfast.status":            int64(0),
+		"attr.read_errors_corrected_by_eccfast.status_reason":     "",
+		"attr.read_errors_corrected_by_eccfast.thresh":            int64(0),
+		"attr.read_errors_corrected_by_eccfast.transformed_value": int64(0),
+		"attr.read_errors_corrected_by_eccfast.value":             int64(300357663),
+		"power_cycle_count": int64(10),
+		"power_on_hours":    int64(10),
+		"temp":              int64(50)},
+		fields)
+}
+
+func TestSmart_Flatten_NVMe(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	smart := measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  pkg.DeviceProtocolNvme,
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Status:          0,
+		Attributes: map[string]measurements.SmartAttribute{
+			"available_spare": &measurements.SmartNvmeAttribute{
+				AttributeId: "available_spare",
+				Value:       int64(100),
+			},
+		},
+	}
+
+	//test
+	tags, fields := smart.Flatten()
+
+	//assert
+	require.Equal(t, map[string]string{"device_protocol": "NVMe", "device_wwn": "test-wwn"}, tags)
+	require.Equal(t, map[string]interface{}{
+		"attr.available_spare.attribute_id":      "available_spare",
+		"attr.available_spare.failure_rate":      float64(0),
+		"attr.available_spare.status":            int64(0),
+		"attr.available_spare.status_reason":     "",
+		"attr.available_spare.thresh":            int64(0),
+		"attr.available_spare.transformed_value": int64(0),
+		"attr.available_spare.value":             int64(100),
+		"power_cycle_count":                      int64(10),
+		"power_on_hours":                         int64(10),
+		"temp":                                   int64(50)}, fields)
+}
+
+func TestNewSmartFromInfluxDB_ATA(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	attrs := map[string]interface{}{
+		"_time":                    timeNow,
+		"device_wwn":               "test-wwn",
+		"device_protocol":          pkg.DeviceProtocolAta,
+		"attr.1.attribute_id":      "1",
+		"attr.1.failure_rate":      float64(0),
+		"attr.1.raw_string":        "108",
+		"attr.1.raw_value":         int64(108),
+		"attr.1.status":            int64(0),
+		"attr.1.status_reason":     "",
+		"attr.1.thresh":            int64(54),
+		"attr.1.transformed_value": int64(0),
+		"attr.1.value":             int64(135),
+		"attr.1.when_failed":       "",
+		"attr.1.worst":             int64(135),
+		"power_cycle_count":        int64(10),
+		"power_on_hours":           int64(10),
+		"temp":                     int64(50),
+	}
+
+	//test
+	smart, err := measurements.NewSmartFromInfluxDB(attrs)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, &measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  "ATA",
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Attributes: map[string]measurements.SmartAttribute{
+			"1": &measurements.SmartAtaAttribute{
+				AttributeId: 1,
+				Value:       135,
+				Threshold:   54,
+				Worst:       135,
+				RawValue:    108,
+				RawString:   "108",
+				WhenFailed:  "",
+			},
+		}, Status: 0}, smart)
+}
+
+func TestNewSmartFromInfluxDB_NVMe(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	attrs := map[string]interface{}{
+		"_time":                                  timeNow,
+		"device_wwn":                             "test-wwn",
+		"device_protocol":                        pkg.DeviceProtocolNvme,
+		"attr.available_spare.attribute_id":      "available_spare",
+		"attr.available_spare.failure_rate":      float64(0),
+		"attr.available_spare.status":            int64(0),
+		"attr.available_spare.status_reason":     "",
+		"attr.available_spare.thresh":            int64(0),
+		"attr.available_spare.transformed_value": int64(0),
+		"attr.available_spare.value":             int64(100),
+		"power_cycle_count":                      int64(10),
+		"power_on_hours":                         int64(10),
+		"temp":                                   int64(50),
+	}
+
+	//test
+	smart, err := measurements.NewSmartFromInfluxDB(attrs)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, &measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  "NVMe",
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Attributes: map[string]measurements.SmartAttribute{
+			"available_spare": &measurements.SmartNvmeAttribute{
+				AttributeId: "available_spare",
+				Value:       int64(100),
+			},
+		}, Status: 0}, smart)
+}
+
+func TestNewSmartFromInfluxDB_SCSI(t *testing.T) {
+	//setup
+	timeNow := time.Now()
+	attrs := map[string]interface{}{
+		"_time":           timeNow,
+		"device_wwn":      "test-wwn",
+		"device_protocol": pkg.DeviceProtocolScsi,
+		"attr.read_errors_corrected_by_eccfast.attribute_id":      "read_errors_corrected_by_eccfast",
+		"attr.read_errors_corrected_by_eccfast.failure_rate":      float64(0),
+		"attr.read_errors_corrected_by_eccfast.status":            int64(0),
+		"attr.read_errors_corrected_by_eccfast.status_reason":     "",
+		"attr.read_errors_corrected_by_eccfast.thresh":            int64(0),
+		"attr.read_errors_corrected_by_eccfast.transformed_value": int64(0),
+		"attr.read_errors_corrected_by_eccfast.value":             int64(300357663),
+		"power_cycle_count": int64(10),
+		"power_on_hours":    int64(10),
+		"temp":              int64(50),
+	}
+
+	//test
+	smart, err := measurements.NewSmartFromInfluxDB(attrs)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, &measurements.Smart{
+		Date:            timeNow,
+		DeviceWWN:       "test-wwn",
+		DeviceProtocol:  "SCSI",
+		Temp:            50,
+		PowerOnHours:    10,
+		PowerCycleCount: 10,
+		Attributes: map[string]measurements.SmartAttribute{
+			"read_errors_corrected_by_eccfast": &measurements.SmartScsiAttribute{
+				AttributeId: "read_errors_corrected_by_eccfast",
+				Value:       int64(300357663),
+			},
+		}, Status: 0}, smart)
+}
+
+func TestFromCollectorSmartInfo(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-ata.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusPassed, smartMdl.Status)
+	require.Equal(t, 18, len(smartMdl.Attributes))
+
+	//check that temperature was correctly parsed
+
+	require.Equal(t, int64(163210330144), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).RawValue)
+	require.Equal(t, int64(32), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).TransformedValue)
+}
+
+func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-fail.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusFailedSmart, smartMdl.Status)
+	require.Equal(t, 0, len(smartMdl.Attributes))
+}
+
+func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-fail2.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusFailedScrutiny|pkg.DeviceStatusFailedSmart, smartMdl.Status)
+	require.Equal(t, 17, len(smartMdl.Attributes))
+}
+
+func TestFromCollectorSmartInfo_Nvme(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-nvme.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusPassed, smartMdl.Status)
+	require.Equal(t, 16, len(smartMdl.Attributes))
+
+	require.Equal(t, int64(111303174), smartMdl.Attributes["host_reads"].(*measurements.SmartNvmeAttribute).Value)
+	require.Equal(t, int64(83170961), smartMdl.Attributes["host_writes"].(*measurements.SmartNvmeAttribute).Value)
+}
+
+func TestFromCollectorSmartInfo_Scsi(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart-scsi.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := measurements.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
+	require.Equal(t, pkg.DeviceStatusPassed, smartMdl.Status)
+	require.Equal(t, 13, len(smartMdl.Attributes))
+
+	require.Equal(t, int64(56), smartMdl.Attributes["scsi_grown_defect_list"].(*measurements.SmartScsiAttribute).Value)
+	require.Equal(t, int64(300357663), smartMdl.Attributes["read_errors_corrected_by_eccfast"].(*measurements.SmartScsiAttribute).Value) //total_errors_corrected
+}

webapp/backend/pkg/models/setting.go

@@ -0,0 +1,5 @@
+package models
+
+// Temperature Format
+// Date Format
+// Device History window

webapp/backend/pkg/models/testdata/helper.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+func main() {
+
+	//webapp/backend/pkg/web/testdata/register-devices-req.json
+	devices := "webapp/backend/pkg/web/testdata/register-devices-req.json"
+
+	smartData := map[string][]string{
+		"0x5000cca264eb01d7": {"webapp/backend/pkg/models/testdata/smart-ata.json", "webapp/backend/pkg/models/testdata/smart-ata-date.json", "webapp/backend/pkg/models/testdata/smart-ata-date2.json"},
+		"0x5000cca264ec3183": {"webapp/backend/pkg/models/testdata/smart-fail2.json"},
+		"0x5002538e40a22954": {"webapp/backend/pkg/models/testdata/smart-nvme.json"},
+		"0x5000cca252c859cc": {"webapp/backend/pkg/models/testdata/smart-scsi.json"},
+		"0x5000cca264ebc248": {"webapp/backend/pkg/models/testdata/smart-scsi2.json"},
+	}
+
+	// send a post request to register devices
+	file, err := os.Open(devices)
+	if err != nil {
+		log.Fatalf("ERROR %v", err)
+	}
+	defer file.Close()
+	_, err = SendPostRequest("http://localhost:9090/api/devices/register", file)
+	if err != nil {
+		log.Fatalf("ERROR %v", err)
+	}
+	//
+
+	for diskId, smartDataFileNames := range smartData {
+		for _, smartDataFileName := range smartDataFileNames {
+			for daysToSubtract := 0; daysToSubtract <= 30; daysToSubtract++ { //add 4 weeks worth of data
+				smartDataReader, err := readSmartDataFileFixTimestamp(daysToSubtract, smartDataFileName)
+				if err != nil {
+					log.Fatalf("ERROR %v", err)
+				}
+
+				_, err = SendPostRequest(fmt.Sprintf("http://localhost:9090/api/device/%s/smart", diskId), smartDataReader)
+				if err != nil {
+					log.Fatalf("ERROR %v", err)
+				}
+			}
+
+		}
+
+	}
+
+}
+
+func SendPostRequest(url string, file io.Reader) ([]byte, error) {
+	response, err := http.Post(url, "application/json", file)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	log.Printf("%v\n", response.Status)
+
+	return ioutil.ReadAll(response.Body)
+}
+
+// InfluxDB will throw an error/ignore any submitted data with a timestamp older than the
+// retention period. Lets fix this by opening test files, modifying the timestamp and returning an io.Reader
+func readSmartDataFileFixTimestamp(daysToSubtract int, smartDataFilepath string) (io.Reader, error) {
+	metricsfile, err := os.Open(smartDataFilepath)
+	if err != nil {
+		return nil, err
+	}
+
+	metricsFileData, err := ioutil.ReadAll(metricsfile)
+	if err != nil {
+		return nil, err
+	}
+	//unmarshal because we need to change the timestamp
+	var smartData collector.SmartInfo
+	err = json.Unmarshal(metricsFileData, &smartData)
+	if err != nil {
+		return nil, err
+	}
+
+	daysToSubtractInHours := time.Duration(-1 * 24 * daysToSubtract)
+	smartData.LocalTime.TimeT = time.Now().Add(daysToSubtractInHours * time.Hour).Unix()
+	updatedSmartDataBytes, err := json.Marshal(smartData)
+
+	return bytes.NewReader(updatedSmartDataBytes), nil
+}

webapp/backend/pkg/models/testdata/smart-ata-date.json

@@ -0,0 +1,846 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.128-flatcar",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "-a",
+      "/dev/sdb"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/sdb",
+    "info_name": "/dev/sdb [SAT]",
+    "type": "sat",
+    "protocol": "ATA"
+  },
+  "model_name": "WDC WD140EDFZ-11A0VA0",
+  "serial_number": "9RK1XXXX",
+  "wwn": {
+    "naa": 5,
+    "oui": 3274,
+    "id": 10283057623
+  },
+  "firmware_version": "81.00A81",
+  "user_capacity": {
+    "blocks": 27344764928,
+    "bytes": 14000519643136
+  },
+  "logical_block_size": 512,
+  "physical_block_size": 4096,
+  "rotation_rate": 5400,
+  "form_factor": {
+    "ata_value": 2,
+    "name": "3.5 inches"
+  },
+  "in_smartctl_database": false,
+  "ata_version": {
+    "string": "ACS-2, ATA8-ACS T13/1699-D revision 4",
+    "major_value": 1020,
+    "minor_value": 41
+  },
+  "sata_version": {
+    "string": "SATA 3.2",
+    "value": 255
+  },
+  "interface_speed": {
+    "max": {
+      "sata_value": 14,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    },
+    "current": {
+      "sata_value": 3,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    }
+  },
+  "local_time": {
+    "time_t": 1637039918,
+    "asctime": "Sun Jun 30 00:03:30 2021 UTC"
+  },
+  "smart_status": {
+    "passed": true
+  },
+  "ata_smart_data": {
+    "offline_data_collection": {
+      "status": {
+        "value": 130,
+        "string": "was completed without error",
+        "passed": true
+      },
+      "completion_seconds": 101
+    },
+    "self_test": {
+      "status": {
+        "value": 241,
+        "string": "in progress, 10% remaining",
+        "remaining_percent": 10
+      },
+      "polling_minutes": {
+        "short": 2,
+        "extended": 1479
+      }
+    },
+    "capabilities": {
+      "values": [
+        91,
+        3
+      ],
+      "exec_offline_immediate_supported": true,
+      "offline_is_aborted_upon_new_cmd": false,
+      "offline_surface_scan_supported": true,
+      "self_tests_supported": true,
+      "conveyance_self_test_supported": false,
+      "selective_self_test_supported": true,
+      "attribute_autosave_enabled": true,
+      "error_logging_supported": true,
+      "gp_logging_supported": true
+    }
+  },
+  "ata_sct_capabilities": {
+    "value": 61,
+    "error_recovery_control_supported": true,
+    "feature_control_supported": true,
+    "data_table_supported": true
+  },
+  "ata_smart_attributes": {
+    "revision": 16,
+    "table": [
+      {
+        "id": 1,
+        "name": "Raw_Read_Error_Rate",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 11,
+          "string": "PO-R-- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 2,
+        "name": "Throughput_Performance",
+        "value": 135,
+        "worst": 135,
+        "thresh": 54,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 108,
+          "string": "108"
+        }
+      },
+      {
+        "id": 3,
+        "name": "Spin_Up_Time",
+        "value": 81,
+        "worst": 81,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 7,
+          "string": "POS--- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 30089675132,
+          "string": "380 (Average 380)"
+        }
+      },
+      {
+        "id": 4,
+        "name": "Start_Stop_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 9,
+          "string": "9"
+        }
+      },
+      {
+        "id": 5,
+        "name": "Reallocated_Sector_Ct",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 51,
+          "string": "PO--CK ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 7,
+        "name": "Seek_Error_Rate",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 8,
+        "name": "Seek_Time_Performance",
+        "value": 133,
+        "worst": 133,
+        "thresh": 20,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 18,
+          "string": "18"
+        }
+      },
+      {
+        "id": 9,
+        "name": "Power_On_Hours",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 1730,
+          "string": "1730"
+        }
+      },
+      {
+        "id": 10,
+        "name": "Spin_Retry_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 12,
+        "name": "Power_Cycle_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 9,
+          "string": "9"
+        }
+      },
+      {
+        "id": 22,
+        "name": "Unknown_Attribute",
+        "value": 100,
+        "worst": 100,
+        "thresh": 25,
+        "when_failed": "",
+        "flags": {
+          "value": 35,
+          "string": "PO---K ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 100,
+          "string": "100"
+        }
+      },
+      {
+        "id": 192,
+        "name": "Power-Off_Retract_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 329,
+          "string": "329"
+        }
+      },
+      {
+        "id": 193,
+        "name": "Load_Cycle_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 329,
+          "string": "329"
+        }
+      },
+      {
+        "id": 194,
+        "name": "Temperature_Celsius",
+        "value": 51,
+        "worst": 51,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 2,
+          "string": "-O---- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 163210330144,
+          "string": "32 (Min/Max 24/38)"
+        }
+      },
+      {
+        "id": 196,
+        "name": "Reallocated_Event_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 197,
+        "name": "Current_Pending_Sector",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 34,
+          "string": "-O---K ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 198,
+        "name": "Offline_Uncorrectable",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 8,
+          "string": "---R-- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 199,
+        "name": "UDMA_CRC_Error_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      }
+    ]
+  },
+  "power_on_time": {
+    "hours": 1730
+  },
+  "power_cycle_count": 9,
+  "temperature": {
+    "current": 32
+  },
+  "ata_smart_error_log": {
+    "summary": {
+      "revision": 1,
+      "count": 0
+    }
+  },
+  "ata_smart_self_test_log": {
+    "standard": {
+      "revision": 1,
+      "table": [
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1708
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1684
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1661
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1636
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1624
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1541
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1517
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1493
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1469
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1445
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1439
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1373
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1349
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1325
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1301
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1277
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1253
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1252
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1205
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1181
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1157
+        }
+      ],
+      "count": 21,
+      "error_count_total": 0,
+      "error_count_outdated": 0
+    }
+  },
+  "ata_smart_selective_self_test_log": {
+    "revision": 1,
+    "table": [
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      }
+    ],
+    "flags": {
+      "value": 0,
+      "remainder_scan_enabled": false
+    },
+    "power_up_scan_resume_minutes": 0
+  }
+}

webapp/backend/pkg/models/testdata/smart-ata-date2.json

@@ -0,0 +1,846 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.128-flatcar",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "-a",
+      "/dev/sdb"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/sdb",
+    "info_name": "/dev/sdb [SAT]",
+    "type": "sat",
+    "protocol": "ATA"
+  },
+  "model_name": "WDC WD140EDFZ-11A0VA0",
+  "serial_number": "9RK1XXXX",
+  "wwn": {
+    "naa": 5,
+    "oui": 3274,
+    "id": 10283057623
+  },
+  "firmware_version": "81.00A81",
+  "user_capacity": {
+    "blocks": 27344764928,
+    "bytes": 14000519643136
+  },
+  "logical_block_size": 512,
+  "physical_block_size": 4096,
+  "rotation_rate": 5400,
+  "form_factor": {
+    "ata_value": 2,
+    "name": "3.5 inches"
+  },
+  "in_smartctl_database": false,
+  "ata_version": {
+    "string": "ACS-2, ATA8-ACS T13/1699-D revision 4",
+    "major_value": 1020,
+    "minor_value": 41
+  },
+  "sata_version": {
+    "string": "SATA 3.2",
+    "value": 255
+  },
+  "interface_speed": {
+    "max": {
+      "sata_value": 14,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    },
+    "current": {
+      "sata_value": 3,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    }
+  },
+  "local_time": {
+    "time_t": 1637039918,
+    "asctime": "Tue Feb 23 00:03:30 2021 UTC"
+  },
+  "smart_status": {
+    "passed": true
+  },
+  "ata_smart_data": {
+    "offline_data_collection": {
+      "status": {
+        "value": 130,
+        "string": "was completed without error",
+        "passed": true
+      },
+      "completion_seconds": 101
+    },
+    "self_test": {
+      "status": {
+        "value": 241,
+        "string": "in progress, 10% remaining",
+        "remaining_percent": 10
+      },
+      "polling_minutes": {
+        "short": 2,
+        "extended": 1479
+      }
+    },
+    "capabilities": {
+      "values": [
+        91,
+        3
+      ],
+      "exec_offline_immediate_supported": true,
+      "offline_is_aborted_upon_new_cmd": false,
+      "offline_surface_scan_supported": true,
+      "self_tests_supported": true,
+      "conveyance_self_test_supported": false,
+      "selective_self_test_supported": true,
+      "attribute_autosave_enabled": true,
+      "error_logging_supported": true,
+      "gp_logging_supported": true
+    }
+  },
+  "ata_sct_capabilities": {
+    "value": 61,
+    "error_recovery_control_supported": true,
+    "feature_control_supported": true,
+    "data_table_supported": true
+  },
+  "ata_smart_attributes": {
+    "revision": 16,
+    "table": [
+      {
+        "id": 1,
+        "name": "Raw_Read_Error_Rate",
+        "value": 90,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 11,
+          "string": "PO-R-- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 2,
+        "name": "Throughput_Performance",
+        "value": 125,
+        "worst": 135,
+        "thresh": 54,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 118,
+          "string": "108"
+        }
+      },
+      {
+        "id": 3,
+        "name": "Spin_Up_Time",
+        "value": 71,
+        "worst": 81,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 7,
+          "string": "POS--- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 30089675142,
+          "string": "380 (Average 380)"
+        }
+      },
+      {
+        "id": 4,
+        "name": "Start_Stop_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 19,
+          "string": "9"
+        }
+      },
+      {
+        "id": 5,
+        "name": "Reallocated_Sector_Ct",
+        "value": 90,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 51,
+          "string": "PO--CK ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 7,
+        "name": "Seek_Error_Rate",
+        "value": 90,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 8,
+        "name": "Seek_Time_Performance",
+        "value": 123,
+        "worst": 133,
+        "thresh": 20,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 28,
+          "string": "18"
+        }
+      },
+      {
+        "id": 9,
+        "name": "Power_On_Hours",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 1740,
+          "string": "1730"
+        }
+      },
+      {
+        "id": 10,
+        "name": "Spin_Retry_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 12,
+        "name": "Power_Cycle_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 19,
+          "string": "9"
+        }
+      },
+      {
+        "id": 22,
+        "name": "Unknown_Attribute",
+        "value": 90,
+        "worst": 100,
+        "thresh": 25,
+        "when_failed": "",
+        "flags": {
+          "value": 35,
+          "string": "PO---K ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 110,
+          "string": "100"
+        }
+      },
+      {
+        "id": 192,
+        "name": "Power-Off_Retract_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 339,
+          "string": "329"
+        }
+      },
+      {
+        "id": 193,
+        "name": "Load_Cycle_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 339,
+          "string": "329"
+        }
+      },
+      {
+        "id": 194,
+        "name": "Temperature_Celsius",
+        "value": 41,
+        "worst": 51,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 2,
+          "string": "-O---- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 163210330154,
+          "string": "32 (Min/Max 24/38)"
+        }
+      },
+      {
+        "id": 196,
+        "name": "Reallocated_Event_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 197,
+        "name": "Current_Pending_Sector",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 34,
+          "string": "-O---K ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 198,
+        "name": "Offline_Uncorrectable",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 8,
+          "string": "---R-- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      },
+      {
+        "id": 199,
+        "name": "UDMA_CRC_Error_Count",
+        "value": 90,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 10,
+          "string": "0"
+        }
+      }
+    ]
+  },
+  "power_on_time": {
+    "hours": 3030
+  },
+  "power_cycle_count": 9,
+  "temperature": {
+    "current": 62
+  },
+  "ata_smart_error_log": {
+    "summary": {
+      "revision": 1,
+      "count": 0
+    }
+  },
+  "ata_smart_self_test_log": {
+    "standard": {
+      "revision": 1,
+      "table": [
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1708
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1684
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1661
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1636
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1624
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1541
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1517
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1493
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1469
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1445
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1439
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1373
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1349
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1325
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1301
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1277
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1253
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1252
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1205
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1181
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1157
+        }
+      ],
+      "count": 21,
+      "error_count_total": 0,
+      "error_count_outdated": 0
+    }
+  },
+  "ata_smart_selective_self_test_log": {
+    "revision": 1,
+    "table": [
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      }
+    ],
+    "flags": {
+      "value": 0,
+      "remainder_scan_enabled": false
+    },
+    "power_up_scan_resume_minutes": 0
+  }
+}

webapp/backend/pkg/models/testdata/smart-ata-full.json

@@ -70,7 +70,7 @@
     }
   },
   "local_time": {
-    "time_t": 1600014563,
+    "time_t": 1637039918,
     "asctime": "Sun Sep 13 16:29:23 2020 UTC"
   },
   "read_lookahead": {

webapp/backend/pkg/models/testdata/smart-ata.json

@@ -69,7 +69,7 @@
     }
   },
   "local_time": {
-    "time_t": 1592697810,
+    "time_t": 1637039918,
     "asctime": "Sun Jun 21 00:03:30 2020 UTC"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-ata2.json

@@ -66,7 +66,7 @@
     }
   },
   "local_time": {
-    "time_t": 1564664713,
+    "time_t": 1637039918,
     "asctime": "Thu Aug 01 15:05:13 2019 WEDT"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-fail2.json

@@ -70,7 +70,7 @@
     }
   },
   "local_time": {
-    "time_t": 1594216103,
+    "time_t": 1637039918,
     "asctime": "Wed Jul  8 15:48:23 2020 CEST"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-megaraid0.json

@@ -79,7 +79,7 @@
     }
   },
   "local_time": {
-    "time_t": 1598297918,
+    "time_t": 1637039918,
     "asctime": "Mon Aug 24 21:38:38 2020 CEST"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-megaraid1.json

@@ -79,7 +79,7 @@
     }
   },
   "local_time": {
-    "time_t": 1598297922,
+    "time_t": 1637039918,
     "asctime": "Mon Aug 24 21:38:42 2020 CEST"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-nvme.json

@@ -59,7 +59,7 @@
   },
   "logical_block_size": 512,
   "local_time": {
-    "time_t": 1591790462,
+    "time_t": 1637039918,
     "asctime": "Wed Jun 10 14:01:02 2020 CEST"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-nvme2.json

@@ -67,7 +67,7 @@
   },
   "logical_block_size": 512,
   "local_time": {
-    "time_t": 1600619090,
+    "time_t": 1637039918,
     "asctime": "Sun Sep 20 16:24:50 2020 Europe"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-raid.json

@@ -43,7 +43,7 @@
     "name": "disk"
   },
   "local_time": {
-    "time_t": 1570609867,
+    "time_t": 1637039918,
     "asctime": "Wed Oct 09 10:31:07 2019 RDT"
   },
   "temperature": {

webapp/backend/pkg/models/testdata/smart-scsi.json

@@ -26,7 +26,7 @@
     "name": "disk"
   },
   "local_time": {
-    "time_t": 1598048822,
+    "time_t": 1637039918,
     "asctime": "Fri Aug 21 22:27:02 2020 UTC"
   },
   "smart_status": {

webapp/backend/pkg/models/testdata/smart-scsi2.json

@@ -44,7 +44,7 @@
     "name": "disk"
   },
   "local_time": {
-    "time_t": 1545001755,
+    "time_t": 1637039918,
     "asctime": "Sun Dec 16 17:09:15 2018 CST"
   },
   "smart_status": {

webapp/backend/pkg/notify/notify.go

@@ -258,7 +258,6 @@ func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *sho
 		(*params)["title"] = subject
 	case "slack":
 		(*params)["title"] = subject
-		(*params)["thumb_url"] = logoUrl
 	case "smtp":
 		(*params)["subject"] = subject
 	case "standard":

webapp/backend/pkg/thresholds/ata_attribute_metadata.go

@@ -1,4 +1,4 @@
-package metadata
+package thresholds
 
 const AtaSmartAttributeDisplayTypeRaw = "raw"
 const AtaSmartAttributeDisplayTypeNormalized = "normalized"
@@ -6,15 +6,15 @@ const AtaSmartAttributeDisplayTypeTransformed = "transformed"
 
 type AtaAttributeMetadata struct {
 	ID          int64  `json:"-"`
-	DisplayName string `json:"-"`
+	DisplayName string `json:"display_name"`
 	Ideal       string `json:"ideal"`
 	Critical    bool   `json:"critical"`
 	Description string `json:"description"`
 
-	Transform          func(int, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
-	TransformValueUnit string                         `json:"transform_value_unit,omitempty"`
-	ObservedThresholds []ObservedThreshold            `json:"observed_thresholds,omitempty"` //these thresholds must match the DisplayType
-	DisplayType        string                         `json:"display_type"`                  //"raw" "normalized" or "transformed"
+	Transform          func(int64, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
+	TransformValueUnit string                           `json:"transform_value_unit,omitempty"`
+	ObservedThresholds []ObservedThreshold              `json:"observed_thresholds,omitempty"` //these thresholds must match the DisplayType
+	DisplayType        string                           `json:"display_type"`                  //"raw" "normalized" or "transformed"
 }
 
 const ObservedThresholdIdealLow = "low"
@@ -1014,7 +1014,7 @@ var AtaMetadata = map[int]AtaAttributeMetadata{
 		Ideal:       ObservedThresholdIdealLow,
 		Critical:    false,
 		Description: "Indicates the device temperature, if the appropriate sensor is fitted. Lowest byte of the raw value contains the exact temperature value (Celsius degrees).",
-		Transform: func(normValue int, rawValue int64, rawString string) int64 {
+		Transform: func(normValue int64, rawValue int64, rawString string) int64 {
 			return rawValue & 0b11111111
 		},
 		TransformValueUnit: "°C",

webapp/backend/pkg/thresholds/nvme_attribute_metadata.go

@@ -1,4 +1,4 @@
-package metadata
+package thresholds
 
 // https://media.kingston.com/support/downloads/MKP_521.6_SMART-DCP1000_attribute.pdf
 // https://www.percona.com/blog/2017/02/09/using-nvme-command-line-tools-to-check-nvme-flash-health/
@@ -6,14 +6,14 @@ package metadata
 // https://www.micromat.com/product_manuals/drive_scope_manual_01.pdf
 type NvmeAttributeMetadata struct {
 	ID          string `json:"-"`
-	DisplayName string `json:"-"`
+	DisplayName string `json:"display_name"`
 	Ideal       string `json:"ideal"`
 	Critical    bool   `json:"critical"`
 	Description string `json:"description"`
 
-	Transform          func(int, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
-	TransformValueUnit string                         `json:"transform_value_unit,omitempty"`
-	DisplayType        string                         `json:"display_type"` //"raw" "normalized" or "transformed"
+	Transform          func(int64, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
+	TransformValueUnit string                           `json:"transform_value_unit,omitempty"`
+	DisplayType        string                           `json:"display_type"` //"raw" "normalized" or "transformed"
 }
 
 var NmveMetadata = map[string]NvmeAttributeMetadata{

webapp/backend/pkg/thresholds/scsi_attribute_metadata.go

@@ -1,15 +1,15 @@
-package metadata
+package thresholds
 
 type ScsiAttributeMetadata struct {
 	ID          string `json:"-"`
-	DisplayName string `json:"-"`
+	DisplayName string `json:"display_name"`
 	Ideal       string `json:"ideal"`
 	Critical    bool   `json:"critical"`
 	Description string `json:"description"`
 
-	Transform          func(int, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
-	TransformValueUnit string                         `json:"transform_value_unit,omitempty"`
-	DisplayType        string                         `json:"display_type"` //"raw" "normalized" or "transformed"
+	Transform          func(int64, int64, string) int64 `json:"-"` //this should be a method to extract/tranform the normalized or raw data to a chartable format. Str
+	TransformValueUnit string                           `json:"transform_value_unit,omitempty"`
+	DisplayType        string                           `json:"display_type"` //"raw" "normalized" or "transformed"
 }
 
 var ScsiMetadata = map[string]ScsiAttributeMetadata{
@@ -21,96 +21,96 @@ var ScsiMetadata = map[string]ScsiAttributeMetadata{
 		Critical:    true,
 		Description: "",
 	},
-	"read.errors_corrected_by_eccfast": {
-		ID:          "read.errors_corrected_by_eccfast",
+	"read_errors_corrected_by_eccfast": {
+		ID:          "read_errors_corrected_by_eccfast",
 		DisplayName: "Read Errors Corrected by ECC Fast",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"read.errors_corrected_by_eccdelayed": {
-		ID:          "read.errors_corrected_by_eccdelayed",
+	"read_errors_corrected_by_eccdelayed": {
+		ID:          "read_errors_corrected_by_eccdelayed",
 		DisplayName: "Read Errors Corrected by ECC Delayed",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"read.errors_corrected_by_rereads_rewrites": {
-		ID:          "read.errors_corrected_by_rereads_rewrites",
+	"read_errors_corrected_by_rereads_rewrites": {
+		ID:          "read_errors_corrected_by_rereads_rewrites",
 		DisplayName: "Read Errors Corrected by ReReads/ReWrites",
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
 		Description: "",
 	},
-	"read.total_errors_corrected": {
-		ID:          "read.total_errors_corrected",
+	"read_total_errors_corrected": {
+		ID:          "read_total_errors_corrected",
 		DisplayName: "Read Total Errors Corrected",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"read.correction_algorithm_invocations": {
-		ID:          "read.correction_algorithm_invocations",
+	"read_correction_algorithm_invocations": {
+		ID:          "read_correction_algorithm_invocations",
 		DisplayName: "Read Correction Algorithm Invocations",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"read.total_uncorrected_errors": {
-		ID:          "read.total_uncorrected_errors",
+	"read_total_uncorrected_errors": {
+		ID:          "read_total_uncorrected_errors",
 		DisplayName: "Read Total Uncorrected Errors",
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
 		Description: "",
 	},
-	"write.errors_corrected_by_eccfast": {
-		ID:          "write.errors_corrected_by_eccfast",
+	"write_errors_corrected_by_eccfast": {
+		ID:          "write_errors_corrected_by_eccfast",
 		DisplayName: "Write Errors Corrected by ECC Fast",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"write.errors_corrected_by_eccdelayed": {
-		ID:          "write.errors_corrected_by_eccdelayed",
+	"write_errors_corrected_by_eccdelayed": {
+		ID:          "write_errors_corrected_by_eccdelayed",
 		DisplayName: "Write Errors Corrected by ECC Delayed",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"write.errors_corrected_by_rereads_rewrites": {
-		ID:          "write.errors_corrected_by_rereads_rewrites",
+	"write_errors_corrected_by_rereads_rewrites": {
+		ID:          "write_errors_corrected_by_rereads_rewrites",
 		DisplayName: "Write Errors Corrected by ReReads/ReWrites",
 		DisplayType: "",
 		Ideal:       "low",
 		Critical:    true,
 		Description: "",
 	},
-	"write.total_errors_corrected": {
-		ID:          "write.total_errors_corrected",
+	"write_total_errors_corrected": {
+		ID:          "write_total_errors_corrected",
 		DisplayName: "Write Total Errors Corrected",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"write.correction_algorithm_invocations": {
-		ID:          "write.correction_algorithm_invocations",
+	"write_correction_algorithm_invocations": {
+		ID:          "write_correction_algorithm_invocations",
 		DisplayName: "Write Correction Algorithm Invocations",
 		DisplayType: "",
 		Ideal:       "",
 		Critical:    false,
 		Description: "",
 	},
-	"write.total_uncorrected_errors": {
-		ID:          "write.total_uncorrected_errors",
+	"write_total_uncorrected_errors": {
+		ID:          "write_total_uncorrected_errors",
 		DisplayName: "Write Total Uncorrected Errors",
 		DisplayType: "",
 		Ideal:       "low",

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.3.12"
+const VERSION = "0.4.0"

webapp/backend/pkg/web/handler/get_device_details.go

@@ -1,53 +1,44 @@
 package handler
 
 import (
-	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"gorm.io/gorm"
 	"net/http"
 )
 
 func GetDeviceDetails(c *gin.Context) {
-	db := c.MustGet("DB").(*gorm.DB)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
-	device := dbModels.Device{}
-
-	if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
-		return db.Order("smarts.created_at DESC").Limit(40)
-	}).
-		Preload("SmartResults.AtaAttributes").
-		Preload("SmartResults.NvmeAttributes").
-		Preload("SmartResults.ScsiAttributes").
-		Where("wwn = ?", c.Param("wwn")).
-		First(&device).Error; err != nil {
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 
+	device, err := deviceRepo.GetDeviceDetails(c, c.Param("wwn"))
+	if err != nil {
 		logger.Errorln("An error occurred while retrieving device details", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
-	if err := device.SquashHistory(); err != nil {
-		logger.Errorln("An error occurred while squashing device history", err)
-		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
-		return
+	durationKey, exists := c.GetQuery("duration_key")
+	if !exists {
+		durationKey = "forever"
 	}
 
-	if err := device.ApplyMetadataRules(); err != nil {
-		logger.Errorln("An error occurred while applying scrutiny thresholds & rules", err)
+	smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), durationKey, nil)
+	if err != nil {
+		logger.Errorln("An error occurred while retrieving device smart results", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
 	var deviceMetadata interface{}
 	if device.IsAta() {
-		deviceMetadata = metadata.AtaMetadata
+		deviceMetadata = thresholds.AtaMetadata
 	} else if device.IsNvme() {
-		deviceMetadata = metadata.NmveMetadata
+		deviceMetadata = thresholds.NmveMetadata
 	} else if device.IsScsi() {
-		deviceMetadata = metadata.ScsiMetadata
+		deviceMetadata = thresholds.ScsiMetadata
 	}
 
-	c.JSON(http.StatusOK, gin.H{"success": true, "data": device, "metadata": deviceMetadata})
+	c.JSON(http.StatusOK, gin.H{"success": true, "data": map[string]interface{}{"device": device, "smart_results": smartResults}, "metadata": deviceMetadata})
 }

webapp/backend/pkg/web/handler/get_devices_summary.go

@@ -1,31 +1,28 @@
 package handler
 
 import (
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"gorm.io/gorm"
 	"net/http"
 )
 
 func GetDevicesSummary(c *gin.Context) {
-	db := c.MustGet("DB").(*gorm.DB)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
-	devices := []dbModels.Device{}
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 
-	//We need the last x (for now all) Smart objects for each Device, so that we can graph Temperature
-	//We also need the last
-	if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
-		return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
-	}).
-		Find(&devices).Error; err != nil {
-		logger.Errorln("Could not get device summary from DB", err)
+	summary, err := deviceRepo.GetSummary(c)
+	if err != nil {
+		logger.Errorln("An error occurred while retrieving device summary", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
 	c.JSON(http.StatusOK, gin.H{
 		"success": true,
-		"data":    devices,
+		"data": map[string]interface{}{
+			"summary": summary,
+			//"temperature": tem
+		},
 	})
 }

webapp/backend/pkg/web/handler/get_devices_summary_temp_history.go

@@ -0,0 +1,32 @@
+package handler
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+)
+
+func GetDevicesSummaryTempHistory(c *gin.Context) {
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	durationKey, exists := c.GetQuery("duration_key")
+	if !exists {
+		durationKey = "week"
+	}
+
+	tempHistory, err := deviceRepo.GetSmartTemperatureHistory(c, durationKey)
+	if err != nil {
+		logger.Errorln("An error occurred while retrieving summary/temp history", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"success": true,
+		"data": map[string]interface{}{
+			"temp_history": tempHistory,
+		},
+	})
+}

webapp/backend/pkg/web/handler/register_devices.go

@@ -1,22 +1,20 @@
 package handler
 
 import (
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/gin-gonic/gin"
-	"gorm.io/gorm"
-	"gorm.io/gorm/clause"
-
 	"github.com/sirupsen/logrus"
 	"net/http"
 )
 
 // register devices that are detected by various collectors.
-// This function is run everytime a collector is about to start a run. It can be used to update device data.
+// This function is run everytime a collector is about to start a run. It can be used to update device metadata.
 func RegisterDevices(c *gin.Context) {
-	db := c.MustGet("DB").(*gorm.DB)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 
-	var collectorDeviceWrapper dbModels.DeviceWrapper
+	var collectorDeviceWrapper models.DeviceWrapper
 	err := c.BindJSON(&collectorDeviceWrapper)
 	if err != nil {
 		logger.Errorln("Cannot parse detected devices", err)
@@ -28,11 +26,7 @@ func RegisterDevices(c *gin.Context) {
 	for _, dev := range collectorDeviceWrapper.Data {
 		//insert devices into DB (and update specified columns if device is already registered)
 		// update device fields that may change: (DeviceType, HostID)
-		if err := db.Clauses(clause.OnConflict{
-			Columns:   []clause.Column{{Name: "wwn"}},
-			DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
-		}).Create(&dev).Error; err != nil {
-
+		if err := deviceRepo.RegisterDevice(c, dev); err != nil {
 			errs = append(errs, err)
 		}
 	}
@@ -44,7 +38,7 @@ func RegisterDevices(c *gin.Context) {
 		})
 		return
 	} else {
-		c.JSON(http.StatusOK, dbModels.DeviceWrapper{
+		c.JSON(http.StatusOK, models.DeviceWrapper{
 			Success: true,
 			Data:    collectorDeviceWrapper.Data,
 		})

webapp/backend/pkg/web/handler/send_test_notification.go

@@ -1,8 +1,9 @@
 package handler
 
 import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
@@ -20,7 +21,7 @@ func SendTestNotification(c *gin.Context) {
 		Payload: notify.Payload{
 			FailureType:  "EmailTest",
 			DeviceSerial: "FAKEWDDJ324KSO",
-			DeviceType:   dbModels.DeviceProtocolAta,
+			DeviceType:   pkg.DeviceProtocolAta,
 			DeviceName:   "/dev/sda",
 			Test:         true,
 		},
@@ -33,7 +34,7 @@ func SendTestNotification(c *gin.Context) {
 			"errors":  []string{err.Error()},
 		})
 	} else {
-		c.JSON(http.StatusOK, dbModels.DeviceWrapper{
+		c.JSON(http.StatusOK, models.DeviceWrapper{
 			Success: true,
 		})
 	}

webapp/backend/pkg/web/handler/upload_device_metrics.go

@@ -1,20 +1,24 @@
 package handler
 
 import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	"gorm.io/gorm"
 	"net/http"
 )
 
 func UploadDeviceMetrics(c *gin.Context) {
-	db := c.MustGet("DB").(*gorm.DB)
+	//db := c.MustGet("DB").(*gorm.DB)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 	appConfig := c.MustGet("CONFIG").(config.Interface)
+	//influxWriteDb := c.MustGet("INFLUXDB_WRITE").(*api.WriteAPIBlocking)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	//appConfig := c.MustGet("CONFIG").(config.Interface)
 
 	var collectorSmartData collector.SmartInfo
 	err := c.BindJSON(&collectorSmartData)
@@ -25,39 +29,49 @@ func UploadDeviceMetrics(c *gin.Context) {
 	}
 
 	//update the device information if necessary
-	var device dbModels.Device
-	db.Where("wwn = ?", c.Param("wwn")).First(&device)
-	device.UpdateFromCollectorSmartInfo(collectorSmartData)
-	if err := db.Model(&device).Updates(device).Error; err != nil {
-		logger.Errorln("An error occurred while updating device data from smartctl metrics", err)
+	updatedDevice, err := deviceRepo.UpdateDevice(c, c.Param("wwn"), collectorSmartData)
+	if err != nil {
+		logger.Errorln("An error occurred while updating device data from smartctl metrics:", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
 	// insert smart info
-	deviceSmartData := dbModels.Smart{}
-	err = deviceSmartData.FromCollectorSmartInfo(c.Param("wwn"), collectorSmartData)
+	smartData, err := deviceRepo.SaveSmartAttributes(c, c.Param("wwn"), collectorSmartData)
 	if err != nil {
-		logger.Errorln("Could not process SMART metrics", err)
-		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
-		return
-	}
-	if err := db.Create(&deviceSmartData).Error; err != nil {
 		logger.Errorln("An error occurred while saving smartctl metrics", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
 
+	if smartData.Status != pkg.DeviceStatusPassed {
+		//there is a failure detected by Scrutiny, update the device status on the homepage.
+		updatedDevice, err = deviceRepo.UpdateDeviceStatus(c, c.Param("wwn"), smartData.Status)
+		if err != nil {
+			logger.Errorln("An error occurred while updating device status", err)
+			c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+			return
+		}
+	}
+
+	// save smart temperature data (ignore failures)
+	err = deviceRepo.SaveSmartTemperature(c, c.Param("wwn"), updatedDevice.DeviceProtocol, collectorSmartData)
+	if err != nil {
+		logger.Errorln("An error occurred while saving smartctl temp data", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
 	//check for error
-	if deviceSmartData.SmartStatus == dbModels.SmartStatusFailed {
+	if updatedDevice.DeviceStatus != pkg.DeviceStatusPassed {
 		//send notifications
 		testNotify := notify.Notify{
 			Config: appConfig,
 			Payload: notify.Payload{
 				FailureType:  notify.NotifyFailureTypeSmartFailure,
-				DeviceName:   device.DeviceName,
-				DeviceType:   device.DeviceProtocol,
-				DeviceSerial: device.SerialNumber,
+				DeviceName:   updatedDevice.DeviceName,
+				DeviceType:   updatedDevice.DeviceProtocol,
+				DeviceSerial: updatedDevice.SerialNumber,
 				Test:         false,
 			},
 			Logger: logger,

webapp/backend/pkg/web/middleware/repository.go

@@ -0,0 +1,22 @@
+package middleware
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+)
+
+func RepositoryMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc {
+
+	deviceRepo, err := database.NewScrutinyRepository(appConfig, globalLogger)
+	if err != nil {
+		panic(err)
+	}
+
+	//TODO: determine where we can call defer deviceRepo.Close()
+	return func(c *gin.Context) {
+		c.Set("DEVICE_REPOSITORY", deviceRepo)
+		c.Next()
+	}
+}

webapp/backend/pkg/web/middleware/sqlite3.go

@@ -1,59 +0,0 @@
-package middleware
-
-import (
-	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
-	"github.com/gin-gonic/gin"
-	"github.com/sirupsen/logrus"
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-)
-
-func DatabaseMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc {
-
-	//var database *gorm.DB
-	fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
-	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
-		//TODO: figure out how to log database queries again.
-		//Logger: logger
-	})
-	if err != nil {
-		panic("Failed to connect to database!")
-	}
-
-	//database.SetLogger()
-	database.AutoMigrate(&db.Device{})
-	database.AutoMigrate(&db.SelfTest{})
-	database.AutoMigrate(&db.Smart{})
-	database.AutoMigrate(&db.SmartAtaAttribute{})
-	database.AutoMigrate(&db.SmartNvmeAttribute{})
-	database.AutoMigrate(&db.SmartScsiAttribute{})
-
-	//TODO: detrmine where we can call defer database.Close()
-	return func(c *gin.Context) {
-		c.Set("DB", database)
-		c.Next()
-	}
-}
-
-// GormLogger is a custom logger for Gorm, making it use logrus.
-type GormLogger struct{ Logger logrus.FieldLogger }
-
-// Print handles log events from Gorm for the custom logger.
-func (gl *GormLogger) Print(v ...interface{}) {
-	switch v[0] {
-	case "sql":
-		gl.Logger.WithFields(
-			logrus.Fields{
-				"module":  "gorm",
-				"type":    "sql",
-				"rows":    v[5],
-				"src_ref": v[1],
-				"values":  v[4],
-			},
-		).Debug(v[3])
-	case "log":
-		gl.Logger.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2])
-	}
-}

webapp/backend/pkg/web/server.go

@@ -23,7 +23,7 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 	r := gin.New()
 
 	r.Use(middleware.LoggerMiddleware(logger))
-	r.Use(middleware.DatabaseMiddleware(ae.Config, logger))
+	r.Use(middleware.RepositoryMiddleware(ae.Config, logger))
 	r.Use(middleware.ConfigMiddleware(ae.Config))
 	r.Use(gin.Recovery())
 
@@ -40,11 +40,12 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 			})
 			api.POST("/health/notify", handler.SendTestNotification) //check if notifications are configured correctly
 
-			api.POST("/devices/register", handler.RegisterDevices)      //used by Collector to register new devices and retrieve filtered list
-			api.GET("/summary", handler.GetDevicesSummary)              //used by Dashboard
-			api.POST("/device/:wwn/smart", handler.UploadDeviceMetrics) //used by Collector to upload data
-			api.POST("/device/:wwn/selftest", handler.UploadDeviceSelfTests)
-			api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details
+      api.POST("/devices/register", handler.RegisterDevices)         //used by Collector to register new devices and retrieve filtered list
+      api.GET("/summary", handler.GetDevicesSummary)                 //used by Dashboard
+      api.GET("/summary/temp", handler.GetDevicesSummaryTempHistory) //used by Dashboard (Temperature history dropdown)
+      api.POST("/device/:wwn/smart", handler.UploadDeviceMetrics)    //used by Collector to upload data
+      api.POST("/device/:wwn/selftest", handler.UploadDeviceSelfTests)
+      api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details
 		}
 	}
 

webapp/backend/pkg/web/server_test.go

@@ -1,14 +1,17 @@
 package web_test
 
 import (
+	"bytes"
 	"encoding/json"
-	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
-	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web"
 	"github.com/golang/mock/gomock"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -16,12 +19,76 @@ import (
 	"path"
 	"strings"
 	"testing"
+	"time"
 )
 
-const (
-	emptyBasePath = ""
-	notEmptyBasePath = "/basepath"
-)
+/*
+All tests in this file require the existance of a influxDB listening on port 8086
+
+docker run --rm -it -p 8086:8086 \
+-e DOCKER_INFLUXDB_INIT_MODE=setup \
+-e DOCKER_INFLUXDB_INIT_USERNAME=admin \
+-e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
+-e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
+-e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
+-e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \
+influxdb:2.0
+*/
+
+//func TestMain(m *testing.M) {
+//	setup()
+//	code := m.Run()
+//	shutdown()
+//	os.Exit(code)
+//}
+
+// InfluxDB will throw an error/ignore any submitted data with a timestamp older than the
+// retention period. Lets fix this by opening test files, modifying the timestamp and returning an io.Reader
+func helperReadSmartDataFileFixTimestamp(t *testing.T, smartDataFilepath string) io.Reader {
+	metricsfile, err := os.Open(smartDataFilepath)
+	require.NoError(t, err)
+
+	metricsFileData, err := ioutil.ReadAll(metricsfile)
+	require.NoError(t, err)
+
+	//unmarshal because we need to change the timestamp
+	var smartData collector.SmartInfo
+	err = json.Unmarshal(metricsFileData, &smartData)
+	require.NoError(t, err)
+	smartData.LocalTime.TimeT = time.Now().Unix()
+	updatedSmartDataBytes, err := json.Marshal(smartData)
+
+	return bytes.NewReader(updatedSmartDataBytes)
+}
+
+func TestHealthRoute(t *testing.T) {
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
+
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
+	}
+
+	ae := web.AppEngine{
+		Config: fakeConfig,
+	}
+
+	router := ae.Setup(logrus.New())
 
 var basePathTestCases = []string{
 	emptyBasePath,
@@ -62,145 +129,146 @@ func TestHealthRoute(t *testing.T) {
 }
 
 func TestRegisterDevicesRoute(t *testing.T) {
-	for _, basePath := range basePathTestCases {
-		t.Run(fmt.Sprintf(`with basePath "%s"`, basePath), func (tt *testing.T) {
-			//setup
-			parentPath, _ := ioutil.TempDir("", "")
-			mockCtrl := gomock.NewController(tt)
-			fakeConfig := mock_config.NewMockInterface(mockCtrl)
-			fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes()
-			fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
-			fakeConfig.EXPECT().GetString("web.src.backend.basepath").Return(basePath).AnyTimes()
-			ae := web.AppEngine{
-				Config: fakeConfig,
-			}
-			router := ae.Setup(logrus.New())
-			file, err := os.Open("testdata/register-devices-req.json")
-			require.NoError(tt, err)
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
+	}
 
-			//test
-			w := httptest.NewRecorder()
-			req, _ := http.NewRequest("POST", basePath + "/api/devices/register", file)
-			router.ServeHTTP(w, req)
-
-			//assert
-			require.Equal(tt, 200, w.Code)
-
-			// tear down
-			mockCtrl.Finish()
-			os.RemoveAll(parentPath)
-		})
+	ae := web.AppEngine{
+		Config: fakeConfig,
 	}
 }
 
 func TestUploadDeviceMetricsRoute(t *testing.T) {
-	for _, basePath := range basePathTestCases {
-		t.Run(fmt.Sprintf(`with basePath "%s"`, basePath), func(tt *testing.T) {
-			//setup
-			parentPath, _ := ioutil.TempDir("", "")
-			mockCtrl := gomock.NewController(tt)
-			fakeConfig := mock_config.NewMockInterface(mockCtrl)
-			fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-			fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
-			fakeConfig.EXPECT().GetString("web.src.backend.basepath").Return(basePath).AnyTimes()
-			ae := web.AppEngine{
-				Config: fakeConfig,
-			}
-			router := ae.Setup(logrus.New())
-			devicesfile, err := os.Open("testdata/register-devices-single-req.json")
-			require.NoError(tt, err)
-
-			metricsfile, err := os.Open("testdata/upload-device-metrics-req.json")
-			require.NoError(tt, err)
-
-			//test
-			wr := httptest.NewRecorder()
-			req, _ := http.NewRequest("POST", basePath + "/api/devices/register", devicesfile)
-			router.ServeHTTP(wr, req)
-			require.Equal(tt, 200, wr.Code)
-
-			mr := httptest.NewRecorder()
-			req, _ = http.NewRequest("POST", basePath + "/api/device/0x5000cca264eb01d7/smart", metricsfile)
-			router.ServeHTTP(mr, req)
-			require.Equal(tt, 200, mr.Code)
-
-			//assert
-
-			// tear down
-			mockCtrl.Finish()
-			os.RemoveAll(parentPath)
-		})
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
 	}
+
+	ae := web.AppEngine{
+		Config: fakeConfig,
+	}
+	router := ae.Setup(logrus.New())
+	devicesfile, err := os.Open("testdata/register-devices-single-req.json")
+	require.NoError(t, err)
+
+	metricsfile := helperReadSmartDataFileFixTimestamp(t, "testdata/upload-device-metrics-req.json")
+
+	//test
+	wr := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/api/devices/register", devicesfile)
+	router.ServeHTTP(wr, req)
+	require.Equal(t, 200, wr.Code)
+
+	mr := httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/api/device/0x5000cca264eb01d7/smart", metricsfile)
+	router.ServeHTTP(mr, req)
+	require.Equal(t, 200, mr.Code)
+
+	//assert
 }
 
 func TestPopulateMultiple(t *testing.T) {
-	for _, basePath := range basePathTestCases {
-		t.Run(fmt.Sprintf(`with basePath "%s"`, basePath), func(tt *testing.T) {
-			//setup
-			parentPath, _ := ioutil.TempDir("", "")
-			mockCtrl := gomock.NewController(tt)
-			fakeConfig := mock_config.NewMockInterface(mockCtrl)
-			//fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
-			fakeConfig.EXPECT().GetStringSlice("notify.urls").Return([]string{}).AnyTimes()
-			fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-			fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
-			fakeConfig.EXPECT().GetString("web.src.backend.basepath").Return(basePath).AnyTimes()
-			ae := web.AppEngine{
-				Config: fakeConfig,
-			}
-			router := ae.Setup(logrus.New())
-			devicesfile, err := os.Open("testdata/register-devices-req.json")
-			require.NoError(tt, err)
-
-			metricsfile, err := os.Open("../models/testdata/smart-ata.json")
-			require.NoError(tt, err)
-			failfile, err := os.Open("../models/testdata/smart-fail2.json")
-			require.NoError(tt, err)
-			nvmefile, err := os.Open("../models/testdata/smart-nvme.json")
-			require.NoError(tt, err)
-			scsifile, err := os.Open("../models/testdata/smart-scsi.json")
-			require.NoError(tt, err)
-			scsi2file, err := os.Open("../models/testdata/smart-scsi2.json")
-			require.NoError(tt, err)
-
-			//test
-			wr := httptest.NewRecorder()
-			req, _ := http.NewRequest("POST", basePath + "/api/devices/register", devicesfile)
-			router.ServeHTTP(wr, req)
-			require.Equal(tt, 200, wr.Code)
-
-			mr := httptest.NewRecorder()
-			req, _ = http.NewRequest("POST", basePath + "/api/device/0x5000cca264eb01d7/smart", metricsfile)
-			router.ServeHTTP(mr, req)
-			require.Equal(tt, 200, mr.Code)
-
-			fr := httptest.NewRecorder()
-			req, _ = http.NewRequest("POST", basePath + "/api/device/0x5000cca264ec3183/smart", failfile)
-			router.ServeHTTP(fr, req)
-			require.Equal(tt, 200, fr.Code)
-
-			nr := httptest.NewRecorder()
-			req, _ = http.NewRequest("POST", basePath + "/api/device/0x5002538e40a22954/smart", nvmefile)
-			router.ServeHTTP(nr, req)
-			require.Equal(tt, 200, nr.Code)
-
-			sr := httptest.NewRecorder()
-			req, _ = http.NewRequest("POST", basePath + "/api/device/0x5000cca252c859cc/smart", scsifile)
-			router.ServeHTTP(sr, req)
-			require.Equal(tt, 200, sr.Code)
-
-			s2r := httptest.NewRecorder()
-			req, _ = http.NewRequest("POST", basePath + "/api/device/0x5000cca264ebc248/smart", scsi2file)
-			router.ServeHTTP(s2r, req)
-			require.Equal(tt, 200, s2r.Code)
-
-			//assert
-
-			// tear down
-			mockCtrl.Finish()
-			os.RemoveAll(parentPath)
-		})
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	//fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
+	fakeConfig.EXPECT().GetStringSlice("notify.urls").Return([]string{}).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
 	}
+
+	ae := web.AppEngine{
+		Config: fakeConfig,
+	}
+	router := ae.Setup(logrus.New())
+	devicesfile, err := os.Open("testdata/register-devices-req.json")
+	require.NoError(t, err)
+
+	metricsfile := helperReadSmartDataFileFixTimestamp(t, "../models/testdata/smart-ata.json")
+	failfile := helperReadSmartDataFileFixTimestamp(t, "../models/testdata/smart-fail2.json")
+	nvmefile := helperReadSmartDataFileFixTimestamp(t, "../models/testdata/smart-nvme.json")
+	scsifile := helperReadSmartDataFileFixTimestamp(t, "../models/testdata/smart-scsi.json")
+	scsi2file := helperReadSmartDataFileFixTimestamp(t, "../models/testdata/smart-scsi2.json")
+
+	//test
+	wr := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/api/devices/register", devicesfile)
+	router.ServeHTTP(wr, req)
+	require.Equal(t, 200, wr.Code)
+
+	mr := httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/api/device/0x5000cca264eb01d7/smart", metricsfile)
+	router.ServeHTTP(mr, req)
+	require.Equal(t, 200, mr.Code)
+
+	fr := httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/api/device/0x5000cca264ec3183/smart", failfile)
+	router.ServeHTTP(fr, req)
+	require.Equal(t, 200, fr.Code)
+
+	nr := httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/api/device/0x5002538e40a22954/smart", nvmefile)
+	router.ServeHTTP(nr, req)
+	require.Equal(t, 200, nr.Code)
+
+	sr := httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/api/device/0x5000cca252c859cc/smart", scsifile)
+	router.ServeHTTP(sr, req)
+	require.Equal(t, 200, sr.Code)
+
+	s2r := httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/api/device/0x5000cca264ebc248/smart", scsi2file)
+	router.ServeHTTP(s2r, req)
+	require.Equal(t, 200, s2r.Code)
+
+	//assert
 }
 
 //TODO: this test should use a recorded request/response playback.
@@ -229,173 +297,168 @@ func TestPopulateMultiple(t *testing.T) {
 //}
 
 func TestSendTestNotificationRoute_WebhookFailure(t *testing.T) {
-	for _, basePath := range basePathTestCases {
-		t.Run(fmt.Sprintf(`with basePath "%s"`, basePath), func(tt *testing.T) {
-			//setup
-			parentPath, _ := ioutil.TempDir("", "")
-			mockCtrl := gomock.NewController(tt)
-			fakeConfig := mock_config.NewMockInterface(mockCtrl)
-			fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-			fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
-			fakeConfig.EXPECT().GetString("web.src.backend.basepath").Return(basePath).AnyTimes()
-			fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"})
-			ae := web.AppEngine{
-				Config: fakeConfig,
-			}
-			router := ae.Setup(logrus.New())
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"})
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
+	}
 
-			//test
-			wr := httptest.NewRecorder()
-			req, _ := http.NewRequest("POST", basePath + "/api/health/notify", strings.NewReader("{}"))
-			router.ServeHTTP(wr, req)
-
-			//assert
-			require.Equal(tt, 500, wr.Code)
-
-			// tear down
-			mockCtrl.Finish()
-			os.RemoveAll(parentPath)
-		})
+	ae := web.AppEngine{
+		Config: fakeConfig,
 	}
 }
 
 func TestSendTestNotificationRoute_ScriptFailure(t *testing.T) {
-	for _, basePath := range basePathTestCases {
-		t.Run(fmt.Sprintf(`with basePath "%s"`, basePath), func(tt *testing.T) {
-			//setup
-			parentPath, _ := ioutil.TempDir("", "")
-			mockCtrl := gomock.NewController(tt)
-			fakeConfig := mock_config.NewMockInterface(mockCtrl)
-			fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-			fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
-			fakeConfig.EXPECT().GetString("web.src.backend.basepath").Return(basePath).AnyTimes()
-			fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"})
-			ae := web.AppEngine{
-				Config: fakeConfig,
-			}
-			router := ae.Setup(logrus.New())
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"})
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
+	}
 
-			//test
-			wr := httptest.NewRecorder()
-			req, _ := http.NewRequest("POST", basePath + "/api/health/notify", strings.NewReader("{}"))
-			router.ServeHTTP(wr, req)
-
-			//assert
-			require.Equal(tt, 500, wr.Code)
-
-			// tear down
-			mockCtrl.Finish()
-			os.RemoveAll(parentPath)
-		})
+	ae := web.AppEngine{
+		Config: fakeConfig,
 	}
 }
 
 func TestSendTestNotificationRoute_ScriptSuccess(t *testing.T) {
-	for _, basePath := range basePathTestCases {
-		t.Run(fmt.Sprintf(`with basePath "%s"`, basePath), func(tt *testing.T) {
-			//setup
-			//setup
-			parentPath, _ := ioutil.TempDir("", "")
-			defer os.RemoveAll(parentPath)
-			mockCtrl := gomock.NewController(t)
-			defer mockCtrl.Finish()
-			fakeConfig := mock_config.NewMockInterface(mockCtrl)
-			fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-			fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
-			fakeConfig.EXPECT().GetString("web.src.backend.basepath").Return(basePath).AnyTimes()
-			fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///usr/bin/env"})
-			ae := web.AppEngine{
-				Config: fakeConfig,
-			}
-			router := ae.Setup(logrus.New())
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///usr/bin/env"})
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
+	}
 
-			//test
-			wr := httptest.NewRecorder()
-			req, _ := http.NewRequest("POST", basePath + "/api/health/notify", strings.NewReader("{}"))
-			router.ServeHTTP(wr, req)
-
-			//assert
-			require.Equal(t, 200, wr.Code)
-		})
+	ae := web.AppEngine{
+		Config: fakeConfig,
 	}
 }
 
 func TestSendTestNotificationRoute_ShoutrrrFailure(t *testing.T) {
-	for _, basePath := range basePathTestCases {
-		t.Run(fmt.Sprintf(`with basePath "%s"`, basePath), func(tt *testing.T) {
-			//setup
-			parentPath, _ := ioutil.TempDir("", "")
-			mockCtrl := gomock.NewController(tt)
-			fakeConfig := mock_config.NewMockInterface(mockCtrl)
-			fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-			fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
-			fakeConfig.EXPECT().GetString("web.src.backend.basepath").Return(basePath).AnyTimes()
-			fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"})
-			ae := web.AppEngine{
-				Config: fakeConfig,
-			}
-			router := ae.Setup(logrus.New())
-
-			//test
-			wr := httptest.NewRecorder()
-			req, _ := http.NewRequest("POST", basePath + "/api/health/notify", strings.NewReader("{}"))
-			router.ServeHTTP(wr, req)
-
-			//assert
-			require.Equal(tt, 500, wr.Code)
-
-			// tear down
-			mockCtrl.Finish()
-			os.RemoveAll(parentPath)
-		})
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"})
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
+	}
+	ae := web.AppEngine{
+		Config: fakeConfig,
 	}
 }
 
 func TestGetDevicesSummaryRoute_Nvme(t *testing.T) {
-	for _, basePath := range basePathTestCases {
-		t.Run(fmt.Sprintf(`with basePath "%s"`, basePath), func(tt *testing.T) {
-			//setup
-			parentPath, _ := ioutil.TempDir("", "")
-			mockCtrl := gomock.NewController(tt)
-			fakeConfig := mock_config.NewMockInterface(mockCtrl)
-			fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-			fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
-			fakeConfig.EXPECT().GetString("web.src.backend.basepath").Return(basePath).AnyTimes()
-			ae := web.AppEngine{
-				Config: fakeConfig,
-			}
-			router := ae.Setup(logrus.New())
-			devicesfile, err := os.Open("testdata/register-devices-req-2.json")
-			require.NoError(tt, err)
-
-			metricsfile, err := os.Open("../models/testdata/smart-nvme2.json")
-			require.NoError(tt, err)
-
-			//test
-			wr := httptest.NewRecorder()
-			req, _ := http.NewRequest("POST", basePath + "/api/devices/register", devicesfile)
-			router.ServeHTTP(wr, req)
-			require.Equal(tt, 200, wr.Code)
-
-			mr := httptest.NewRecorder()
-			req, _ = http.NewRequest("POST", basePath + "/api/device/a4c8e8ed-11a0-4c97-9bba-306440f1b944/smart", metricsfile)
-			router.ServeHTTP(mr, req)
-			require.Equal(tt, 200, mr.Code)
-
-			sr := httptest.NewRecorder()
-			req, _ = http.NewRequest("GET", basePath + "/api/summary", nil)
-			router.ServeHTTP(sr, req)
-			require.Equal(t, 200, sr.Code)
-			var device dbModels.DeviceWrapper
-			json.Unmarshal(sr.Body.Bytes(), &device)
-
-			//assert
-			require.Equal(tt, "a4c8e8ed-11a0-4c97-9bba-306440f1b944", device.Data[0].WWN)
-			require.Equal(tt, "passed", device.Data[0].SmartResults[0].SmartStatus)
-
-			// tear down
-			mockCtrl.Finish()
-			os.RemoveAll(parentPath)
-		})
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.influxdb.port").Return("8086").AnyTimes()
+	fakeConfig.EXPECT().IsSet("web.influxdb.token").Return(true).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes()
+	fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes()
+	fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes()
+	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{})
+	if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions {
+		// when running test suite in github actions, we run an influxdb service as a sidecar.
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes()
+	} else {
+		fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes()
 	}
+
+	ae := web.AppEngine{
+		Config: fakeConfig,
+	}
+	router := ae.Setup(logrus.New())
+	devicesfile, err := os.Open("testdata/register-devices-req-2.json")
+	require.NoError(t, err)
+
+	metricsfile := helperReadSmartDataFileFixTimestamp(t, "../models/testdata/smart-nvme2.json")
+
+	//test
+	wr := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/api/devices/register", devicesfile)
+	router.ServeHTTP(wr, req)
+	require.Equal(t, 200, wr.Code)
+
+	mr := httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/api/device/a4c8e8ed-11a0-4c97-9bba-306440f1b944/smart", metricsfile)
+	router.ServeHTTP(mr, req)
+	require.Equal(t, 200, mr.Code)
+
+	sr := httptest.NewRecorder()
+	req, _ = http.NewRequest("GET", "/api/summary", nil)
+	router.ServeHTTP(sr, req)
+	require.Equal(t, 200, sr.Code)
+	var deviceSummary models.DeviceSummaryWrapper
+	err = json.Unmarshal(sr.Body.Bytes(), &deviceSummary)
+	require.NoError(t, err)
+
+	//assert
+	require.Equal(t, "a4c8e8ed-11a0-4c97-9bba-306440f1b944", deviceSummary.Data.Summary["a4c8e8ed-11a0-4c97-9bba-306440f1b944"].Device.WWN)
+	require.Equal(t, pkg.DeviceStatusFailedScrutiny, deviceSummary.Data.Summary["a4c8e8ed-11a0-4c97-9bba-306440f1b944"].Device.DeviceStatus)
 }

webapp/frontend/src/app/modules/dashboard/dashboard.component.html

@@ -1,4 +1,5 @@
-<div *ngIf="data && data.data && data.data.length > 0; else emptyDashboard">
+
+<div *ngIf="data && data.data && data.data.summary; else emptyDashboard">
     <div class="flex flex-col flex-auto w-full p-8 xs:p-2">
 
         <div class="flex flex-wrap w-full">
@@ -47,38 +48,36 @@
             </div>
 
             <div class="flex flex-wrap w-full">
-
-                <div *ngFor="let disk of data.data | deviceSort" class="flex gt-sm:w-1/2 min-w-80 p-4">
-                    <div [ngClass]="{'border-green': disk.smart_results[0]?.smart_status == 'passed',
-                                    'border-red': disk.smart_results[0]?.smart_status == 'failed' }"
+                <div *ngFor="let summary of data.data.summary | keyvalue" class="flex gt-sm:w-1/2 min-w-80 p-4">
+                    <div [ngClass]="{ 'border-green': summary.value.device.device_status == 0 && summary.value.smart,
+                                    'border-red': summary.value.device.device_status != 0 }"
                          class="relative flex flex-col flex-auto p-6 pr-3 pb-3 bg-card rounded border-l-4 shadow-md overflow-hidden">
                         <div class="absolute bottom-0 right-0 w-24 h-24 -m-6">
                             <mat-icon class="icon-size-96 opacity-12 text-green"
-                                      *ngIf="disk.smart_results[0]?.smart_status == 'passed'"
+                                      *ngIf="summary.value.device.device_status == 0 && summary.value.smart"
                                       [svgIcon]="'heroicons_outline:check-circle'"></mat-icon>
                             <mat-icon class="icon-size-96 opacity-12 text-red"
-                                      *ngIf="disk.smart_results[0]?.smart_status == 'failed'"
+                                      *ngIf="summary.value.device.device_status != 0"
                                       [svgIcon]="'heroicons_outline:exclamation-circle'"></mat-icon>
                             <mat-icon class="icon-size-96 opacity-12 text-yellow"
-                                      *ngIf="!disk.smart_results[0]"
+                                      *ngIf="!summary.value.smart"
                                       [svgIcon]="'heroicons_outline:question-mark-circle'"></mat-icon>
                         </div>
                         <div class="flex items-center">
                             <div class="flex flex-col">
-                                <a [routerLink]="'/device/'+ disk.wwn"
-                                   class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceTitle(disk)}}</a>
-                                <div [ngClass]="{'text-green': disk.smart_results[0]?.smart_status == 'passed',
-                                    'text-red': disk.smart_results[0]?.smart_status == 'failed' }" class="font-medium text-sm" *ngIf="disk.smart_results[0]">
-                                    Last Updated on {{disk.smart_results[0]?.date | date:'MMMM dd, yyyy - HH:mm' }}
+                                <a [routerLink]="'/device/'+ summary.value.device.wwn"
+                                   class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceTitle(summary.value.device)}}</a>
+                                <div [ngClass]="classDeviceLastUpdatedOn(summary.value)" class="font-medium text-sm" *ngIf="summary.value.smart">
+                                    Last Updated on {{summary.value.smart.collector_date | date:'MMMM dd, yyyy - HH:mm' }}
                                 </div>
                             </div>
-                            <div class="ml-auto" *ngIf="disk.smart_results">
+                            <div class="ml-auto" *ngIf="summary.value.device">
                                 <button mat-icon-button
                                         [matMenuTriggerFor]="previousStatementMenu">
                                     <mat-icon [svgIcon]="'more_vert'"></mat-icon>
                                 </button>
                                 <mat-menu #previousStatementMenu="matMenu">
-                                    <a mat-menu-item [routerLink]="'/device/'+ disk.wwn">
+                                    <a mat-menu-item [routerLink]="'/device/'+ summary.value.device.wwn">
                                     <span class="flex items-center">
                                         <mat-icon class="icon-size-20 mr-3"
                                                   [svgIcon]="'payment'"></mat-icon>
@@ -90,22 +89,22 @@
                         </div>
                         <div class="flex flex-row flex-wrap mt-4 -mx-6">
                             <div class="flex flex-col mx-6 my-3 xs:w-full">
-                                <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">S.M.A.R.T</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="disk.smart_results[0]; else unknownStatus">{{ disk.smart_results[0]?.smart_status | titlecase}}</div>
+                                <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Status</div>
+                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.collector_date; else unknownStatus">{{ deviceStatusString(summary.value.device.device_status) | titlecase}}</div>
                                 <ng-template #unknownStatus><div class="mt-2 font-medium text-3xl leading-none">No Data</div></ng-template>
                             </div>
                             <div class="flex flex-col mx-6 my-3 xs:w-full">
                                 <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Temperature</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="disk.smart_results[0]; else unknownTemp">{{ disk.smart_results[0]?.temp }}°C</div>
+                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.collector_date; else unknownTemp">{{ summary.value.smart?.temp }}°C</div>
                                 <ng-template #unknownTemp><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
                             </div>
                             <div class="flex flex-col mx-6 my-3 xs:w-full">
                                 <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Capacity</div>
-                                <div class="mt-2 font-medium text-3xl leading-none">{{ disk.capacity | fileSize}}</div>
+                                <div class="mt-2 font-medium text-3xl leading-none">{{ summary.value.device.capacity | fileSize}}</div>
                             </div>
                             <div class="flex flex-col mx-6 my-3 xs:w-full">
                                 <div class="font-semibold text-xs text-hint uppercase tracking-wider leading-none">Powered On</div>
-                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="disk.smart_results[0]?.power_on_hours; else unknownPoweredOn">{{ humanizeDuration(disk.smart_results[0]?.power_on_hours * 60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] }) }}</div>
+                                <div class="mt-2 font-medium text-3xl leading-none" *ngIf="summary.value.smart?.power_on_hours; else unknownPoweredOn">{{ humanizeDuration(summary.value.smart?.power_on_hours * 60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] }) }}</div>
                                 <ng-template #unknownPoweredOn><div class="mt-2 font-medium text-3xl leading-none">--</div></ng-template>
                             </div>
                         </div>
@@ -126,21 +125,20 @@
                                 <button class="h-8 min-h-8 px-2"
                                         matTooltip="not yet implemented"
                                         mat-button
-                                        [matMenuTriggerFor]="accountBalanceMenu">
-                                    <span class="font-medium text-sm text-hint">12 months</span>
+                                        [matMenuTriggerFor]="tempRangeMenu">
+                                    <span class="font-medium text-sm text-hint">1 week</span>
                                 </button>
-                                <mat-menu #accountBalanceMenu="matMenu">
-                                    <button mat-menu-item>3 months</button>
-                                    <button mat-menu-item>6 months</button>
-                                    <button mat-menu-item>9 months</button>
+                                <mat-menu #tempRangeMenu="matMenu">
+                                    <button mat-menu-item>1 month</button>
                                     <button mat-menu-item>12 months</button>
+                                    <button mat-menu-item>all time</button>
                                 </mat-menu>
                             </div>
                         </div>
 
                     </div>
                     <div class="flex flex-col flex-auto">
-                        <apx-chart class="flex-auto w-full h-full"
+                        <apx-chart *ngIf="temperatureOptions" class="flex-auto w-full h-full"
                                    [chart]="temperatureOptions.chart"
                                    [colors]="temperatureOptions.colors"
                                    [fill]="temperatureOptions.fill"

webapp/frontend/src/app/modules/dashboard/dashboard.component.ts

@@ -84,16 +84,23 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
     private _deviceDataTemperatureSeries() {
         var deviceTemperatureSeries = []
 
-        for(let device of this.data.data){
+        console.log("DEVICE DATA SUMMARY", this.data)
+
+        for(const wwn in this.data.data.summary){
+            var deviceSummary = this.data.data.summary[wwn]
+            if (!deviceSummary.temp_history){
+                continue
+            }
             var deviceSeriesMetadata = {
-                name: `/dev/${device.device_name}`,
+                name: `/dev/${deviceSummary.device.device_name}`,
                 data: []
             }
-            for(let smartResults of device.smart_results){
-                let newDate = new Date(smartResults.CreatedAt);
+
+            for(let tempHistory of deviceSummary.temp_history){
+                let newDate = new Date(tempHistory.date);
                 deviceSeriesMetadata.data.push({
                     x: newDate,
-                    y: smartResults.temp
+                    y: tempHistory.temp
                 })
             }
             deviceTemperatureSeries.push(deviceSeriesMetadata)
@@ -181,6 +188,34 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
         return title.join(' - ')
     }
 
+    deviceStatusString(deviceStatus){
+        if(deviceStatus == 0){
+            return "passed"
+        } else {
+            return "failed"
+        }
+    }
+
+    classDeviceLastUpdatedOn(deviceSummary){
+        if (deviceSummary.device.device_status !== 0) {
+            return 'text-red' // if the device has failed, always highlight in red
+        } else if(deviceSummary.device.device_status === 0 && deviceSummary.smart){
+            if(moment().subtract(14, 'd').isBefore(deviceSummary.smart.collector_date)){
+                // this device was updated in the last 2 weeks.
+                return 'text-green'
+            } else if(moment().subtract(1, 'm').isBefore(deviceSummary.smart.collector_date)){
+                // this device was updated in the last month
+                return 'text-yellow'
+            } else{
+                // last updated more than a month ago.
+                return 'text-red'
+            }
+
+        } else {
+            return ''
+        }
+    }
+
     /**
      * Track by function for ngFor loops
      *

webapp/frontend/src/app/modules/detail/detail.component.html

@@ -53,59 +53,73 @@
         <div class="flex flex-auto w-1/4 p-4 lt-md:w-full">
             <treo-card class="flex flex-auto p-4 pt-6 flex-col flex-auto filter-list">
                 <div class="flex items-center justify-between">
-                    <div class="text-2xl font-semibold leading-tight">/dev/{{data.data.device_name}}</div>
+                    <div class="text-2xl font-semibold leading-tight">/dev/{{device?.device_name}}</div>
                 </div>
                 <div class="flex flex-col my-2 grid grid-cols-2">
-                    <div *ngIf="data.data.host_id" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.host_id}}</div>
+                    <div *ngIf="device" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>
+                            <span class="inline-flex items-center font-bold text-xs px-2 py-2px rounded-full tracking-wide uppercase"
+                                   [ngClass]="{'red-200': device?.device_status != 0,
+                                  'green-200': device?.device_status == 0}">
+                                <span class="w-2 h-2 rounded-full mr-2"
+                                      [ngClass]="{'bg-red': device?.device_status != 0,
+                                  'bg-green': device?.device_status == 0}"></span>
+                                <span class="pr-2px leading-relaxed whitespace-no-wrap">{{device?.device_status == 0 ? 'passed' : 'failed'}}</span>
+                            </span>
+                        </div>
+                        <div class="text-secondary text-md">Status</div>
+                    </div>
+
+                    <div *ngIf="device?.host_id" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.host_id}}</div>
                         <div class="text-secondary text-md">Host ID</div>
                     </div>
-                    <div *ngIf="data.data.device_type && data.data.device_type != 'ata' && data.data.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.device_type | uppercase}}</div>
+                    <div *ngIf="device?.device_type && device?.device_type != 'ata' && device?.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.device_type | uppercase}}</div>
                         <div class="text-secondary text-md">Device Type</div>
                     </div>
-                    <div *ngIf="data.data.manufacturer" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.manufacturer}}</div>
+                    <div *ngIf="device?.manufacturer" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.manufacturer}}</div>
                         <div class="text-secondary text-md">Model Family</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.model_name}}</div>
+                        <div>{{device?.model_name}}</div>
                         <div class="text-secondary text-md">Device Model</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.serial_number}}</div>
+                        <div>{{device?.serial_number}}</div>
                         <div class="text-secondary text-md">Serial Number</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.wwn}}</div>
+                        <div>{{device?.wwn}}</div>
                         <div class="text-secondary text-md">LU WWN Device Id</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.firmware}}</div>
+                        <div>{{device?.firmware}}</div>
                         <div class="text-secondary text-md">Firmware Version</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.capacity | fileSize}}</div>
+                        <div>{{device?.capacity | fileSize}}</div>
                         <div class="text-secondary text-md">Capacity</div>
                     </div>
-                    <div *ngIf="data.data.rotational_speed" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.rotational_speed}} RPM</div>
+                    <div *ngIf="device?.rotational_speed" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.rotational_speed}} RPM</div>
                         <div class="text-secondary text-md">Rotation Rate</div>
                     </div>
-                    <div *ngIf="data.data.device_protocol" class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.device_protocol}}</div>
+                    <div *ngIf="device?.device_protocol" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{device?.device_protocol}}</div>
                         <div class="text-secondary text-md">Protocol</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.smart_results[0]?.power_cycle_count}}</div>
+                        <div>{{smart_results[0]?.power_cycle_count}}</div>
                         <div class="text-secondary text-md">Power Cycle Count</div>
                     </div>
-                    <div *ngIf="data.data.smart_results[0]?.power_on_hours"  class="my-2 col-span-2 lt-md:col-span-1">
-                        <div matTooltip="{{humanizeDuration(data.data.smart_results[0]?.power_on_hours * 60 * 60 * 1000, { conjunction: ' and ', serialComma: false })}}">{{humanizeDuration(data.data.smart_results[0]?.power_on_hours *60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] })}}</div>
+                    <div *ngIf="smart_results[0]?.power_on_hours"  class="my-2 col-span-2 lt-md:col-span-1">
+                        <div matTooltip="{{humanizeDuration(smart_results[0]?.power_on_hours * 60 * 60 * 1000, { conjunction: ' and ', serialComma: false })}}">{{humanizeDuration(smart_results[0]?.power_on_hours *60 * 60 * 1000, { round: true, largest: 1, units: ['y', 'd', 'h'] })}}</div>
                         <div class="text-secondary text-md">Powered On</div>
                     </div>
                     <div class="my-2 col-span-2 lt-md:col-span-1">
-                        <div>{{data.data.smart_results[0]?.temp}}°C</div>
+                        <div>{{smart_results[0]?.temp}}°C</div>
                         <div class="text-secondary text-md">Temperature</div>
                     </div>
                 </div>
@@ -115,7 +129,7 @@
         <div class="flex flex-auto w-3/4 p-4 lt-md:w-full">
             <div class="flex flex-col flex-auto w-full bg-card shadow-md rounded ">
                 <div class="p-6">
-                    <div class="font-bold text-md text-secondary uppercase tracking-wider">S.M.A.R.T {{data.data.device_protocol}} Attributes</div>
+                    <div class="font-bold text-md text-secondary uppercase tracking-wider">S.M.A.R.T {{device?.device_protocol}} Attributes</div>
                     <div class="text-sm text-hint font-medium">{{this.smartAttributeDataSource.data.length}} visible, {{getHiddenAttributes()}} hidden</div>
                 </div>
                 <div class="overflow-auto">
@@ -139,15 +153,15 @@
                             <td mat-cell
                                 *matCellDef="let attribute">
                                 <span class="inline-flex items-center font-bold text-xs px-2 py-2px rounded-full tracking-wide uppercase"
-                                      [ngClass]="{'red-200': attribute.status === 'failed',
-                                      'green-200': attribute.status === 'passed',
-                                      'yellow-200': attribute.status === 'warn'
+                                      [ngClass]="{'red-200': getAttributeStatusName(attribute.status) === 'failed',
+                                      'green-200': getAttributeStatusName(attribute.status) === 'passed',
+                                      'yellow-200': getAttributeStatusName(attribute.status) === 'warn'
                                       }">
                                     <span class="w-2 h-2 rounded-full mr-2"
-                                          [ngClass]="{'bg-red': attribute.status === 'failed',
-                                      'bg-green': attribute.status === 'passed',
-                                      'bg-yellow': attribute.status === 'warn'}"></span>
-                                    <span class="pr-2px leading-relaxed whitespace-no-wrap" matTooltip="{{attribute.status_reason}}">{{attribute.status}}</span>
+                                          [ngClass]="{'bg-red': getAttributeStatusName(attribute.status) === 'failed',
+                                      'bg-green': getAttributeStatusName(attribute.status) === 'passed',
+                                      'bg-yellow': getAttributeStatusName(attribute.status) === 'warn'}"></span>
+                                    <span class="pr-2px leading-relaxed whitespace-no-wrap" matTooltip="{{attribute.status_reason}}">{{getAttributeStatusName(attribute.status)}}</span>
                                 </span>
                             </td>
                         </ng-container>
@@ -183,7 +197,7 @@
                             <td mat-cell
                                 *matCellDef="let attribute">
                                 <span class="pr-6 whitespace-no-wrap" matTooltip="{{getAttributeDescription(attribute)}}">
-                                    {{attribute.name}} <mat-icon *ngIf="getAttributeDescription(attribute)" class="icon-size-10" [svgIcon]="'info'"></mat-icon>
+                                    {{getAttributeName(attribute)}} <mat-icon *ngIf="getAttributeDescription(attribute)" class="icon-size-10" [svgIcon]="'info'"></mat-icon>
                                 </span>
                             </td>
                         </ng-container>

webapp/frontend/src/app/modules/detail/detail.component.ts

@@ -19,7 +19,12 @@ import humanizeDuration from 'humanize-duration';
 export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     onlyCritical: boolean = true;
-    data: any;
+    // data: any;
+
+    metadata: any;
+    device: any;
+    smart_results: any[];
+
     commonSparklineOptions: Partial<ApexOptions>;
     smartAttributeDataSource: MatTableDataSource<any>;
     smartAttributeTableColumns: string[];
@@ -66,10 +71,14 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
             .subscribe((data) => {
 
                 // Store the data
-                this.data = data;
+                // this.data = data;
+                this.device = data.data.device;
+                this.smart_results = data.data.smart_results
+                this.metadata = data.metadata;
+
 
                 // Store the table data
-                this.smartAttributeDataSource.data = this._generateSmartAttributeTableDataSource(data.data.smart_results);
+                this.smartAttributeDataSource.data = this._generateSmartAttributeTableDataSource(this.smart_results);
 
                 // Prepare the chart data
                 this._prepareChartData();
@@ -98,8 +107,28 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     // -----------------------------------------------------------------------------------------------------
     // @ Private methods
     // -----------------------------------------------------------------------------------------------------
+    getAttributeStatusName(attribute_status){
+        if(attribute_status == 0){
+            return "passed"
+        } else if (attribute_status == 1){
+            return "warn"
+        } else if (attribute_status == 2){
+            return "failed"
+        }
+        return
+    }
+
+    getAttributeName(attribute_data){
+        let attribute_metadata = this.metadata[attribute_data.attribute_id]
+        if(!attribute_metadata){
+            return 'Unknown Attribute Name'
+        } else {
+            return attribute_metadata.display_name
+        }
+        return
+    }
     getAttributeDescription(attribute_data){
-        let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+        let attribute_metadata = this.metadata[attribute_data.attribute_id]
         if(!attribute_metadata){
             return 'Unknown'
         } else {
@@ -110,7 +139,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     getAttributeValue(attribute_data){
         if(this.isAta()) {
-            let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+            let attribute_metadata = this.metadata[attribute_data.attribute_id]
             if(!attribute_metadata){
                 return attribute_data.value
             } else if (attribute_metadata.display_type == "raw") {
@@ -128,7 +157,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     getAttributeValueType(attribute_data){
         if(this.isAta()) {
-            let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+            let attribute_metadata = this.metadata[attribute_data.attribute_id]
             if(!attribute_metadata){
                 return ''
             } else {
@@ -141,14 +170,14 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     getAttributeIdeal(attribute_data){
         if(this.isAta()){
-            return this.data.metadata[attribute_data.attribute_id]?.display_type == "raw" ? this.data.metadata[attribute_data.attribute_id]?.ideal : ''
+            return this.metadata[attribute_data.attribute_id]?.display_type == "raw" ? this.metadata[attribute_data.attribute_id]?.ideal : ''
         } else {
-            return this.data.metadata[attribute_data.attribute_id]?.ideal
+            return this.metadata[attribute_data.attribute_id]?.ideal
         }
     }
 
     getAttributeWorst(attribute_data){
-        let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+        let attribute_metadata = this.metadata[attribute_data.attribute_id]
         if(!attribute_metadata){
             return attribute_data.worst
         } else {
@@ -158,7 +187,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
 
     getAttributeThreshold(attribute_data){
         if(this.isAta()){
-            let attribute_metadata = this.data.metadata[attribute_data.attribute_id]
+            let attribute_metadata = this.metadata[attribute_data.attribute_id]
             if(!attribute_metadata || attribute_metadata.display_type == "normalized"){
                 return attribute_data.thresh
             } else {
@@ -175,29 +204,30 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     }
 
     getAttributeCritical(attribute_data){
-        return this.data.metadata[attribute_data.attribute_id]?.critical
+        return this.metadata[attribute_data.attribute_id]?.critical
     }
     getHiddenAttributes(){
-        let attributes_list
-        if(this.isAta()){
-            attributes_list = this.data.data.smart_results[0]?.ata_attributes
-        } else if(this.isNvme()){
-            attributes_list = this.data.data.smart_results[0]?.nvme_attributes
-        } else {
-            attributes_list = this.data.data.smart_results[0]?.scsi_attributes
+        if (!this.smart_results || this.smart_results.length == 0) {
+            return 0
         }
 
-        return attributes_list.length - this.smartAttributeDataSource.data.length
+        let attributes_length = 0
+        let attributes = this.smart_results[0]?.attrs
+        if (attributes) {
+            attributes_length = Object.keys(attributes).length
+        }
+
+        return attributes_length - this.smartAttributeDataSource.data.length
     }
 
     isAta(): boolean {
-        return this.data.data.device_protocol == 'ATA'
+        return this.device.device_protocol == 'ATA'
     }
     isScsi(): boolean {
-        return this.data.data.device_protocol == 'SCSI'
+        return this.device.device_protocol == 'SCSI'
     }
     isNvme(): boolean {
-        return this.data.data.device_protocol == 'NVMe'
+        return this.device.device_protocol == 'NVMe'
     }
 
     private _generateSmartAttributeTableDataSource(smart_results){
@@ -207,35 +237,44 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
             return smartAttributeDataSource
         }
         var latest_smart_result = smart_results[0];
-        let attributes_list = []
+        let attributes = {}
         if(this.isScsi()) {
             this.smartAttributeTableColumns = ['status', 'name', 'value', 'thresh', 'history'];
-            attributes_list = latest_smart_result.scsi_attributes
+            attributes = latest_smart_result.attrs
         } else if(this.isNvme()){
             this.smartAttributeTableColumns = ['status', 'name', 'value', 'thresh', 'ideal', 'history'];
-            attributes_list = latest_smart_result.nvme_attributes
+            attributes = latest_smart_result.attrs
         } else {
             //ATA
-            attributes_list = latest_smart_result.ata_attributes
+            attributes = latest_smart_result.attrs
             this.smartAttributeTableColumns = ['status', 'id', 'name', 'value', 'worst', 'thresh','ideal', 'failure', 'history'];
         }
 
+        for(const attrId in attributes){
+            var attr = attributes[attrId]
 
-        for(let attr of attributes_list){
             //chart history data
             if (!attr.chartData) {
-                var rawHistory = (attr.history || []).map(hist_attr => this.getAttributeValue(hist_attr)).reverse()
-                rawHistory.push(this.getAttributeValue(attr))
-                attr.chartData = [
+
+
+                var attrHistory = []
+                for (let smart_result of smart_results){
+                    attrHistory.push(this.getAttributeValue(smart_result.attrs[attrId]))
+                }
+
+                // var rawHistory = (attr.history || []).map(hist_attr => this.getAttributeValue(hist_attr)).reverse()
+                // rawHistory.push(this.getAttributeValue(attr))
+
+                attributes[attrId].chartData = [
                     {
                         name: "chart-line-sparkline",
-                        data: rawHistory
+                        data: attrHistory
                     }
                 ]
             }
             //determine when to include the attributes in table.
 
-            if(!this.onlyCritical || this.onlyCritical && this.data.metadata[attr.attribute_id]?.critical || attr.value < attr.thresh){
+            if(!this.onlyCritical || this.onlyCritical && this.metadata[attr.attribute_id]?.critical || attr.value < attr.thresh){
                 smartAttributeDataSource.push(attr)
             }
         }
@@ -297,7 +336,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
     }
     toggleOnlyCritical(){
         this.onlyCritical = !this.onlyCritical
-        this.smartAttributeDataSource.data = this._generateSmartAttributeTableDataSource(this.data.data.smart_results);
+        this.smartAttributeDataSource.data = this._generateSmartAttributeTableDataSource(this.smart_results);
 
     }