init

webapp/backend/pkg/config/config.go

@@ -0,0 +1,116 @@
+package config
+
+import (
+	"github.com/analogj/go-util/utils"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
+	"github.com/spf13/viper"
+	"log"
+	"os"
+)
+
+// When initializing this class the following methods must be called:
+// Config.New
+// Config.Init
+// This is done automatically when created via the Factory.
+type configuration struct {
+	*viper.Viper
+}
+
+//Viper uses the following precedence order. Each item takes precedence over the item below it:
+// explicit call to Set
+// flag
+// env
+// config
+// key/value store
+// default
+
+func (c *configuration) Init() error {
+	c.Viper = viper.New()
+	//set defaults
+	c.SetDefault("web.listen.port", "8080")
+	c.SetDefault("web.listen.host", "0.0.0.0")
+	c.SetDefault("web.src.frontend.path", "/scrutiny/web")
+
+	c.SetDefault("web.database.location", "/scrutiny/config/scrutiny.db")
+
+	c.SetDefault("disks.include", []string{})
+	c.SetDefault("disks.exclude", []string{})
+
+	c.SetDefault("notify.metric.script", "/scrutiny/config/notify-metrics.sh")
+	c.SetDefault("notify.long.script", "/scrutiny/config/notify-long-test.sh")
+	c.SetDefault("notify.short.script", "/scrutiny/config/notify-short-test.sh")
+
+	c.SetDefault("collect.metric.enable", true)
+	c.SetDefault("collect.metric.command", "-a -o on -S on")
+	c.SetDefault("collect.long.enable", true)
+	c.SetDefault("collect.long.command", "-a -o on -S on")
+	c.SetDefault("collect.short.enable", true)
+	c.SetDefault("collect.short.command", "-a -o on -S on")
+
+	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
+	c.SetConfigType("yaml")
+	//c.SetConfigName("drawbridge")
+	//c.AddConfigPath("$HOME/")
+
+	//CLI options will be added via the `Set()` function
+	return nil
+}
+
+func (c *configuration) ReadConfig(configFilePath string) error {
+	configFilePath, err := utils.ExpandPath(configFilePath)
+	if err != nil {
+		return err
+	}
+
+	if !utils.FileExists(configFilePath) {
+		log.Printf("No configuration file found at %v. Skipping", configFilePath)
+		return errors.ConfigFileMissingError("The configuration file could not be found.")
+	}
+
+	//validate config file contents
+	//err = c.ValidateConfigFile(configFilePath)
+	//if err != nil {
+	//	log.Printf("Config file at `%v` is invalid: %s", configFilePath, err)
+	//	return err
+	//}
+
+	log.Printf("Loading configuration file: %s", configFilePath)
+
+	config_data, err := os.Open(configFilePath)
+	if err != nil {
+		log.Printf("Error reading configuration file: %s", err)
+		return err
+	}
+
+	err = c.MergeConfig(config_data)
+	if err != nil {
+		return err
+	}
+
+	return c.ValidateConfig()
+}
+
+// This function ensures that the merged config works correctly.
+func (c *configuration) ValidateConfig() error {
+
+	////deserialize Questions
+	//questionsMap := map[string]Question{}
+	//err := c.UnmarshalKey("questions", &questionsMap)
+	//
+	//if err != nil {
+	//	log.Printf("questions could not be deserialized correctly. %v", err)
+	//	return err
+	//}
+	//
+	//for _, v := range questionsMap {
+	//
+	//	typeContent, ok := v.Schema["type"].(string)
+	//	if !ok || len(typeContent) == 0 {
+	//		return errors.QuestionSyntaxError("`type` is required for questions")
+	//	}
+	//}
+	//
+	//
+
+	return nil
+}

webapp/backend/pkg/config/factory.go

@@ -0,0 +1,9 @@
+package config
+
+func Create() (Interface, error) {
+	config := new(configuration)
+	if err := config.Init(); err != nil {
+		return nil, err
+	}
+	return config, nil
+}

webapp/backend/pkg/config/interface.go

@@ -0,0 +1,23 @@
+package config
+
+import (
+	"github.com/spf13/viper"
+)
+
+// Create mock using:
+// mockgen -source=pkg/config/interface.go -destination=pkg/config/mock/mock_config.go
+type Interface interface {
+	Init() error
+	ReadConfig(configFilePath string) error
+	Set(key string, value interface{})
+	SetDefault(key string, value interface{})
+
+	AllSettings() map[string]interface{}
+	IsSet(key string) bool
+	Get(key string) interface{}
+	GetBool(key string) bool
+	GetInt(key string) int
+	GetString(key string) string
+	GetStringSlice(key string) []string
+	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
+}

webapp/backend/pkg/database/sqlite3.go

@@ -0,0 +1,30 @@
+package database
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/gin-gonic/gin"
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+)
+
+func DatabaseHandler(dbPath string) gin.HandlerFunc {
+	//var database *gorm.DB
+	fmt.Printf("Trying to connect to database stored: %s", dbPath)
+	database, err := gorm.Open("sqlite3", dbPath)
+
+	if err != nil {
+		panic("Failed to connect to database!")
+	}
+
+	database.AutoMigrate(&db.Device{})
+	database.AutoMigrate(&db.SelfTest{})
+	database.AutoMigrate(&db.Smart{})
+	database.AutoMigrate(&db.SmartAttribute{})
+
+	//TODO: detrmine where we can call defer database.Close()
+	return func(c *gin.Context) {
+		c.Set("DB", database)
+		c.Next()
+	}
+}

webapp/backend/pkg/errors/errors.go

@@ -0,0 +1,26 @@
+package errors
+
+import (
+	"fmt"
+)
+
+// Raised when config file is missing
+type ConfigFileMissingError string
+
+func (str ConfigFileMissingError) Error() string {
+	return fmt.Sprintf("ConfigFileMissingError: %q", string(str))
+}
+
+// Raised when the config file doesnt match schema
+type ConfigValidationError string
+
+func (str ConfigValidationError) Error() string {
+	return fmt.Sprintf("ConfigValidationError: %q", string(str))
+}
+
+// Raised when a dependency (like smartd or ssh-agent) is missing
+type DependencyMissingError string
+
+func (str DependencyMissingError) Error() string {
+	return fmt.Sprintf("DependencyMissingError: %q", string(str))
+}

webapp/backend/pkg/errors/errors_test.go

@@ -0,0 +1,34 @@
+package errors_test
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+//func TestCheckErr_WithoutError(t *testing.T) {
+//	t.Parallel()
+//
+//	//assert
+//	require.NotPanics(t, func() {
+//		errors.CheckErr(nil)
+//	})
+//}
+
+//func TestCheckErr_Error(t *testing.T) {
+//	t.Parallel()
+//
+//	//assert
+//	require.Panics(t, func() {
+//		errors.CheckErr(stderrors.New("This is an error"))
+//	})
+//}
+
+func TestErrors(t *testing.T) {
+	t.Parallel()
+
+	//assert
+	require.Implements(t, (*error)(nil), errors.ConfigFileMissingError("test"), "should implement the error interface")
+	require.Implements(t, (*error)(nil), errors.ConfigValidationError("test"), "should implement the error interface")
+	require.Implements(t, (*error)(nil), errors.DependencyMissingError("test"), "should implement the error interface")
+}

webapp/backend/pkg/models/collector/smart.go

@@ -0,0 +1,186 @@
+package collector
+
+type SmartInfo struct {
+	JSONFormatVersion []int `json:"json_format_version"`
+	Smartctl          struct {
+		Version      []int    `json:"version"`
+		SvnRevision  string   `json:"svn_revision"`
+		PlatformInfo string   `json:"platform_info"`
+		BuildInfo    string   `json:"build_info"`
+		Argv         []string `json:"argv"`
+		ExitStatus   int      `json:"exit_status"`
+		Messages     []struct {
+			String   string `json:"string"`
+			Severity string `json:"severity"`
+		} `json:"messages"`
+	} `json:"smartctl"`
+	Device struct {
+		Name     string `json:"name"`
+		InfoName string `json:"info_name"`
+		Type     string `json:"type"`
+		Protocol string `json:"protocol"`
+	} `json:"device"`
+	ModelName    string `json:"model_name"`
+	SerialNumber string `json:"serial_number"`
+	Wwn          struct {
+		Naa int   `json:"naa"`
+		Oui int   `json:"oui"`
+		ID  int64 `json:"id"`
+	} `json:"wwn"`
+	FirmwareVersion string `json:"firmware_version"`
+	UserCapacity    struct {
+		Blocks int64 `json:"blocks"`
+		Bytes  int64 `json:"bytes"`
+	} `json:"user_capacity"`
+	LogicalBlockSize  int `json:"logical_block_size"`
+	PhysicalBlockSize int `json:"physical_block_size"`
+	RotationRate      int `json:"rotation_rate"`
+	FormFactor        struct {
+		AtaValue int    `json:"ata_value"`
+		Name     string `json:"name"`
+	} `json:"form_factor"`
+	InSmartctlDatabase bool `json:"in_smartctl_database"`
+	AtaVersion         struct {
+		String     string `json:"string"`
+		MajorValue int    `json:"major_value"`
+		MinorValue int    `json:"minor_value"`
+	} `json:"ata_version"`
+	SataVersion struct {
+		String string `json:"string"`
+		Value  int    `json:"value"`
+	} `json:"sata_version"`
+	InterfaceSpeed struct {
+		Max struct {
+			SataValue      int    `json:"sata_value"`
+			String         string `json:"string"`
+			UnitsPerSecond int    `json:"units_per_second"`
+			BitsPerUnit    int    `json:"bits_per_unit"`
+		} `json:"max"`
+		Current struct {
+			SataValue      int    `json:"sata_value"`
+			String         string `json:"string"`
+			UnitsPerSecond int    `json:"units_per_second"`
+			BitsPerUnit    int    `json:"bits_per_unit"`
+		} `json:"current"`
+	} `json:"interface_speed"`
+	LocalTime struct {
+		TimeT   int64  `json:"time_t"`
+		Asctime string `json:"asctime"`
+	} `json:"local_time"`
+	SmartStatus struct {
+		Passed bool `json:"passed"`
+	} `json:"smart_status"`
+	AtaSmartData struct {
+		OfflineDataCollection struct {
+			Status struct {
+				Value  int    `json:"value"`
+				String string `json:"string"`
+				Passed bool   `json:"passed"`
+			} `json:"status"`
+			CompletionSeconds int `json:"completion_seconds"`
+		} `json:"offline_data_collection"`
+		SelfTest struct {
+			Status struct {
+				Value            int    `json:"value"`
+				String           string `json:"string"`
+				RemainingPercent int    `json:"remaining_percent"`
+			} `json:"status"`
+			PollingMinutes struct {
+				Short    int `json:"short"`
+				Extended int `json:"extended"`
+			} `json:"polling_minutes"`
+		} `json:"self_test"`
+		Capabilities struct {
+			Values                        []int `json:"values"`
+			ExecOfflineImmediateSupported bool  `json:"exec_offline_immediate_supported"`
+			OfflineIsAbortedUponNewCmd    bool  `json:"offline_is_aborted_upon_new_cmd"`
+			OfflineSurfaceScanSupported   bool  `json:"offline_surface_scan_supported"`
+			SelfTestsSupported            bool  `json:"self_tests_supported"`
+			ConveyanceSelfTestSupported   bool  `json:"conveyance_self_test_supported"`
+			SelectiveSelfTestSupported    bool  `json:"selective_self_test_supported"`
+			AttributeAutosaveEnabled      bool  `json:"attribute_autosave_enabled"`
+			ErrorLoggingSupported         bool  `json:"error_logging_supported"`
+			GpLoggingSupported            bool  `json:"gp_logging_supported"`
+		} `json:"capabilities"`
+	} `json:"ata_smart_data"`
+	AtaSctCapabilities struct {
+		Value                         int  `json:"value"`
+		ErrorRecoveryControlSupported bool `json:"error_recovery_control_supported"`
+		FeatureControlSupported       bool `json:"feature_control_supported"`
+		DataTableSupported            bool `json:"data_table_supported"`
+	} `json:"ata_sct_capabilities"`
+	AtaSmartAttributes struct {
+		Revision int `json:"revision"`
+		Table    []struct {
+			ID         int    `json:"id"`
+			Name       string `json:"name"`
+			Value      int    `json:"value"`
+			Worst      int    `json:"worst"`
+			Thresh     int    `json:"thresh"`
+			WhenFailed string `json:"when_failed"`
+			Flags      struct {
+				Value         int    `json:"value"`
+				String        string `json:"string"`
+				Prefailure    bool   `json:"prefailure"`
+				UpdatedOnline bool   `json:"updated_online"`
+				Performance   bool   `json:"performance"`
+				ErrorRate     bool   `json:"error_rate"`
+				EventCount    bool   `json:"event_count"`
+				AutoKeep      bool   `json:"auto_keep"`
+			} `json:"flags"`
+			Raw struct {
+				Value  int64  `json:"value"`
+				String string `json:"string"`
+			} `json:"raw"`
+		} `json:"table"`
+	} `json:"ata_smart_attributes"`
+	PowerOnTime struct {
+		Hours int64 `json:"hours"`
+	} `json:"power_on_time"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+	Temperature     struct {
+		Current int64 `json:"current"`
+	} `json:"temperature"`
+	AtaSmartErrorLog struct {
+		Summary struct {
+			Revision int `json:"revision"`
+			Count    int `json:"count"`
+		} `json:"summary"`
+	} `json:"ata_smart_error_log"`
+	AtaSmartSelfTestLog struct {
+		Standard struct {
+			Revision int `json:"revision"`
+			Table    []struct {
+				Type struct {
+					Value  int    `json:"value"`
+					String string `json:"string"`
+				} `json:"type"`
+				Status struct {
+					Value  int    `json:"value"`
+					String string `json:"string"`
+					Passed bool   `json:"passed"`
+				} `json:"status"`
+				LifetimeHours int `json:"lifetime_hours"`
+			} `json:"table"`
+			Count              int `json:"count"`
+			ErrorCountTotal    int `json:"error_count_total"`
+			ErrorCountOutdated int `json:"error_count_outdated"`
+		} `json:"standard"`
+	} `json:"ata_smart_self_test_log"`
+	AtaSmartSelectiveSelfTestLog struct {
+		Revision int `json:"revision"`
+		Table    []struct {
+			LbaMin int `json:"lba_min"`
+			LbaMax int `json:"lba_max"`
+			Status struct {
+				Value  int    `json:"value"`
+				String string `json:"string"`
+			} `json:"status"`
+		} `json:"table"`
+		Flags struct {
+			Value                int  `json:"value"`
+			RemainderScanEnabled bool `json:"remainder_scan_enabled"`
+		} `json:"flags"`
+		PowerUpScanResumeMinutes int `json:"power_up_scan_resume_minutes"`
+	} `json:"ata_smart_selective_self_test_log"`
+}

webapp/backend/pkg/models/db/device.go

@@ -0,0 +1,110 @@
+package db
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"strings"
+	"time"
+)
+
+type DeviceRespWrapper struct {
+	Success bool     `json:"success"`
+	Errors  []error  `json:"errors"`
+	Data    []Device `json:"data"`
+}
+
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN string `json:"wwn" gorm:"primary_key"`
+
+	DeviceName     string `json:"device_name"`
+	Manufacturer   string `json:"manufacturer"`
+	ModelName      string `json:"model_name"`
+	InterfaceType  string `json:"interface_type"`
+	InterfaceSpeed string `json:"interface_speed"`
+	SerialNumber   string `json:"serial_number"`
+	Firmware       string `json:"firmware"`
+	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
+
+	SmartResults []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
+}
+
+//This method requires a device with an array of SmartResults.
+//It will remove all SmartResults other than the first (the latest one)
+//All removed SmartResults, will be processed, grouping SmartAttribute by attribute_id
+// and adding theme to an array called History.
+func (dv *Device) SquashHistory() error {
+	if len(dv.SmartResults) <= 1 {
+		return nil //no history found. ignore
+	}
+
+	latestSmartResultSlice := dv.SmartResults[0:1]
+	historicalSmartResultSlice := dv.SmartResults[1:]
+
+	//re-assign the latest slice to the SmartResults field
+	dv.SmartResults = latestSmartResultSlice
+
+	//process the historical slice
+	history := map[int][]SmartAttribute{}
+	for _, smartResult := range historicalSmartResultSlice {
+		for _, smartAttribute := range smartResult.SmartAttributes {
+			if _, ok := history[smartAttribute.AttributeId]; !ok {
+				history[smartAttribute.AttributeId] = []SmartAttribute{}
+			}
+			history[smartAttribute.AttributeId] = append(history[smartAttribute.AttributeId], smartAttribute)
+		}
+	}
+
+	//now assign the historical slices to the SmartAttributes in the latest SmartResults
+	for sandx, smartAttribute := range dv.SmartResults[0].SmartAttributes {
+		if attributeHistory, ok := history[smartAttribute.AttributeId]; ok {
+			dv.SmartResults[0].SmartAttributes[sandx].History = attributeHistory
+		}
+	}
+
+	return nil
+}
+
+func (dv *Device) ApplyMetadataRules() error {
+	//embed metadata in the latest smart attributes object
+	if len(dv.SmartResults) > 0 {
+		for ndx, attr := range dv.SmartResults[0].SmartAttributes {
+			if strings.ToUpper(attr.WhenFailed) == SmartWhenFailedFailingNow {
+				//this attribute has previously failed
+				dv.SmartResults[0].SmartAttributes[ndx].Status = SmartAttributeStatusFailed
+				dv.SmartResults[0].SmartAttributes[ndx].StatusReason = "Attribute is failing manufacturer SMART threshold"
+
+			} else if strings.ToUpper(attr.WhenFailed) == SmartWhenFailedInThePast {
+				dv.SmartResults[0].SmartAttributes[ndx].Status = SmartAttributeStatusWarning
+				dv.SmartResults[0].SmartAttributes[ndx].StatusReason = "Attribute has previously failed manufacturer SMART threshold"
+			}
+
+			if smartMetadata, ok := metadata.AtaSmartAttributes[attr.AttributeId]; ok {
+				dv.SmartResults[0].SmartAttributes[ndx].MetadataObservedThresholdStatus(smartMetadata)
+			}
+
+			//check if status is blank, set to "passed"
+			if len(dv.SmartResults[0].SmartAttributes[ndx].Status) == 0 {
+				dv.SmartResults[0].SmartAttributes[ndx].Status = SmartAttributeStatusPassed
+			}
+		}
+	}
+	return nil
+}
+
+func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
+	dv.InterfaceSpeed = info.InterfaceSpeed.Current.String
+	dv.Firmware = info.FirmwareVersion
+	dv.RotationSpeed = info.RotationRate
+	dv.Capacity = info.UserCapacity.Bytes
+	dv.FormFactor = info.FormFactor.Name
+	//dv.SmartSupport =
+	return nil
+}

webapp/backend/pkg/models/db/selftest.go

@@ -0,0 +1,15 @@
+package db
+
+import "time"
+
+type SelfTest struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	DeviceWWN string
+	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
+
+	Date time.Time
+}

webapp/backend/pkg/models/db/smart.go

@@ -0,0 +1,150 @@
+package db
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/jinzhu/gorm"
+	"time"
+)
+
+const SmartWhenFailedFailingNow = "FAILING_NOW"
+const SmartWhenFailedInThePast = "IN_THE_PAST"
+
+type Smart struct {
+	gorm.Model
+
+	DeviceWWN string `json:"device_wwn"`
+	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
+
+	TestDate    time.Time `json:"date"`
+	SmartStatus string    `json:"smart_status"`
+
+	//Metrics
+	Temp            int64 `json:"temp"`
+	PowerOnHours    int64 `json:"power_on_hours"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+
+	SmartAttributes []SmartAttribute `json:"smart_attributes" gorm:"foreignkey:SmartId"`
+}
+
+func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error {
+	sm.DeviceWWN = wwn
+	sm.TestDate = time.Unix(info.LocalTime.TimeT, 0)
+
+	//smart metrics
+	sm.Temp = info.Temperature.Current
+	sm.PowerCycleCount = info.PowerCycleCount
+	sm.PowerOnHours = info.PowerOnTime.Hours
+
+	sm.SmartAttributes = []SmartAttribute{}
+	for _, collectorAttr := range info.AtaSmartAttributes.Table {
+		attrModel := SmartAttribute{
+			AttributeId: collectorAttr.ID,
+			Name:        collectorAttr.Name,
+			Value:       collectorAttr.Value,
+			Worst:       collectorAttr.Worst,
+			Threshold:   collectorAttr.Thresh,
+			RawValue:    collectorAttr.Raw.Value,
+			RawString:   collectorAttr.Raw.String,
+			WhenFailed:  collectorAttr.WhenFailed,
+		}
+
+		//now that we've parsed the data from the smartctl response, lets match it against our metadata rules and add additional Scrutiny specific data.
+		if smartMetadata, ok := metadata.AtaSmartAttributes[collectorAttr.ID]; ok {
+			attrModel.Name = smartMetadata.DisplayName
+			if smartMetadata.Transform != nil {
+				attrModel.TransformedValue = smartMetadata.Transform(attrModel.Value, attrModel.RawValue, attrModel.RawString)
+			}
+		}
+		sm.SmartAttributes = append(sm.SmartAttributes, attrModel)
+	}
+
+	if info.SmartStatus.Passed {
+		sm.SmartStatus = "passed"
+	} else {
+		sm.SmartStatus = "failed"
+	}
+	return nil
+}
+
+const SmartAttributeStatusPassed = "passed"
+const SmartAttributeStatusFailed = "failed"
+const SmartAttributeStatusWarning = "warn"
+
+type SmartAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId int    `json:"attribute_id"`
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Worst       int    `json:"worst"`
+	Threshold   int    `json:"thresh"`
+	RawValue    int64  `json:"raw_value"`
+	RawString   string `json:"raw_string"`
+	WhenFailed  string `json:"when_failed"`
+
+	TransformedValue int64            `json:"transformed_value"`
+	Status           string           `gorm:"-" json:"status,omitempty"`
+	StatusReason     string           `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64          `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartAttribute `gorm:"-" json:"history,omitempty"`
+}
+
+// compare the attribute (raw, normalized, transformed) value to observed thresholds, and update status if necessary
+func (sa *SmartAttribute) MetadataObservedThresholdStatus(smartMetadata metadata.AtaSmartAttribute) {
+	//TODO: multiple rules
+	// try to predict the failure rates for observed thresholds that have 0 failure rate and error bars.
+	// - if the attribute is critical
+	//		- the failure rate is over 10 - set to failed
+	//		- the attribute does not match any threshold, set to warn
+	// - if the attribute is not critical
+	//		- if failure rate is above 20 - set to failed
+	// 		- if failure rate is above 10 but below 20 - set to warn
+
+	//update the smart attribute status based on Observed thresholds.
+	var value int64
+	if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeNormalized {
+		value = int64(sa.Value)
+	} else if smartMetadata.DisplayType == metadata.AtaSmartAttributeDisplayTypeTransformed {
+		value = sa.TransformedValue
+	} else {
+		value = sa.RawValue
+	}
+
+	for _, obsThresh := range smartMetadata.ObservedThresholds {
+
+		//check if "value" is in this bucket
+		if ((obsThresh.Low == obsThresh.High) && value == obsThresh.Low) ||
+			(obsThresh.Low < value && value <= obsThresh.High) {
+			sa.FailureRate = obsThresh.AnnualFailureRate
+
+			if smartMetadata.Critical {
+				if obsThresh.AnnualFailureRate >= 0.10 {
+					sa.Status = SmartAttributeStatusFailed
+					sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
+				}
+			} else {
+				if obsThresh.AnnualFailureRate >= 0.20 {
+					sa.Status = SmartAttributeStatusFailed
+					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
+				} else if obsThresh.AnnualFailureRate >= 0.10 {
+					sa.Status = SmartAttributeStatusWarning
+					sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
+				}
+			}
+
+			//we've found the correct bucket, we can drop out of this loop
+			return
+		}
+	}
+	// no bucket found
+	if smartMetadata.Critical {
+		sa.Status = SmartAttributeStatusWarning
+		sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
+	}
+
+	return
+}

webapp/backend/pkg/models/db/smart_test.go

@@ -0,0 +1,42 @@
+package db_test
+
+import (
+	"encoding/json"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/stretchr/testify/require"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestFromCollectorSmartInfo(t *testing.T) {
+	//setup
+	smartDataFile, err := os.Open("../testdata/smart.json")
+	require.NoError(t, err)
+	defer smartDataFile.Close()
+
+	var smartJson collector.SmartInfo
+
+	smartDataBytes, err := ioutil.ReadAll(smartDataFile)
+	require.NoError(t, err)
+	err = json.Unmarshal(smartDataBytes, &smartJson)
+	require.NoError(t, err)
+
+	//test
+	smartMdl := db.Smart{}
+	err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson)
+
+	//assert
+	require.NoError(t, err)
+	require.Equal(t, smartMdl.DeviceWWN, "WWN-test")
+	require.Equal(t, smartMdl.SmartStatus, "PASSED")
+
+	//check that temperature was correctly parsed
+	for _, attr := range smartMdl.SmartAttributes {
+		if attr.AttributeId == 194 {
+			require.Equal(t, int64(163210330144), attr.RawValue)
+			require.Equal(t, int64(32), attr.TransformedValue)
+		}
+	}
+}

webapp/backend/pkg/models/testdata/smart.json

@@ -0,0 +1,846 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.128-flatcar",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-j",
+      "-a",
+      "/dev/sdb"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/sdb",
+    "info_name": "/dev/sdb [SAT]",
+    "type": "sat",
+    "protocol": "ATA"
+  },
+  "model_name": "WDC WD140EDFZ-11A0VA0",
+  "serial_number": "9RK1XXXX",
+  "wwn": {
+    "naa": 5,
+    "oui": 3274,
+    "id": 10283057623
+  },
+  "firmware_version": "81.00A81",
+  "user_capacity": {
+    "blocks": 27344764928,
+    "bytes": 14000519643136
+  },
+  "logical_block_size": 512,
+  "physical_block_size": 4096,
+  "rotation_rate": 5400,
+  "form_factor": {
+    "ata_value": 2,
+    "name": "3.5 inches"
+  },
+  "in_smartctl_database": false,
+  "ata_version": {
+    "string": "ACS-2, ATA8-ACS T13/1699-D revision 4",
+    "major_value": 1020,
+    "minor_value": 41
+  },
+  "sata_version": {
+    "string": "SATA 3.2",
+    "value": 255
+  },
+  "interface_speed": {
+    "max": {
+      "sata_value": 14,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    },
+    "current": {
+      "sata_value": 3,
+      "string": "6.0 Gb/s",
+      "units_per_second": 60,
+      "bits_per_unit": 100000000
+    }
+  },
+  "local_time": {
+    "time_t": 1592697810,
+    "asctime": "Sun Jun 21 00:03:30 2020 UTC"
+  },
+  "smart_status": {
+    "passed": true
+  },
+  "ata_smart_data": {
+    "offline_data_collection": {
+      "status": {
+        "value": 130,
+        "string": "was completed without error",
+        "passed": true
+      },
+      "completion_seconds": 101
+    },
+    "self_test": {
+      "status": {
+        "value": 241,
+        "string": "in progress, 10% remaining",
+        "remaining_percent": 10
+      },
+      "polling_minutes": {
+        "short": 2,
+        "extended": 1479
+      }
+    },
+    "capabilities": {
+      "values": [
+        91,
+        3
+      ],
+      "exec_offline_immediate_supported": true,
+      "offline_is_aborted_upon_new_cmd": false,
+      "offline_surface_scan_supported": true,
+      "self_tests_supported": true,
+      "conveyance_self_test_supported": false,
+      "selective_self_test_supported": true,
+      "attribute_autosave_enabled": true,
+      "error_logging_supported": true,
+      "gp_logging_supported": true
+    }
+  },
+  "ata_sct_capabilities": {
+    "value": 61,
+    "error_recovery_control_supported": true,
+    "feature_control_supported": true,
+    "data_table_supported": true
+  },
+  "ata_smart_attributes": {
+    "revision": 16,
+    "table": [
+      {
+        "id": 1,
+        "name": "Raw_Read_Error_Rate",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 11,
+          "string": "PO-R-- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 2,
+        "name": "Throughput_Performance",
+        "value": 135,
+        "worst": 135,
+        "thresh": 54,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 108,
+          "string": "108"
+        }
+      },
+      {
+        "id": 3,
+        "name": "Spin_Up_Time",
+        "value": 81,
+        "worst": 81,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 7,
+          "string": "POS--- ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 30089675132,
+          "string": "380 (Average 380)"
+        }
+      },
+      {
+        "id": 4,
+        "name": "Start_Stop_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 9,
+          "string": "9"
+        }
+      },
+      {
+        "id": 5,
+        "name": "Reallocated_Sector_Ct",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 51,
+          "string": "PO--CK ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 7,
+        "name": "Seek_Error_Rate",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 8,
+        "name": "Seek_Time_Performance",
+        "value": 133,
+        "worst": 133,
+        "thresh": 20,
+        "when_failed": "",
+        "flags": {
+          "value": 4,
+          "string": "--S--- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": true,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 18,
+          "string": "18"
+        }
+      },
+      {
+        "id": 9,
+        "name": "Power_On_Hours",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 1730,
+          "string": "1730"
+        }
+      },
+      {
+        "id": 10,
+        "name": "Spin_Retry_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 1,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 12,
+        "name": "Power_Cycle_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 9,
+          "string": "9"
+        }
+      },
+      {
+        "id": 22,
+        "name": "Unknown_Attribute",
+        "value": 100,
+        "worst": 100,
+        "thresh": 25,
+        "when_failed": "",
+        "flags": {
+          "value": 35,
+          "string": "PO---K ",
+          "prefailure": true,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 100,
+          "string": "100"
+        }
+      },
+      {
+        "id": 192,
+        "name": "Power-Off_Retract_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 329,
+          "string": "329"
+        }
+      },
+      {
+        "id": 193,
+        "name": "Load_Cycle_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 18,
+          "string": "-O--C- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 329,
+          "string": "329"
+        }
+      },
+      {
+        "id": 194,
+        "name": "Temperature_Celsius",
+        "value": 51,
+        "worst": 51,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 2,
+          "string": "-O---- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 163210330144,
+          "string": "32 (Min/Max 24/38)"
+        }
+      },
+      {
+        "id": 196,
+        "name": "Reallocated_Event_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 50,
+          "string": "-O--CK ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": true,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 197,
+        "name": "Current_Pending_Sector",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 34,
+          "string": "-O---K ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": false,
+          "event_count": false,
+          "auto_keep": true
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 198,
+        "name": "Offline_Uncorrectable",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 8,
+          "string": "---R-- ",
+          "prefailure": false,
+          "updated_online": false,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      },
+      {
+        "id": 199,
+        "name": "UDMA_CRC_Error_Count",
+        "value": 100,
+        "worst": 100,
+        "thresh": 0,
+        "when_failed": "",
+        "flags": {
+          "value": 10,
+          "string": "-O-R-- ",
+          "prefailure": false,
+          "updated_online": true,
+          "performance": false,
+          "error_rate": true,
+          "event_count": false,
+          "auto_keep": false
+        },
+        "raw": {
+          "value": 0,
+          "string": "0"
+        }
+      }
+    ]
+  },
+  "power_on_time": {
+    "hours": 1730
+  },
+  "power_cycle_count": 9,
+  "temperature": {
+    "current": 32
+  },
+  "ata_smart_error_log": {
+    "summary": {
+      "revision": 1,
+      "count": 0
+    }
+  },
+  "ata_smart_self_test_log": {
+    "standard": {
+      "revision": 1,
+      "table": [
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1708
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1684
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1661
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1636
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1624
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1541
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1517
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1493
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1469
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1445
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1439
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1373
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1349
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1325
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1301
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1277
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1253
+        },
+        {
+          "type": {
+            "value": 2,
+            "string": "Extended offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1252
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1205
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1181
+        },
+        {
+          "type": {
+            "value": 1,
+            "string": "Short offline"
+          },
+          "status": {
+            "value": 0,
+            "string": "Completed without error",
+            "passed": true
+          },
+          "lifetime_hours": 1157
+        }
+      ],
+      "count": 21,
+      "error_count_total": 0,
+      "error_count_outdated": 0
+    }
+  },
+  "ata_smart_selective_self_test_log": {
+    "revision": 1,
+    "table": [
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      },
+      {
+        "lba_min": 0,
+        "lba_max": 0,
+        "status": {
+          "value": 241,
+          "string": "Not_testing"
+        }
+      }
+    ],
+    "flags": {
+      "value": 0,
+      "remainder_scan_enabled": false
+    },
+    "power_up_scan_resume_minutes": 0
+  }
+}

webapp/backend/pkg/version/version.go

@@ -0,0 +1,5 @@
+package version
+
+// VERSION is the app-global version string, which will be replaced with a
+// new value during packaging
+const VERSION = "1.0.0"

webapp/backend/pkg/web/cron.go

@@ -0,0 +1,8 @@
+package web
+
+// the following cronjobs need to be defined here:
+// - get storage information for all approved disks
+// - run short test against approved disks
+// - run long test against approved disks
+// - get S.M.A.R.T. metrics from approved disks
+// - clean up / resolution for time-series data in sqlite database.

webapp/backend/pkg/web/disk.go

@@ -0,0 +1,77 @@
+package web
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/jaypipes/ghw"
+)
+
+func RetrieveStorageDevices() ([]db.Device, error) {
+
+	block, err := ghw.Block()
+	if err != nil {
+		fmt.Printf("Error getting block storage info: %v", err)
+		return nil, err
+	}
+
+	approvedDisks := []db.Device{}
+	for _, disk := range block.Disks {
+		//TODO: always allow if in approved list
+		fmt.Printf(" %v\n", disk)
+
+		// ignore optical drives and floppy disks
+		if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
+			fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
+			continue
+		}
+
+		// ignore removable disks
+		if disk.IsRemovable {
+			fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
+			continue
+		}
+
+		// ignore virtual disks & mobile phone storage devices
+		if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
+			fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
+			continue
+		}
+
+		// ignore NVMe devices (not currently supported) TBA
+		if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME {
+			fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String())
+			continue
+		}
+
+		// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
+		if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
+			fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
+			continue
+		}
+
+		//TODO: remove if in excluded list
+
+		diskModel := db.Device{
+			WWN:           disk.WWN,
+			Manufacturer:  disk.Vendor,
+			ModelName:     disk.Model,
+			InterfaceType: disk.StorageController.String(),
+			//InterfaceSpeed: string
+			SerialNumber: disk.SerialNumber,
+			Capacity:     int64(disk.SizeBytes),
+			//Firmware       string
+			//RotationSpeed  int
+
+			DeviceName: disk.Name,
+		}
+		if len(diskModel.WWN) == 0 {
+			//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
+			//diskname as identifier if WWN is not present
+			diskModel.WWN = disk.Name
+		}
+
+		approvedDisks = append(approvedDisks, diskModel)
+	}
+
+	return approvedDisks, nil
+}

webapp/backend/pkg/web/server.go

@@ -0,0 +1,167 @@
+package web
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/gin-gonic/gin"
+	"github.com/jinzhu/gorm"
+	log "github.com/sirupsen/logrus"
+	"net/http"
+)
+
+type AppEngine struct {
+	Config config.Interface
+}
+
+func (ae *AppEngine) Start() error {
+	r := gin.Default()
+
+	r.Use(database.DatabaseHandler(ae.Config.GetString("web.database.location")))
+
+	api := r.Group("/api")
+	{
+		api.GET("/health", func(c *gin.Context) {
+			c.JSON(http.StatusOK, gin.H{
+				"success": true,
+			})
+		})
+
+		//TODO: notifications
+		api.GET("/devices", GetDevicesHandler)
+		api.GET("/summary", GetDevicesSummary)
+		api.POST("/device/:wwn/smart", UploadDeviceSmartData)
+		api.POST("/device/:wwn/selftest", UploadDeviceSelfTestData)
+
+		api.GET("/device/:wwn/details", GetDeviceDetails)
+	}
+
+	//Static request routing
+	r.StaticFS("/web", http.Dir(ae.Config.GetString("web.src.frontend.path")))
+
+	//redirect base url to /web
+	r.GET("/", func(c *gin.Context) {
+		c.Redirect(http.StatusFound, "/web")
+	})
+
+	//catch-all, serve index page.
+	r.NoRoute(func(c *gin.Context) {
+		c.File(fmt.Sprintf("%s/index.html", ae.Config.GetString("web.src.frontend.path")))
+	})
+
+	return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port"))) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080")
+}
+
+// Get all active disks for processing by collectors
+func GetDevicesHandler(c *gin.Context) {
+	storageDevices, err := RetrieveStorageDevices()
+
+	db := c.MustGet("DB").(*gorm.DB)
+	for _, dev := range storageDevices {
+		//insert devices into DB if not already there.
+		db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev)
+	}
+
+	if err != nil {
+		c.JSON(http.StatusOK, gin.H{
+			"success": false,
+		})
+	} else {
+		c.JSON(http.StatusOK, gin.H{
+			"success": true,
+			"data":    storageDevices,
+		})
+	}
+}
+
+func UploadDeviceSmartData(c *gin.Context) {
+	db := c.MustGet("DB").(*gorm.DB)
+
+	var collectorSmartData collector.SmartInfo
+	err := c.BindJSON(&collectorSmartData)
+	if err != nil {
+		//TODO: cannot parse smart data
+		log.Error("Cannot parse SMART data")
+		c.JSON(http.StatusOK, gin.H{"success": false})
+
+	}
+
+	//update the device information if necessary
+	var device dbModels.Device
+	db.Where("wwn = ?", c.Param("wwn")).First(&device)
+	device.UpdateFromCollectorSmartInfo(collectorSmartData)
+	db.Model(&device).Updates(device)
+
+	// insert smart info
+	deviceSmartData := dbModels.Smart{}
+	err = deviceSmartData.FromCollectorSmartInfo(c.Param("wwn"), collectorSmartData)
+	if err != nil {
+		c.JSON(http.StatusOK, gin.H{"success": false})
+		return
+	}
+	db.Create(&deviceSmartData)
+
+	c.JSON(http.StatusOK, gin.H{"success": true})
+}
+
+func UploadDeviceSelfTestData(c *gin.Context) {
+
+}
+
+func GetDeviceDetails(c *gin.Context) {
+	db := c.MustGet("DB").(*gorm.DB)
+	device := dbModels.Device{}
+
+	db.Debug().
+		Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
+			return db.Order("smarts.created_at DESC").Limit(40)
+		}).
+		Preload("SmartResults.SmartAttributes").
+		Where("wwn = ?", c.Param("wwn")).
+		First(&device)
+
+	device.SquashHistory()
+	device.ApplyMetadataRules()
+
+	c.JSON(http.StatusOK, gin.H{"success": true, "data": device, "lookup": metadata.AtaSmartAttributes})
+
+}
+
+func GetDevicesSummary(c *gin.Context) {
+	db := c.MustGet("DB").(*gorm.DB)
+
+	devices := []dbModels.Device{}
+
+	//OLD: cant seem to figure out how to get the latest SmartResults for each Device, so instead
+	// we're going to assume that results were retrieved at the same time, so we'll just get the last x number of results
+	//var devicesCount int
+	//db.Table("devices").Count(&devicesCount)
+
+	//We need the last x (for now all) Smart objects for each Device, so that we can graph Temperature
+	//We also need the last
+	db.Debug().
+		Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
+			return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
+		}).
+		//Preload("SmartResults").
+		// Preload("SmartResults.SmartAttributes").
+		Find(&devices)
+
+	//for _, dev := range devices {
+	//	log.Printf("===== device: %s\n", dev.WWN)
+	//	log.Print(len(dev.SmartResults))
+	//}
+	//a, _ := json.Marshal(devices) //get json byte array
+	//n := len(a)   //Find the length of the byte array
+	//s := string(a[:n]) //convert to string
+	//log.Print(s) //write to response
+
+	c.JSON(http.StatusOK, gin.H{
+		"success": true,
+		"data":    devices,
+	})
+	//c.Data(http.StatusOK, "application/json", a)
+}