WIP settings system.

- updated dbdiagrams schema
- [BREAKING] force failure if `notify.filter_attributes` or `notify.level` is set
- added Settings table (and default values during migration)
- Added Save Settings and Get Settings functions.
- Added web API endpoints for getting and saving settings.
- Deprecated old Notify* constants. Created new MetricsStatus* and MetricsNotifyLevel constants.

docs/dbdiagram.io.txt

@@ -1,62 +1,88 @@
 
 // SQLite Table(s)
-Table device {
-  created_at timestamp
 
-  wwn varchar [pk]
+Table Device  {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time
+	UpdatedAt time
+	DeletedAt time
 
-  //user provided
+	WWN string
-  label varchar
-  host_id varchar
 
-  // smartctl provided
+	DeviceName     string
-  device_name varchar
+	DeviceUUID     string
-  manufacturer varchar
+	DeviceSerialID string
-  model_name varchar
+	DeviceLabel    string
-  interface_type varchar
-  interface_speed varchar
-  serial_number varchar
-  firmware varchar
-  rotational_speed varchar
-  capacity varchar
-  form_factor varchar
-  smart_support varchar
-  device_protocol varchar
-  device_type varchar
 
+	Manufacturer   string
+	ModelName      string
+	InterfaceType  string
+	InterfaceSpeed string
+	SerialNumber   string
+	Firmware       string
+	RotationSpeed  int
+	Capacity       int64
+	FormFactor     string
+	SmartSupport   bool
+	DeviceProtocol string//protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string//device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string
+	HostId string
+
+	// Data set by Scrutiny
+	DeviceStatus enum
+}
+
+Table Setting {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+
+	SettingKeyName  string
+    SettingKeyDescription  string
+	SettingDataType string
+
+	SettingValueNumeric int64
+	SettingValueString  string
 }
 
 
 // InfluxDB Tables
-Table device_temperature {
+Table SmartTemperature {
-  //timestamp
+    Date time
-  created_at timestamp
+	DeviceWWN      string //(tag)
-
+	Temp int64
-  //tags (indexed & queryable)
-  device_wwn varchar [pk]
-
-  //fields
-  temp bigint
 }
 
 
-Table smart_ata_results {
+Table Smart {
-  //timestamp
+    Date           time
-  created_at timestamp
+	DeviceWWN      string //(tag)
+	DeviceProtocol string
 
-  //tags (indexed & queryable)
+	//Metrics (fields)
-  device_wwn varchar [pk]
+	Temp            int64
-  smart_status varchar
+	PowerOnHours    int64
-  scrutiny_status varchar
+	PowerCycleCount int64
 
+	//Smart Status
+	Status enum
 
-
+	//SMART Attributes (fields)
-  //fields
+	Attr_ID_AttributeId int
-  temp bigint
+	Attr_ID_Value       int64
-  power_on_hours bigint
+	Attr_ID_Threshold   int64
-  power_cycle_count bigint
+	Attr_ID_Worst       int64
-
+    Attr_ID_RawValue    int64
+	Attr_ID_RawString   string
+	Attr_ID_WhenFailed  string
+	//Generated data
+	Attr_ID_TransformedValue int64
+	Attr_ID_Status           enum
+	Attr_ID_StatusReason     string
+	Attr_ID_FailureRate      float64
 
  }
 
-Ref: device.wwn < smart_ata_results.device_wwn
+Ref: Device.WWN < Smart.DeviceWWN
+Ref: Device.WWN < SmartTemperature.DeviceWWN

webapp/backend/pkg/config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"github.com/analogj/go-util/utils"
-	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
 	"github.com/spf13/viper"
 	"log"
@@ -39,8 +38,6 @@ func (c *configuration) Init() error {
 	c.SetDefault("log.file", "")
 
 	c.SetDefault("notify.urls", []string{})
-	c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll)
-	c.SetDefault("notify.level", pkg.NotifyLevelFail)
 
 	c.SetDefault("web.influxdb.scheme", "http")
 	c.SetDefault("web.influxdb.host", "localhost")
@@ -55,17 +52,6 @@ func (c *configuration) Init() error {
 	//c.SetDefault("disks.include", []string{})
 	//c.SetDefault("disks.exclude", []string{})
 
-	//c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh")
-	//c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh")
-	//c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh")
-
-	//c.SetDefault("collect.metric.enable", true)
-	//c.SetDefault("collect.metric.command", "-a -o on -S on")
-	//c.SetDefault("collect.long.enable", true)
-	//c.SetDefault("collect.long.command", "-a -o on -S on")
-	//c.SetDefault("collect.short.enable", true)
-	//c.SetDefault("collect.short.command", "-a -o on -S on")
-
 	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
 	c.SetConfigType("yaml")
 	//c.SetConfigName("drawbridge")
@@ -77,7 +63,7 @@ func (c *configuration) Init() error {
 	c.AutomaticEnv()
 
 	//CLI options will be added via the `Set()` function
-	return nil
+	return c.ValidateConfig()
 }
 
 func (c *configuration) ReadConfig(configFilePath string) error {
@@ -120,24 +106,18 @@ func (c *configuration) ReadConfig(configFilePath string) error {
 // This function ensures that the merged config works correctly.
 func (c *configuration) ValidateConfig() error {
 
-	////deserialize Questions
+	//the following keys are deprecated, and no longer supported
-	//questionsMap := map[string]Question{}
+	/*
-	//err := c.UnmarshalKey("questions", &questionsMap)
+		- notify.filter_attributes (replaced by metrics.status.filter_attributes SETTING)
-	//
+		- notify.level (replaced by metrics.notify.level and metrics.status.threshold SETTING)
-	//if err != nil {
+	*/
-	//	log.Printf("questions could not be deserialized correctly. %v", err)
+	//TODO add docs and upgrade doc.
-	//	return err
+	if c.IsSet("notify.filter_attributes") {
-	//}
+		return errors.ConfigValidationError("`notify.filter_attributes` configuration option is deprecated. Replaced by option in Dashboard Settings page")
-	//
+	}
-	//for _, v := range questionsMap {
+	if c.IsSet("notify.level") {
-	//
+		return errors.ConfigValidationError("`notify.level` configuration option is deprecated. Replaced by option in Dashboard Settings page")
-	//	typeContent, ok := v.Schema["type"].(string)
+	}
-	//	if !ok || len(typeContent) == 0 {
-	//		return errors.QuestionSyntaxError("`type` is required for questions")
-	//	}
-	//}
-	//
-	//
 
 	return nil
 }

webapp/backend/pkg/constants.go

@@ -4,17 +4,11 @@ const DeviceProtocolAta = "ATA"
 const DeviceProtocolScsi = "SCSI"
 const DeviceProtocolNvme = "NVMe"
 
-const NotifyFilterAttributesAll = "all"
-const NotifyFilterAttributesCritical = "critical"
-
-const NotifyLevelFail = "fail"
-const NotifyLevelFailScrutiny = "fail_scrutiny"
-const NotifyLevelFailSmart = "fail_smart"
-
 //go:generate stringer -type=AttributeStatus
+// AttributeStatus bitwise flag, 1,2,4,8,16,32,etc
 type AttributeStatus uint8
+
 const (
-	// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
 	AttributeStatusPassed          AttributeStatus = 0
 	AttributeStatusFailedSmart     AttributeStatus = 1
 	AttributeStatusWarningScrutiny AttributeStatus = 2
@@ -30,9 +24,10 @@ func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^
 func AttributeStatusHas(b, flag AttributeStatus) bool               { return b&flag != 0 }
 
 //go:generate stringer -type=DeviceStatus
+// DeviceStatus bitwise flag, 1,2,4,8,16,32,etc
 type DeviceStatus uint8
+
 const (
-	// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
 	DeviceStatusPassed         DeviceStatus = 0
 	DeviceStatusFailedSmart    DeviceStatus = 1
 	DeviceStatusFailedScrutiny DeviceStatus = 2
@@ -42,3 +37,44 @@ func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus    { return b | flag }
 func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus  { return b &^ flag }
 func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
 func DeviceStatusHas(b, flag DeviceStatus) bool            { return b&flag != 0 }
+
+// Metrics Specific Filtering & Threshold Constants
+type MetricsNotifyLevel int64
+
+const (
+	MetricsNotifyLevelWarn MetricsNotifyLevel = 1
+	MetricsNotifyLevelFail MetricsNotifyLevel = 2
+)
+
+type MetricsStatusFilterAttributes int64
+
+const (
+	MetricsStatusFilterAttributesAll      MetricsStatusFilterAttributes = 0
+	MetricsStatusFilterAttributesCritical MetricsStatusFilterAttributes = 1
+)
+
+// MetricsStatusThreshold bitwise flag, 1,2,4,8,16,32,etc
+type MetricsStatusThreshold int64
+
+const (
+	MetricsStatusThresholdSmart    MetricsStatusThreshold = 1
+	MetricsStatusThresholdScrutiny MetricsStatusThreshold = 2
+
+	//shortcut
+	MetricsStatusThresholdBoth MetricsStatusThreshold = 3
+)
+
+// Deprecated
+const NotifyFilterAttributesAll = "all"
+
+// Deprecated
+const NotifyFilterAttributesCritical = "critical"
+
+// Deprecated
+const NotifyLevelFail = "fail"
+
+// Deprecated
+const NotifyLevelFailScrutiny = "fail_scrutiny"
+
+// Deprecated
+const NotifyLevelFailSmart = "fail_smart"

webapp/backend/pkg/database/interface.go

@@ -28,4 +28,7 @@ type DeviceRepo interface {
 
 	GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
 	GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
+
+	GetSettings(ctx context.Context) (*models.Settings, error)
+	SaveSettings(ctx context.Context, settings models.Settings) error
 }

webapp/backend/pkg/database/migrations/m20220716214900/setting.go

@@ -9,6 +9,7 @@ type Setting struct {
 	gorm.Model
 
 	SettingKeyName        string `json:"setting_key_name"`
+	SettingKeyDescription string `json:"setting_key_description"`
 	SettingDataType       string `json:"setting_data_type"`
 
 	SettingValueNumeric int64  `json:"setting_value_numeric"`

webapp/backend/pkg/database/scrutiny_repository_migrations.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100"
@@ -281,7 +282,33 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 			Migrate: func(tx *gorm.DB) error {
 
 				// adding the settings table.
-				return tx.AutoMigrate(m20220716214900.Setting{})
+				err := tx.AutoMigrate(m20220716214900.Setting{})
+				if err != nil {
+					return err
+				}
+				//add defaults.
+
+				var defaultSettings = []m20220716214900.Setting{
+					{
+						SettingKeyName:        "metrics.notify.level",
+						SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)",
+						SettingDataType:       "numeric",
+						SettingValueNumeric:   int64(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn'
+					},
+					{
+						SettingKeyName:        "metrics.status.filter_attributes",
+						SettingKeyDescription: "Determines which attributes should impact device status",
+						SettingDataType:       "numeric",
+						SettingValueNumeric:   int64(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or  'critical'
+					},
+					{
+						SettingKeyName:        "metrics.status.threshold",
+						SettingKeyDescription: "Determines which threshold should impact device status",
+						SettingDataType:       "string",
+						SettingValueNumeric:   int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both'
+					},
+				}
+				return tx.Create(&defaultSettings).Error
 			},
 		},
 	})

webapp/backend/pkg/database/scrutiny_repository_settings.go

@@ -0,0 +1,33 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+)
+
+func (sr *scrutinyRepository) GetSettings(ctx context.Context) (*models.Settings, error) {
+	settingsEntries := []models.SettingEntry{}
+	if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
+		return nil, fmt.Errorf("Could not get settings from DB: %v", err)
+	}
+
+	settings := models.Settings{}
+	settings.PopulateFromSettingEntries(settingsEntries)
+
+	return &settings, nil
+}
+func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error {
+
+	//get current settings
+	settingsEntries := []models.SettingEntry{}
+	if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
+		return fmt.Errorf("Could not get settings from DB: %v", err)
+	}
+
+	// override with values from settings object
+	settingsEntries = settings.UpdateSettingEntries(settingsEntries)
+
+	// store in database.
+	return sr.gormClient.Updates(&settingsEntries).Error
+}

webapp/backend/pkg/models/setting.go

@@ -1,16 +0,0 @@
-package models
-
-import (
-	"gorm.io/gorm"
-)
-
-type Setting struct {
-	//GORM attributes, see: http://gorm.io/docs/conventions.html
-	gorm.Model
-
-	SettingKeyName  string `json:"setting_key_name"`
-	SettingDataType string `json:"setting_data_type"`
-
-	SettingValueNumeric int64  `json:"setting_value_numeric"`
-	SettingValueString  string `json:"setting_value_string"`
-}

webapp/backend/pkg/models/setting_entry.go

@@ -0,0 +1,22 @@
+package models
+
+import (
+	"gorm.io/gorm"
+)
+
+// SettingEntry matches a setting row in the database
+type SettingEntry struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	gorm.Model
+
+	SettingKeyName        string `json:"setting_key_name" gorm:"unique;not null"`
+	SettingKeyDescription string `json:"setting_key_description"`
+	SettingDataType       string `json:"setting_data_type"`
+
+	SettingValueNumeric int64  `json:"setting_value_numeric"`
+	SettingValueString  string `json:"setting_value_string"`
+}
+
+func (s SettingEntry) TableName() string {
+	return "settings"
+}

webapp/backend/pkg/models/settings.go

@@ -0,0 +1,35 @@
+package models
+
+import "github.com/analogj/scrutiny/webapp/backend/pkg"
+
+// Settings is made up of parsed SettingEntry objects retrieved from the database
+type Settings struct {
+	MetricsNotifyLevel            pkg.MetricsNotifyLevel            `json:"metrics_notify_level"`
+	MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics_status_filter_attributes"`
+	MetricsStatusThreshold        pkg.MetricsStatusThreshold        `json:"metrics_status_threshold"`
+}
+
+func (s *Settings) PopulateFromSettingEntries(entries []SettingEntry) {
+	for _, entry := range entries {
+		if entry.SettingKeyName == "metrics.notify.level" {
+			s.MetricsNotifyLevel = pkg.MetricsNotifyLevel(entry.SettingValueNumeric)
+		} else if entry.SettingKeyName == "metrics.status.filter_attributes" {
+			s.MetricsStatusFilterAttributes = pkg.MetricsStatusFilterAttributes(entry.SettingValueNumeric)
+		} else if entry.SettingKeyName == "metrics.status.threshold" {
+			s.MetricsStatusThreshold = pkg.MetricsStatusThreshold(entry.SettingValueNumeric)
+		}
+	}
+}
+
+func (s *Settings) UpdateSettingEntries(entries []SettingEntry) []SettingEntry {
+	for _, entry := range entries {
+		if entry.SettingKeyName == "metrics.notify.level" {
+			entry.SettingValueNumeric = int64(s.MetricsNotifyLevel)
+		} else if entry.SettingKeyName == "metrics.status.filter_attributes" {
+			entry.SettingValueNumeric = int64(s.MetricsStatusFilterAttributes)
+		} else if entry.SettingKeyName == "metrics.status.threshold" {
+			entry.SettingValueNumeric = int64(s.MetricsStatusThreshold)
+		}
+	}
+	return entries
+}

webapp/backend/pkg/web/handler/get_settings.go

@@ -0,0 +1,25 @@
+package handler
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+)
+
+func GetSettings(c *gin.Context) {
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	settings, err := deviceRepo.GetSettings(c)
+	if err != nil {
+		logger.Errorln("An error occurred while retrieving settings", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"success":  true,
+		"settings": settings,
+	})
+}

webapp/backend/pkg/web/handler/save_settings.go

@@ -0,0 +1,33 @@
+package handler
+
+import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/database"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+)
+
+func SaveSettings(c *gin.Context) {
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
+
+	var settings models.Settings
+	err := c.BindJSON(&settings)
+	if err != nil {
+		logger.Errorln("Cannot parse updated settings", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	err = deviceRepo.SaveSettings(c, settings)
+	if err != nil {
+		logger.Errorln("An error occurred while saving settings", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"success": true,
+	})
+}

webapp/backend/pkg/web/server.go

@@ -50,6 +50,8 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
 			api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details
 			api.DELETE("/device/:wwn", handler.DeleteDevice)          //used by UI to delete device
 
+			api.GET("/settings", handler.GetSettings)   //used to get settings
+			api.POST("/settings", handler.SaveSettings) //used to save settings
 		}
 	}