gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

working settings update.

Browse Source 
Settings are loaded from the DB and added to the AppConfig during startup.
When updating settings, they are stored in AppConfig, and written do  the database.
This commit is contained in:
Jason Kulatunga
2022-07-19 23:12:23 -07:00
parent 99af2b8b16
commit 29bc79996b
14 changed files with 144 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/backend/pkg/config/config.go
+4
View File
@@ -66,6 +66,10 @@ func (c *configuration) Init() error {
return c.ValidateConfig()
}
func (c *configuration) SubKeys(key string) []string {
return c.Sub(key).AllKeys()
}
func (c *configuration) ReadConfig(configFilePath string) error {
//make sure that we specify that this is the correct config path (for eventual WriteConfig() calls)
c.SetConfigFile(configFilePath)
webapp/backend/pkg/config/interface.go
+4
View File
@@ -12,12 +12,16 @@ type Interface interface {
WriteConfig() error
Set(key string, value interface{})
SetDefault(key string, value interface{})
MergeConfigMap(cfg map[string]interface{}) error
AllSettings() map[string]interface{}
AllKeys() []string
SubKeys(key string) []string
IsSet(key string) bool
Get(key string) interface{}
GetBool(key string) bool
GetInt(key string) int
GetInt64(key string) int64
GetString(key string) string
GetStringSlice(key string) []string
UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
webapp/backend/pkg/constants.go
-9
View File
@@ -67,14 +67,5 @@ const (
// Deprecated
const NotifyFilterAttributesAll = "all"
// Deprecated
const NotifyFilterAttributesCritical = "critical"
// Deprecated
const NotifyLevelFail = "fail"
// Deprecated
const NotifyLevelFailScrutiny = "fail_scrutiny"
// Deprecated
const NotifyLevelFailSmart = "fail_smart"
webapp/backend/pkg/database/interface.go
+1 -4
View File
@@ -11,9 +11,6 @@ import (
type DeviceRepo interface {
Close() error
//GetSettings()
//SaveSetting()
RegisterDevice(ctx context.Context, dev models.Device) error
GetDevices(ctx context.Context) ([]models.Device, error)
UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
@@ -29,6 +26,6 @@ type DeviceRepo interface {
GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
GetSettings(ctx context.Context) (*models.Settings, error)
LoadSettings(ctx context.Context) (*models.Settings, error)
SaveSettings(ctx context.Context, settings models.Settings) error
}
webapp/backend/pkg/database/scrutiny_repository_migrations.go
+1 -1
View File
@@ -304,7 +304,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
{
SettingKeyName: "metrics.status.threshold",
SettingKeyDescription: "Determines which threshold should impact device status",
SettingDataType: "string",
SettingDataType: "numeric",
SettingValueNumeric: int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both'
},
}
webapp/backend/pkg/database/scrutiny_repository_settings.go
+56 -7
View File
@@ -4,30 +4,79 @@ import (
"context"
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/mitchellh/mapstructure"
)
func (sr *scrutinyRepository) GetSettings(ctx context.Context) (*models.Settings, error) {
const DBSETTING_SUBKEY = "dbsetting"
// LoadSettings will retrieve settings from the database, store them in the AppConfig object, and return a Settings struct
func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Settings, error) {
settingsEntries := []models.SettingEntry{}
if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
return nil, fmt.Errorf("Could not get settings from DB: %v", err)
}
settings := models.Settings{}
settings.PopulateFromSettingEntries(settingsEntries)
// store retrieved settings in the AppConfig obj
for _, settingsEntry := range settingsEntries {
configKey := fmt.Sprintf("%s.%s", DBSETTING_SUBKEY, settingsEntry.SettingKeyName)
if settingsEntry.SettingDataType == "numeric" {
sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueNumeric)
} else if settingsEntry.SettingDataType == "string" {
sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueString)
}
}
// unmarshal the dbsetting object data to a settings object.
var settings models.Settings
err := sr.appConfig.UnmarshalKey(DBSETTING_SUBKEY, &settings)
if err != nil {
return nil, err
}
return &settings, nil
}
// testing
// curl -d '{"metrics": { "notify": { "level": 5 }, "status": { "filter_attributes": 5, "threshold": 5 } }}' -H "Content-Type: application/json" -X POST http://localhost:9090/api/settings
// SaveSettings will update settings in AppConfig object, then save the settings to the database.
func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error {
//get current settings
//save the entries to the appconfig
settingsMap := &map[string]interface{}{}
err := mapstructure.Decode(settings, &settingsMap)
if err != nil {
return err
}
settingsWrapperMap := map[string]interface{}{}
settingsWrapperMap[DBSETTING_SUBKEY] = *settingsMap
err = sr.appConfig.MergeConfigMap(settingsWrapperMap)
if err != nil {
return err
}
//retrieve current settings from the database
settingsEntries := []models.SettingEntry{}
if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
return fmt.Errorf("Could not get settings from DB: %v", err)
}
// override with values from settings object
settingsEntries = settings.UpdateSettingEntries(settingsEntries)
//update settingsEntries
for ndx, settingsEntry := range settingsEntries {
configKey := fmt.Sprintf("%s.%s", DBSETTING_SUBKEY, settingsEntry.SettingKeyName)
if settingsEntry.SettingDataType == "numeric" {
settingsEntries[ndx].SettingValueNumeric = sr.appConfig.GetInt(configKey)
} else if settingsEntry.SettingDataType == "string" {
settingsEntries[ndx].SettingValueString = sr.appConfig.GetString(configKey)
}
// store in database.
return sr.gormClient.Updates(&settingsEntries).Error
//TODO: this should be `sr.gormClient.Updates(&settingsEntries).Error`
err := sr.gormClient.Model(&models.SettingEntry{}).Where([]uint{settingsEntry.ID}).Select("setting_value_numeric", "setting_value_string").Updates(settingsEntries[ndx]).Error
if err != nil {
return err
}
}
return nil
}
webapp/backend/pkg/models/setting_entry.go
+1 -1
View File
@@ -13,7 +13,7 @@ type SettingEntry struct {
SettingKeyDescription string `json:"setting_key_description"`
SettingDataType string `json:"setting_data_type"`
SettingValueNumeric int64 `json:"setting_value_numeric"`
SettingValueNumeric int `json:"setting_value_numeric"`
SettingValueString string `json:"setting_value_string"`
}
webapp/backend/pkg/models/settings.go
+15 -30
View File
@@ -1,35 +1,20 @@
package models
import "github.com/analogj/scrutiny/webapp/backend/pkg"
// Settings is made up of parsed SettingEntry objects retrieved from the database
//type Settings struct {
// MetricsNotifyLevel pkg.MetricsNotifyLevel `json:"metrics.notify.level" mapstructure:"metrics.notify.level"`
// MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics.status.filter_attributes" mapstructure:"metrics.status.filter_attributes"`
// MetricsStatusThreshold pkg.MetricsStatusThreshold `json:"metrics.status.threshold" mapstructure:"metrics.status.threshold"`
//}
type Settings struct {
MetricsNotifyLevel pkg.MetricsNotifyLevel `json:"metrics_notify_level"`
MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics_status_filter_attributes"`
MetricsStatusThreshold pkg.MetricsStatusThreshold `json:"metrics_status_threshold"`
}
func (s *Settings) PopulateFromSettingEntries(entries []SettingEntry) {
for _, entry := range entries {
if entry.SettingKeyName == "metrics.notify.level" {
s.MetricsNotifyLevel = pkg.MetricsNotifyLevel(entry.SettingValueNumeric)
} else if entry.SettingKeyName == "metrics.status.filter_attributes" {
s.MetricsStatusFilterAttributes = pkg.MetricsStatusFilterAttributes(entry.SettingValueNumeric)
} else if entry.SettingKeyName == "metrics.status.threshold" {
s.MetricsStatusThreshold = pkg.MetricsStatusThreshold(entry.SettingValueNumeric)
}
}
}
func (s *Settings) UpdateSettingEntries(entries []SettingEntry) []SettingEntry {
for _, entry := range entries {
if entry.SettingKeyName == "metrics.notify.level" {
entry.SettingValueNumeric = int64(s.MetricsNotifyLevel)
} else if entry.SettingKeyName == "metrics.status.filter_attributes" {
entry.SettingValueNumeric = int64(s.MetricsStatusFilterAttributes)
} else if entry.SettingKeyName == "metrics.status.threshold" {
entry.SettingValueNumeric = int64(s.MetricsStatusThreshold)
}
}
return entries
Metrics struct {
Notify struct {
Level int `json:"level" mapstructure:"level"`
} `json:"notify" mapstructure:"notify"`
Status struct {
FilterAttributes int `json:"filter_attributes" mapstructure:"filter_attributes"`
Threshold int `json:"threshold" mapstructure:"threshold"`
} `json:"status" mapstructure:"status"`
} `json:"metrics" mapstructure:"metrics"`
}
webapp/backend/pkg/notify/notify.go
+9 -7
View File
@@ -29,20 +29,22 @@ const NotifyFailureTypeSmartFailure = "SmartFailure"
const NotifyFailureTypeScrutinyFailure = "ScrutinyFailure"
// ShouldNotify check if the error Message should be filtered (level mismatch or filtered_attributes)
func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLevel string, notifyFilterAttributes string) bool {
func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes) bool {
// 1. check if the device is healthy
if device.DeviceStatus == pkg.DeviceStatusPassed {
return false
}
//TODO: cannot check for warning notifyLevel yet.
// setup constants for comparison
var requiredDeviceStatus pkg.DeviceStatus
var requiredAttrStatus pkg.AttributeStatus
if notifyLevel == pkg.NotifyLevelFail {
if statusThreshold == pkg.MetricsStatusThresholdBoth {
// either scrutiny or smart failures should trigger an email
requiredDeviceStatus = pkg.DeviceStatusSet(pkg.DeviceStatusFailedSmart, pkg.DeviceStatusFailedScrutiny)
requiredAttrStatus = pkg.AttributeStatusSet(pkg.AttributeStatusFailedSmart, pkg.AttributeStatusFailedScrutiny)
} else if notifyLevel == pkg.NotifyLevelFailSmart {
} else if statusThreshold == pkg.MetricsStatusThresholdSmart {
//only smart failures
requiredDeviceStatus = pkg.DeviceStatusFailedSmart
requiredAttrStatus = pkg.AttributeStatusFailedSmart
@@ -53,9 +55,9 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev
// 2. check if the attributes that are failing should be filtered (non-critical)
// 3. for any unfiltered attribute, store the failure reason (Smart or Scrutiny)
if notifyFilterAttributes == pkg.NotifyFilterAttributesCritical {
if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical {
hasFailingCriticalAttr := false
var statusFailingCrtiticalAttr pkg.AttributeStatus
var statusFailingCriticalAttr pkg.AttributeStatus
for attrId, attrData := range smartAttrs.Attributes {
//find failing attribute
@@ -64,7 +66,7 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev
}
// merge the status's of all critical attributes
statusFailingCrtiticalAttr = pkg.AttributeStatusSet(statusFailingCrtiticalAttr, attrData.GetStatus())
statusFailingCriticalAttr = pkg.AttributeStatusSet(statusFailingCriticalAttr, attrData.GetStatus())
//found a failing attribute, see if its critical
if device.IsScsi() && thresholds.ScsiMetadata[attrId].Critical {
@@ -89,7 +91,7 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev
return false
} else {
// check if any of the critical attributes have a status that we're looking for
return pkg.AttributeStatusHas(statusFailingCrtiticalAttr, requiredAttrStatus)
return pkg.AttributeStatusHas(statusFailingCriticalAttr, requiredAttrStatus)
}
} else {
webapp/backend/pkg/notify/notify_test.go
+35 -35
View File
@@ -15,56 +15,56 @@ func TestShouldNotify_MustSkipPassingDevices(t *testing.T) {
DeviceStatus: pkg.DeviceStatusPassed,
}
smartAttrs := measurements.Smart{}
notifyLevel := pkg.NotifyLevelFail
notifyFilterAttributes := pkg.NotifyFilterAttributesAll
statusThreshold := pkg.MetricsStatusThresholdBoth
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
//assert
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
func TestShouldNotify_NotifyLevelFail_FailingSmartDevice(t *testing.T) {
func TestShouldNotify_MetricsStatusThresholdBoth_FailingSmartDevice(t *testing.T) {
t.Parallel()
//setup
device := models.Device{
DeviceStatus: pkg.DeviceStatusFailedSmart,
}
smartAttrs := measurements.Smart{}
notifyLevel := pkg.NotifyLevelFail
notifyFilterAttributes := pkg.NotifyFilterAttributesAll
statusThreshold := pkg.MetricsStatusThresholdBoth
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
//assert
require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
func TestShouldNotify_NotifyLevelFailSmart_FailingSmartDevice(t *testing.T) {
func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing.T) {
t.Parallel()
//setup
device := models.Device{
DeviceStatus: pkg.DeviceStatusFailedSmart,
}
smartAttrs := measurements.Smart{}
notifyLevel := pkg.NotifyLevelFailSmart
notifyFilterAttributes := pkg.NotifyFilterAttributesAll
statusThreshold := pkg.MetricsStatusThresholdSmart
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
//assert
require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
func TestShouldNotify_NotifyLevelFailScrutiny_FailingSmartDevice(t *testing.T) {
func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testing.T) {
t.Parallel()
//setup
device := models.Device{
DeviceStatus: pkg.DeviceStatusFailedSmart,
}
smartAttrs := measurements.Smart{}
notifyLevel := pkg.NotifyLevelFailScrutiny
notifyFilterAttributes := pkg.NotifyFilterAttributesAll
statusThreshold := pkg.MetricsStatusThresholdScrutiny
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll
//assert
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
func TestShouldNotify_NotifyFilterAttributesCritical_WithCriticalAttrs(t *testing.T) {
func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t *testing.T) {
t.Parallel()
//setup
device := models.Device{
@@ -75,14 +75,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithCriticalAttrs(t *testin
Status: pkg.AttributeStatusFailedSmart,
},
}}
notifyLevel := pkg.NotifyLevelFail
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
statusThreshold := pkg.MetricsStatusThresholdBoth
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
//assert
require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
func TestShouldNotify_NotifyFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) {
func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) {
t.Parallel()
//setup
device := models.Device{
@@ -96,14 +96,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithMultipleCriticalAttrs(t
Status: pkg.AttributeStatusFailedScrutiny,
},
}}
notifyLevel := pkg.NotifyLevelFail
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
statusThreshold := pkg.MetricsStatusThresholdBoth
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
//assert
require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
func TestShouldNotify_NotifyFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) {
func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) {
t.Parallel()
//setup
device := models.Device{
@@ -114,14 +114,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithNoCriticalAttrs(t *test
Status: pkg.AttributeStatusFailedSmart,
},
}}
notifyLevel := pkg.NotifyLevelFail
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
statusThreshold := pkg.MetricsStatusThresholdBoth
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
//assert
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
func TestShouldNotify_NotifyFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) {
func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) {
t.Parallel()
//setup
device := models.Device{
@@ -132,14 +132,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithNoFailingCriticalAttrs(
Status: pkg.AttributeStatusPassed,
},
}}
notifyLevel := pkg.NotifyLevelFail
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
statusThreshold := pkg.MetricsStatusThresholdBoth
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
//assert
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
func TestShouldNotify_NotifyFilterAttributesCritical_NotifyLevelFailSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) {
func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresholdSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) {
t.Parallel()
//setup
device := models.Device{
@@ -153,9 +153,9 @@ func TestShouldNotify_NotifyFilterAttributesCritical_NotifyLevelFailSmart_WithCr
Status: pkg.AttributeStatusFailedScrutiny,
},
}}
notifyLevel := pkg.NotifyLevelFailSmart
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
statusThreshold := pkg.MetricsStatusThresholdSmart
notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical
//assert
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes))
}
webapp/backend/pkg/web/handler/get_settings.go
+1 -1
View File
@@ -11,7 +11,7 @@ func GetSettings(c *gin.Context) {
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
settings, err := deviceRepo.GetSettings(c)
settings, err := deviceRepo.LoadSettings(c)
if err != nil {
logger.Errorln("An error occurred while retrieving settings", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
webapp/backend/pkg/web/handler/upload_device_metrics.go
+6 -1
View File
@@ -67,7 +67,12 @@ func UploadDeviceMetrics(c *gin.Context) {
}
//check for error
if notify.ShouldNotify(updatedDevice, smartData, appConfig.GetString("notify.level"), appConfig.GetString("notify.filter_attributes")) {
if notify.ShouldNotify(
updatedDevice,
smartData,
pkg.MetricsStatusThreshold(appConfig.GetInt("dbsetting.metrics.status.threshold")),
pkg.MetricsStatusFilterAttributes(appConfig.GetInt("dbsetting.metrics.status.filter_attributes")),
) {
//send notifications
liveNotify := notify.New(
webapp/backend/pkg/web/middleware/logger.go
+1 -1
View File
@@ -32,7 +32,7 @@ func LoggerMiddleware(logger logrus.FieldLogger) gin.HandlerFunc {
hostname, err := os.Hostname()
if err != nil {
hostname = "unknow"
hostname = "unknown"
}
return func(c *gin.Context) {
webapp/backend/pkg/web/middleware/repository.go
+9
View File
@@ -1,6 +1,7 @@
package middleware
import (
"context"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/gin-gonic/gin"
@@ -14,6 +15,14 @@ func RepositoryMiddleware(appConfig config.Interface, globalLogger logrus.FieldL
panic(err)
}
// ensure the settings have been loaded into the app config during startup.
_, err = deviceRepo.LoadSettings(context.Background())
if err != nil {
panic(err)
}
//settings.UpdateSettingEntries()
//TODO: determine where we can call defer deviceRepo.Close()
return func(c *gin.Context) {
c.Set("DEVICE_REPOSITORY", deviceRepo)