From 6ca4bd4912489c8aab1bebfba9c6a3d2fc405b7f Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 13 Jul 2022 21:56:58 -0700 Subject: [PATCH 01/28] fix the WORKDIR for collector image. fixes #335 --- docker/Dockerfile.collector | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.collector b/docker/Dockerfile.collector index c4553fd..2839bdf 100644 --- a/docker/Dockerfile.collector +++ b/docker/Dockerfile.collector @@ -14,7 +14,7 @@ RUN make binary-clean binary-collector ######## FROM debian:bullseye-slim as runtime -WORKDIR /scrutiny +WORKDIR /opt/scrutiny ENV PATH="/opt/scrutiny/bin:${PATH}" RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates From e9c1de9664940649d898aa0581520dd329202d7c Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 16 Jul 2022 10:12:30 -0700 Subject: [PATCH 02/28] update support table in README. - freebsd binaries for collector and web working - macos binaries for arm and amd. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1360f49..c5e359f 100644 --- a/README.md +++ b/README.md @@ -239,9 +239,9 @@ scrutiny-collector-metrics run --debug --log-file /tmp/collector.log | linux-arm-6 | :white_check_mark: | | | linux-arm-7 | :white_check_mark: | web/collector only. see [#236](https://github.com/AnalogJ/scrutiny/issues/236) | | linux-arm64 | :white_check_mark: | :white_check_mark: | -| freebsd-amd64 | collector only. see [#238](https://github.com/AnalogJ/scrutiny/issues/238) | | -| macos-amd64 | | :white_check_mark: | -| macos-arm64 | | :white_check_mark: | +| freebsd-amd64 | :white_check_mark: | | +| macos-amd64 | :white_check_mark: | :white_check_mark: | +| macos-arm64 | :white_check_mark: | :white_check_mark: | | windows-amd64 | :white_check_mark: | WIP, see [#15](https://github.com/AnalogJ/scrutiny/issues/15) | | windows-arm64 | :white_check_mark: | | From 5b2746f389ed2bc5edd626baa439099256e5f1bb Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sun, 26 Jun 2022 15:30:52 -0700 Subject: [PATCH 03/28] initial settings table. --- .../migrations/m20220625184300/setting.go | 16 ++++++++++++++++ .../database/scrutiny_repository_migrations.go | 9 +++++++++ webapp/backend/pkg/models/setting.go | 17 ++++++++++++++--- 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 webapp/backend/pkg/database/migrations/m20220625184300/setting.go diff --git a/webapp/backend/pkg/database/migrations/m20220625184300/setting.go b/webapp/backend/pkg/database/migrations/m20220625184300/setting.go new file mode 100644 index 0000000..443c56b --- /dev/null +++ b/webapp/backend/pkg/database/migrations/m20220625184300/setting.go @@ -0,0 +1,16 @@ +package m20220625184300 + +import ( + "gorm.io/gorm" +) + +type Setting struct { + //GORM attributes, see: http://gorm.io/docs/conventions.html + gorm.Model + + SettingKeyName string `json:"setting_key_name"` + SettingDataType string `json:"setting_data_type"` + + SettingValueNumeric int64 `json:"setting_value_numeric"` + SettingValueString string `json:"setting_value_string"` +} diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index bb40add..33fd835 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -7,6 +7,7 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100" + "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220625184300" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" @@ -275,6 +276,14 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { return tx.Where("wwn = ?", "").Delete(&models.Device{}).Error }, }, + { + ID: "m20220716214900", // settings table. + Migrate: func(tx *gorm.DB) error { + + // adding the settings table. + return tx.AutoMigrate(m20220625184300.Setting{}) + }, + }, }) if err := m.Migrate(); err != nil { diff --git a/webapp/backend/pkg/models/setting.go b/webapp/backend/pkg/models/setting.go index d9a1d6b..15238f6 100644 --- a/webapp/backend/pkg/models/setting.go +++ b/webapp/backend/pkg/models/setting.go @@ -1,5 +1,16 @@ package models -// Temperature Format -// Date Format -// Device History window +import ( + "gorm.io/gorm" +) + +type Setting struct { + //GORM attributes, see: http://gorm.io/docs/conventions.html + gorm.Model + + SettingKeyName string `json:"setting_key_name"` + SettingDataType string `json:"setting_data_type"` + + SettingValueNumeric int64 `json:"setting_value_numeric"` + SettingValueString string `json:"setting_value_string"` +} From dd0c3e6fba3b9040e8ea558cdfae0ea6e79446b1 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 16 Jul 2022 22:07:50 -0700 Subject: [PATCH 04/28] rename the migration model package name. --- .../{m20220625184300 => m20220716214900}/setting.go | 2 +- .../backend/pkg/database/scrutiny_repository_migrations.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename webapp/backend/pkg/database/migrations/{m20220625184300 => m20220716214900}/setting.go (93%) diff --git a/webapp/backend/pkg/database/migrations/m20220625184300/setting.go b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go similarity index 93% rename from webapp/backend/pkg/database/migrations/m20220625184300/setting.go rename to webapp/backend/pkg/database/migrations/m20220716214900/setting.go index 443c56b..ba35a71 100644 --- a/webapp/backend/pkg/database/migrations/m20220625184300/setting.go +++ b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go @@ -1,4 +1,4 @@ -package m20220625184300 +package m20220716214900 import ( "gorm.io/gorm" diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 33fd835..6fd3596 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -7,7 +7,7 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100" - "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220625184300" + "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220716214900" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" @@ -277,11 +277,11 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { }, }, { - ID: "m20220716214900", // settings table. + ID: "m20220716214900", // add settings table. Migrate: func(tx *gorm.DB) error { // adding the settings table. - return tx.AutoMigrate(m20220625184300.Setting{}) + return tx.AutoMigrate(m20220716214900.Setting{}) }, }, }) From 99af2b8b16681d05f7cccd004263346067fb9879 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sun, 17 Jul 2022 10:32:28 -0700 Subject: [PATCH 05/28] WIP settings system. - updated dbdiagrams schema - [BREAKING] force failure if `notify.filter_attributes` or `notify.level` is set - added Settings table (and default values during migration) - Added Save Settings and Get Settings functions. - Added web API endpoints for getting and saving settings. - Deprecated old Notify* constants. Created new MetricsStatus* and MetricsNotifyLevel constants. --- docs/{INSTALL_NAS.md.go => INSTALL_NAS.md} | 0 docs/dbdiagram.io.txt | 114 +++++++++++------- webapp/backend/pkg/config/config.go | 46 ++----- webapp/backend/pkg/constants.go | 54 +++++++-- webapp/backend/pkg/database/interface.go | 3 + .../migrations/m20220716214900/setting.go | 5 +- .../scrutiny_repository_migrations.go | 29 ++++- .../database/scrutiny_repository_settings.go | 33 +++++ webapp/backend/pkg/models/setting.go | 16 --- webapp/backend/pkg/models/setting_entry.go | 22 ++++ webapp/backend/pkg/models/settings.go | 35 ++++++ .../backend/pkg/web/handler/get_settings.go | 25 ++++ .../backend/pkg/web/handler/save_settings.go | 33 +++++ webapp/backend/pkg/web/server.go | 2 + 14 files changed, 312 insertions(+), 105 deletions(-) rename docs/{INSTALL_NAS.md.go => INSTALL_NAS.md} (100%) create mode 100644 webapp/backend/pkg/database/scrutiny_repository_settings.go delete mode 100644 webapp/backend/pkg/models/setting.go create mode 100644 webapp/backend/pkg/models/setting_entry.go create mode 100644 webapp/backend/pkg/models/settings.go create mode 100644 webapp/backend/pkg/web/handler/get_settings.go create mode 100644 webapp/backend/pkg/web/handler/save_settings.go diff --git a/docs/INSTALL_NAS.md.go b/docs/INSTALL_NAS.md similarity index 100% rename from docs/INSTALL_NAS.md.go rename to docs/INSTALL_NAS.md diff --git a/docs/dbdiagram.io.txt b/docs/dbdiagram.io.txt index 7d23af7..23265ad 100644 --- a/docs/dbdiagram.io.txt +++ b/docs/dbdiagram.io.txt @@ -1,62 +1,88 @@ // SQLite Table(s) -Table device { - created_at timestamp - wwn varchar [pk] +Table Device { + //GORM attributes, see: http://gorm.io/docs/conventions.html + CreatedAt time + UpdatedAt time + DeletedAt time - //user provided - label varchar - host_id varchar + WWN string - // smartctl provided - device_name varchar - manufacturer varchar - model_name varchar - interface_type varchar - interface_speed varchar - serial_number varchar - firmware varchar - rotational_speed varchar - capacity varchar - form_factor varchar - smart_support varchar - device_protocol varchar - device_type varchar + DeviceName string + DeviceUUID string + DeviceSerialID string + DeviceLabel string + Manufacturer string + ModelName string + InterfaceType string + InterfaceSpeed string + SerialNumber string + Firmware string + RotationSpeed int + Capacity int64 + FormFactor string + SmartSupport bool + DeviceProtocol string//protocol determines which smart attribute types are available (ATA, NVMe, SCSI) + DeviceType string//device type is used for querying with -d/t flag, should only be used by collector. + + // User provided metadata + Label string + HostId string + + // Data set by Scrutiny + DeviceStatus enum +} + +Table Setting { + //GORM attributes, see: http://gorm.io/docs/conventions.html + + SettingKeyName string + SettingKeyDescription string + SettingDataType string + + SettingValueNumeric int64 + SettingValueString string } // InfluxDB Tables -Table device_temperature { - //timestamp - created_at timestamp - - //tags (indexed & queryable) - device_wwn varchar [pk] - - //fields - temp bigint - } +Table SmartTemperature { + Date time + DeviceWWN string //(tag) + Temp int64 +} -Table smart_ata_results { - //timestamp - created_at timestamp +Table Smart { + Date time + DeviceWWN string //(tag) + DeviceProtocol string - //tags (indexed & queryable) - device_wwn varchar [pk] - smart_status varchar - scrutiny_status varchar + //Metrics (fields) + Temp int64 + PowerOnHours int64 + PowerCycleCount int64 + //Smart Status + Status enum - - //fields - temp bigint - power_on_hours bigint - power_cycle_count bigint - + //SMART Attributes (fields) + Attr_ID_AttributeId int + Attr_ID_Value int64 + Attr_ID_Threshold int64 + Attr_ID_Worst int64 + Attr_ID_RawValue int64 + Attr_ID_RawString string + Attr_ID_WhenFailed string + //Generated data + Attr_ID_TransformedValue int64 + Attr_ID_Status enum + Attr_ID_StatusReason string + Attr_ID_FailureRate float64 } -Ref: device.wwn < smart_ata_results.device_wwn +Ref: Device.WWN < Smart.DeviceWWN +Ref: Device.WWN < SmartTemperature.DeviceWWN diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index 5f5d4cc..3a8cf52 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -2,7 +2,6 @@ package config import ( "github.com/analogj/go-util/utils" - "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/errors" "github.com/spf13/viper" "log" @@ -39,8 +38,6 @@ func (c *configuration) Init() error { c.SetDefault("log.file", "") c.SetDefault("notify.urls", []string{}) - c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll) - c.SetDefault("notify.level", pkg.NotifyLevelFail) c.SetDefault("web.influxdb.scheme", "http") c.SetDefault("web.influxdb.host", "localhost") @@ -55,17 +52,6 @@ func (c *configuration) Init() error { //c.SetDefault("disks.include", []string{}) //c.SetDefault("disks.exclude", []string{}) - //c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh") - //c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh") - //c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh") - - //c.SetDefault("collect.metric.enable", true) - //c.SetDefault("collect.metric.command", "-a -o on -S on") - //c.SetDefault("collect.long.enable", true) - //c.SetDefault("collect.long.command", "-a -o on -S on") - //c.SetDefault("collect.short.enable", true) - //c.SetDefault("collect.short.command", "-a -o on -S on") - //if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig c.SetConfigType("yaml") //c.SetConfigName("drawbridge") @@ -77,7 +63,7 @@ func (c *configuration) Init() error { c.AutomaticEnv() //CLI options will be added via the `Set()` function - return nil + return c.ValidateConfig() } func (c *configuration) ReadConfig(configFilePath string) error { @@ -120,24 +106,18 @@ func (c *configuration) ReadConfig(configFilePath string) error { // This function ensures that the merged config works correctly. func (c *configuration) ValidateConfig() error { - ////deserialize Questions - //questionsMap := map[string]Question{} - //err := c.UnmarshalKey("questions", &questionsMap) - // - //if err != nil { - // log.Printf("questions could not be deserialized correctly. %v", err) - // return err - //} - // - //for _, v := range questionsMap { - // - // typeContent, ok := v.Schema["type"].(string) - // if !ok || len(typeContent) == 0 { - // return errors.QuestionSyntaxError("`type` is required for questions") - // } - //} - // - // + //the following keys are deprecated, and no longer supported + /* + - notify.filter_attributes (replaced by metrics.status.filter_attributes SETTING) + - notify.level (replaced by metrics.notify.level and metrics.status.threshold SETTING) + */ + //TODO add docs and upgrade doc. + if c.IsSet("notify.filter_attributes") { + return errors.ConfigValidationError("`notify.filter_attributes` configuration option is deprecated. Replaced by option in Dashboard Settings page") + } + if c.IsSet("notify.level") { + return errors.ConfigValidationError("`notify.level` configuration option is deprecated. Replaced by option in Dashboard Settings page") + } return nil } diff --git a/webapp/backend/pkg/constants.go b/webapp/backend/pkg/constants.go index 9535963..f0a4b9d 100644 --- a/webapp/backend/pkg/constants.go +++ b/webapp/backend/pkg/constants.go @@ -4,17 +4,11 @@ const DeviceProtocolAta = "ATA" const DeviceProtocolScsi = "SCSI" const DeviceProtocolNvme = "NVMe" -const NotifyFilterAttributesAll = "all" -const NotifyFilterAttributesCritical = "critical" - -const NotifyLevelFail = "fail" -const NotifyLevelFailScrutiny = "fail_scrutiny" -const NotifyLevelFailSmart = "fail_smart" - //go:generate stringer -type=AttributeStatus +// AttributeStatus bitwise flag, 1,2,4,8,16,32,etc type AttributeStatus uint8 + const ( - // AttributeStatusPassed binary, 1,2,4,8,16,32,etc AttributeStatusPassed AttributeStatus = 0 AttributeStatusFailedSmart AttributeStatus = 1 AttributeStatusWarningScrutiny AttributeStatus = 2 @@ -30,9 +24,10 @@ func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 } //go:generate stringer -type=DeviceStatus +// DeviceStatus bitwise flag, 1,2,4,8,16,32,etc type DeviceStatus uint8 + const ( - // DeviceStatusPassed binary, 1,2,4,8,16,32,etc DeviceStatusPassed DeviceStatus = 0 DeviceStatusFailedSmart DeviceStatus = 1 DeviceStatusFailedScrutiny DeviceStatus = 2 @@ -42,3 +37,44 @@ func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus { return b | flag } func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus { return b &^ flag } func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag } func DeviceStatusHas(b, flag DeviceStatus) bool { return b&flag != 0 } + +// Metrics Specific Filtering & Threshold Constants +type MetricsNotifyLevel int64 + +const ( + MetricsNotifyLevelWarn MetricsNotifyLevel = 1 + MetricsNotifyLevelFail MetricsNotifyLevel = 2 +) + +type MetricsStatusFilterAttributes int64 + +const ( + MetricsStatusFilterAttributesAll MetricsStatusFilterAttributes = 0 + MetricsStatusFilterAttributesCritical MetricsStatusFilterAttributes = 1 +) + +// MetricsStatusThreshold bitwise flag, 1,2,4,8,16,32,etc +type MetricsStatusThreshold int64 + +const ( + MetricsStatusThresholdSmart MetricsStatusThreshold = 1 + MetricsStatusThresholdScrutiny MetricsStatusThreshold = 2 + + //shortcut + MetricsStatusThresholdBoth MetricsStatusThreshold = 3 +) + +// Deprecated +const NotifyFilterAttributesAll = "all" + +// Deprecated +const NotifyFilterAttributesCritical = "critical" + +// Deprecated +const NotifyLevelFail = "fail" + +// Deprecated +const NotifyLevelFailScrutiny = "fail_scrutiny" + +// Deprecated +const NotifyLevelFailSmart = "fail_smart" diff --git a/webapp/backend/pkg/database/interface.go b/webapp/backend/pkg/database/interface.go index 5ab7084..09bd6d5 100644 --- a/webapp/backend/pkg/database/interface.go +++ b/webapp/backend/pkg/database/interface.go @@ -28,4 +28,7 @@ type DeviceRepo interface { GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) + + GetSettings(ctx context.Context) (*models.Settings, error) + SaveSettings(ctx context.Context, settings models.Settings) error } diff --git a/webapp/backend/pkg/database/migrations/m20220716214900/setting.go b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go index ba35a71..8e38845 100644 --- a/webapp/backend/pkg/database/migrations/m20220716214900/setting.go +++ b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go @@ -8,8 +8,9 @@ type Setting struct { //GORM attributes, see: http://gorm.io/docs/conventions.html gorm.Model - SettingKeyName string `json:"setting_key_name"` - SettingDataType string `json:"setting_data_type"` + SettingKeyName string `json:"setting_key_name"` + SettingKeyDescription string `json:"setting_key_description"` + SettingDataType string `json:"setting_data_type"` SettingValueNumeric int64 `json:"setting_value_numeric"` SettingValueString string `json:"setting_value_string"` diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 6fd3596..2ae6388 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100" @@ -281,7 +282,33 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { Migrate: func(tx *gorm.DB) error { // adding the settings table. - return tx.AutoMigrate(m20220716214900.Setting{}) + err := tx.AutoMigrate(m20220716214900.Setting{}) + if err != nil { + return err + } + //add defaults. + + var defaultSettings = []m20220716214900.Setting{ + { + SettingKeyName: "metrics.notify.level", + SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)", + SettingDataType: "numeric", + SettingValueNumeric: int64(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn' + }, + { + SettingKeyName: "metrics.status.filter_attributes", + SettingKeyDescription: "Determines which attributes should impact device status", + SettingDataType: "numeric", + SettingValueNumeric: int64(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or 'critical' + }, + { + SettingKeyName: "metrics.status.threshold", + SettingKeyDescription: "Determines which threshold should impact device status", + SettingDataType: "string", + SettingValueNumeric: int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both' + }, + } + return tx.Create(&defaultSettings).Error }, }, }) diff --git a/webapp/backend/pkg/database/scrutiny_repository_settings.go b/webapp/backend/pkg/database/scrutiny_repository_settings.go new file mode 100644 index 0000000..05b7e72 --- /dev/null +++ b/webapp/backend/pkg/database/scrutiny_repository_settings.go @@ -0,0 +1,33 @@ +package database + +import ( + "context" + "fmt" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" +) + +func (sr *scrutinyRepository) GetSettings(ctx context.Context) (*models.Settings, error) { + settingsEntries := []models.SettingEntry{} + if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil { + return nil, fmt.Errorf("Could not get settings from DB: %v", err) + } + + settings := models.Settings{} + settings.PopulateFromSettingEntries(settingsEntries) + + return &settings, nil +} +func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error { + + //get current settings + settingsEntries := []models.SettingEntry{} + if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil { + return fmt.Errorf("Could not get settings from DB: %v", err) + } + + // override with values from settings object + settingsEntries = settings.UpdateSettingEntries(settingsEntries) + + // store in database. + return sr.gormClient.Updates(&settingsEntries).Error +} diff --git a/webapp/backend/pkg/models/setting.go b/webapp/backend/pkg/models/setting.go deleted file mode 100644 index 15238f6..0000000 --- a/webapp/backend/pkg/models/setting.go +++ /dev/null @@ -1,16 +0,0 @@ -package models - -import ( - "gorm.io/gorm" -) - -type Setting struct { - //GORM attributes, see: http://gorm.io/docs/conventions.html - gorm.Model - - SettingKeyName string `json:"setting_key_name"` - SettingDataType string `json:"setting_data_type"` - - SettingValueNumeric int64 `json:"setting_value_numeric"` - SettingValueString string `json:"setting_value_string"` -} diff --git a/webapp/backend/pkg/models/setting_entry.go b/webapp/backend/pkg/models/setting_entry.go new file mode 100644 index 0000000..b0f89b3 --- /dev/null +++ b/webapp/backend/pkg/models/setting_entry.go @@ -0,0 +1,22 @@ +package models + +import ( + "gorm.io/gorm" +) + +// SettingEntry matches a setting row in the database +type SettingEntry struct { + //GORM attributes, see: http://gorm.io/docs/conventions.html + gorm.Model + + SettingKeyName string `json:"setting_key_name" gorm:"unique;not null"` + SettingKeyDescription string `json:"setting_key_description"` + SettingDataType string `json:"setting_data_type"` + + SettingValueNumeric int64 `json:"setting_value_numeric"` + SettingValueString string `json:"setting_value_string"` +} + +func (s SettingEntry) TableName() string { + return "settings" +} diff --git a/webapp/backend/pkg/models/settings.go b/webapp/backend/pkg/models/settings.go new file mode 100644 index 0000000..3cf1431 --- /dev/null +++ b/webapp/backend/pkg/models/settings.go @@ -0,0 +1,35 @@ +package models + +import "github.com/analogj/scrutiny/webapp/backend/pkg" + +// Settings is made up of parsed SettingEntry objects retrieved from the database +type Settings struct { + MetricsNotifyLevel pkg.MetricsNotifyLevel `json:"metrics_notify_level"` + MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics_status_filter_attributes"` + MetricsStatusThreshold pkg.MetricsStatusThreshold `json:"metrics_status_threshold"` +} + +func (s *Settings) PopulateFromSettingEntries(entries []SettingEntry) { + for _, entry := range entries { + if entry.SettingKeyName == "metrics.notify.level" { + s.MetricsNotifyLevel = pkg.MetricsNotifyLevel(entry.SettingValueNumeric) + } else if entry.SettingKeyName == "metrics.status.filter_attributes" { + s.MetricsStatusFilterAttributes = pkg.MetricsStatusFilterAttributes(entry.SettingValueNumeric) + } else if entry.SettingKeyName == "metrics.status.threshold" { + s.MetricsStatusThreshold = pkg.MetricsStatusThreshold(entry.SettingValueNumeric) + } + } +} + +func (s *Settings) UpdateSettingEntries(entries []SettingEntry) []SettingEntry { + for _, entry := range entries { + if entry.SettingKeyName == "metrics.notify.level" { + entry.SettingValueNumeric = int64(s.MetricsNotifyLevel) + } else if entry.SettingKeyName == "metrics.status.filter_attributes" { + entry.SettingValueNumeric = int64(s.MetricsStatusFilterAttributes) + } else if entry.SettingKeyName == "metrics.status.threshold" { + entry.SettingValueNumeric = int64(s.MetricsStatusThreshold) + } + } + return entries +} diff --git a/webapp/backend/pkg/web/handler/get_settings.go b/webapp/backend/pkg/web/handler/get_settings.go new file mode 100644 index 0000000..ca66082 --- /dev/null +++ b/webapp/backend/pkg/web/handler/get_settings.go @@ -0,0 +1,25 @@ +package handler + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "net/http" +) + +func GetSettings(c *gin.Context) { + logger := c.MustGet("LOGGER").(logrus.FieldLogger) + deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) + + settings, err := deviceRepo.GetSettings(c) + if err != nil { + logger.Errorln("An error occurred while retrieving settings", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "settings": settings, + }) +} diff --git a/webapp/backend/pkg/web/handler/save_settings.go b/webapp/backend/pkg/web/handler/save_settings.go new file mode 100644 index 0000000..d466169 --- /dev/null +++ b/webapp/backend/pkg/web/handler/save_settings.go @@ -0,0 +1,33 @@ +package handler + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "net/http" +) + +func SaveSettings(c *gin.Context) { + logger := c.MustGet("LOGGER").(logrus.FieldLogger) + deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) + + var settings models.Settings + err := c.BindJSON(&settings) + if err != nil { + logger.Errorln("Cannot parse updated settings", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + err = deviceRepo.SaveSettings(c, settings) + if err != nil { + logger.Errorln("An error occurred while saving settings", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + }) +} diff --git a/webapp/backend/pkg/web/server.go b/webapp/backend/pkg/web/server.go index d2a5cf6..0ef8bc8 100644 --- a/webapp/backend/pkg/web/server.go +++ b/webapp/backend/pkg/web/server.go @@ -50,6 +50,8 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine { api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details api.DELETE("/device/:wwn", handler.DeleteDevice) //used by UI to delete device + api.GET("/settings", handler.GetSettings) //used to get settings + api.POST("/settings", handler.SaveSettings) //used to save settings } } From 29bc79996b77899d8b5ab06452ff51d337b7207e Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Tue, 19 Jul 2022 23:12:23 -0700 Subject: [PATCH 06/28] working settings update. Settings are loaded from the DB and added to the AppConfig during startup. When updating settings, they are stored in AppConfig, and written do the database. --- webapp/backend/pkg/config/config.go | 4 ++ webapp/backend/pkg/config/interface.go | 4 ++ webapp/backend/pkg/constants.go | 9 --- webapp/backend/pkg/database/interface.go | 5 +- .../scrutiny_repository_migrations.go | 2 +- .../database/scrutiny_repository_settings.go | 65 ++++++++++++++--- webapp/backend/pkg/models/setting_entry.go | 2 +- webapp/backend/pkg/models/settings.go | 45 ++++-------- webapp/backend/pkg/notify/notify.go | 16 +++-- webapp/backend/pkg/notify/notify_test.go | 70 +++++++++---------- .../backend/pkg/web/handler/get_settings.go | 2 +- .../pkg/web/handler/upload_device_metrics.go | 7 +- webapp/backend/pkg/web/middleware/logger.go | 2 +- .../backend/pkg/web/middleware/repository.go | 9 +++ 14 files changed, 144 insertions(+), 98 deletions(-) diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index 3a8cf52..ed9ed93 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -66,6 +66,10 @@ func (c *configuration) Init() error { return c.ValidateConfig() } +func (c *configuration) SubKeys(key string) []string { + return c.Sub(key).AllKeys() +} + func (c *configuration) ReadConfig(configFilePath string) error { //make sure that we specify that this is the correct config path (for eventual WriteConfig() calls) c.SetConfigFile(configFilePath) diff --git a/webapp/backend/pkg/config/interface.go b/webapp/backend/pkg/config/interface.go index 8f0b773..51f8400 100644 --- a/webapp/backend/pkg/config/interface.go +++ b/webapp/backend/pkg/config/interface.go @@ -12,12 +12,16 @@ type Interface interface { WriteConfig() error Set(key string, value interface{}) SetDefault(key string, value interface{}) + MergeConfigMap(cfg map[string]interface{}) error AllSettings() map[string]interface{} + AllKeys() []string + SubKeys(key string) []string IsSet(key string) bool Get(key string) interface{} GetBool(key string) bool GetInt(key string) int + GetInt64(key string) int64 GetString(key string) string GetStringSlice(key string) []string UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error diff --git a/webapp/backend/pkg/constants.go b/webapp/backend/pkg/constants.go index f0a4b9d..d05bb15 100644 --- a/webapp/backend/pkg/constants.go +++ b/webapp/backend/pkg/constants.go @@ -67,14 +67,5 @@ const ( // Deprecated const NotifyFilterAttributesAll = "all" -// Deprecated -const NotifyFilterAttributesCritical = "critical" - // Deprecated const NotifyLevelFail = "fail" - -// Deprecated -const NotifyLevelFailScrutiny = "fail_scrutiny" - -// Deprecated -const NotifyLevelFailSmart = "fail_smart" diff --git a/webapp/backend/pkg/database/interface.go b/webapp/backend/pkg/database/interface.go index 09bd6d5..f140c26 100644 --- a/webapp/backend/pkg/database/interface.go +++ b/webapp/backend/pkg/database/interface.go @@ -11,9 +11,6 @@ import ( type DeviceRepo interface { Close() error - //GetSettings() - //SaveSetting() - RegisterDevice(ctx context.Context, dev models.Device) error GetDevices(ctx context.Context) ([]models.Device, error) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) @@ -29,6 +26,6 @@ type DeviceRepo interface { GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) - GetSettings(ctx context.Context) (*models.Settings, error) + LoadSettings(ctx context.Context) (*models.Settings, error) SaveSettings(ctx context.Context, settings models.Settings) error } diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 2ae6388..26f01fd 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -304,7 +304,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { { SettingKeyName: "metrics.status.threshold", SettingKeyDescription: "Determines which threshold should impact device status", - SettingDataType: "string", + SettingDataType: "numeric", SettingValueNumeric: int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both' }, } diff --git a/webapp/backend/pkg/database/scrutiny_repository_settings.go b/webapp/backend/pkg/database/scrutiny_repository_settings.go index 05b7e72..1889b8f 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_settings.go +++ b/webapp/backend/pkg/database/scrutiny_repository_settings.go @@ -4,30 +4,79 @@ import ( "context" "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/mitchellh/mapstructure" ) -func (sr *scrutinyRepository) GetSettings(ctx context.Context) (*models.Settings, error) { +const DBSETTING_SUBKEY = "dbsetting" + +// LoadSettings will retrieve settings from the database, store them in the AppConfig object, and return a Settings struct +func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Settings, error) { settingsEntries := []models.SettingEntry{} if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil { return nil, fmt.Errorf("Could not get settings from DB: %v", err) } - settings := models.Settings{} - settings.PopulateFromSettingEntries(settingsEntries) + // store retrieved settings in the AppConfig obj + for _, settingsEntry := range settingsEntries { + configKey := fmt.Sprintf("%s.%s", DBSETTING_SUBKEY, settingsEntry.SettingKeyName) + if settingsEntry.SettingDataType == "numeric" { + sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueNumeric) + } else if settingsEntry.SettingDataType == "string" { + sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueString) + } + } + + // unmarshal the dbsetting object data to a settings object. + var settings models.Settings + err := sr.appConfig.UnmarshalKey(DBSETTING_SUBKEY, &settings) + if err != nil { + return nil, err + } return &settings, nil } + +// testing +// curl -d '{"metrics": { "notify": { "level": 5 }, "status": { "filter_attributes": 5, "threshold": 5 } }}' -H "Content-Type: application/json" -X POST http://localhost:9090/api/settings +// SaveSettings will update settings in AppConfig object, then save the settings to the database. func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error { - //get current settings + //save the entries to the appconfig + settingsMap := &map[string]interface{}{} + err := mapstructure.Decode(settings, &settingsMap) + if err != nil { + return err + } + settingsWrapperMap := map[string]interface{}{} + settingsWrapperMap[DBSETTING_SUBKEY] = *settingsMap + err = sr.appConfig.MergeConfigMap(settingsWrapperMap) + if err != nil { + return err + } + + //retrieve current settings from the database settingsEntries := []models.SettingEntry{} if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil { return fmt.Errorf("Could not get settings from DB: %v", err) } - // override with values from settings object - settingsEntries = settings.UpdateSettingEntries(settingsEntries) + //update settingsEntries + for ndx, settingsEntry := range settingsEntries { + configKey := fmt.Sprintf("%s.%s", DBSETTING_SUBKEY, settingsEntry.SettingKeyName) - // store in database. - return sr.gormClient.Updates(&settingsEntries).Error + if settingsEntry.SettingDataType == "numeric" { + settingsEntries[ndx].SettingValueNumeric = sr.appConfig.GetInt(configKey) + } else if settingsEntry.SettingDataType == "string" { + settingsEntries[ndx].SettingValueString = sr.appConfig.GetString(configKey) + } + + // store in database. + //TODO: this should be `sr.gormClient.Updates(&settingsEntries).Error` + err := sr.gormClient.Model(&models.SettingEntry{}).Where([]uint{settingsEntry.ID}).Select("setting_value_numeric", "setting_value_string").Updates(settingsEntries[ndx]).Error + if err != nil { + return err + } + + } + return nil } diff --git a/webapp/backend/pkg/models/setting_entry.go b/webapp/backend/pkg/models/setting_entry.go index b0f89b3..48d2c4c 100644 --- a/webapp/backend/pkg/models/setting_entry.go +++ b/webapp/backend/pkg/models/setting_entry.go @@ -13,7 +13,7 @@ type SettingEntry struct { SettingKeyDescription string `json:"setting_key_description"` SettingDataType string `json:"setting_data_type"` - SettingValueNumeric int64 `json:"setting_value_numeric"` + SettingValueNumeric int `json:"setting_value_numeric"` SettingValueString string `json:"setting_value_string"` } diff --git a/webapp/backend/pkg/models/settings.go b/webapp/backend/pkg/models/settings.go index 3cf1431..ec29681 100644 --- a/webapp/backend/pkg/models/settings.go +++ b/webapp/backend/pkg/models/settings.go @@ -1,35 +1,20 @@ package models -import "github.com/analogj/scrutiny/webapp/backend/pkg" - // Settings is made up of parsed SettingEntry objects retrieved from the database +//type Settings struct { +// MetricsNotifyLevel pkg.MetricsNotifyLevel `json:"metrics.notify.level" mapstructure:"metrics.notify.level"` +// MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics.status.filter_attributes" mapstructure:"metrics.status.filter_attributes"` +// MetricsStatusThreshold pkg.MetricsStatusThreshold `json:"metrics.status.threshold" mapstructure:"metrics.status.threshold"` +//} + type Settings struct { - MetricsNotifyLevel pkg.MetricsNotifyLevel `json:"metrics_notify_level"` - MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics_status_filter_attributes"` - MetricsStatusThreshold pkg.MetricsStatusThreshold `json:"metrics_status_threshold"` -} - -func (s *Settings) PopulateFromSettingEntries(entries []SettingEntry) { - for _, entry := range entries { - if entry.SettingKeyName == "metrics.notify.level" { - s.MetricsNotifyLevel = pkg.MetricsNotifyLevel(entry.SettingValueNumeric) - } else if entry.SettingKeyName == "metrics.status.filter_attributes" { - s.MetricsStatusFilterAttributes = pkg.MetricsStatusFilterAttributes(entry.SettingValueNumeric) - } else if entry.SettingKeyName == "metrics.status.threshold" { - s.MetricsStatusThreshold = pkg.MetricsStatusThreshold(entry.SettingValueNumeric) - } - } -} - -func (s *Settings) UpdateSettingEntries(entries []SettingEntry) []SettingEntry { - for _, entry := range entries { - if entry.SettingKeyName == "metrics.notify.level" { - entry.SettingValueNumeric = int64(s.MetricsNotifyLevel) - } else if entry.SettingKeyName == "metrics.status.filter_attributes" { - entry.SettingValueNumeric = int64(s.MetricsStatusFilterAttributes) - } else if entry.SettingKeyName == "metrics.status.threshold" { - entry.SettingValueNumeric = int64(s.MetricsStatusThreshold) - } - } - return entries + Metrics struct { + Notify struct { + Level int `json:"level" mapstructure:"level"` + } `json:"notify" mapstructure:"notify"` + Status struct { + FilterAttributes int `json:"filter_attributes" mapstructure:"filter_attributes"` + Threshold int `json:"threshold" mapstructure:"threshold"` + } `json:"status" mapstructure:"status"` + } `json:"metrics" mapstructure:"metrics"` } diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index bfc6510..657b007 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -29,20 +29,22 @@ const NotifyFailureTypeSmartFailure = "SmartFailure" const NotifyFailureTypeScrutinyFailure = "ScrutinyFailure" // ShouldNotify check if the error Message should be filtered (level mismatch or filtered_attributes) -func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLevel string, notifyFilterAttributes string) bool { +func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes) bool { // 1. check if the device is healthy if device.DeviceStatus == pkg.DeviceStatusPassed { return false } + //TODO: cannot check for warning notifyLevel yet. + // setup constants for comparison var requiredDeviceStatus pkg.DeviceStatus var requiredAttrStatus pkg.AttributeStatus - if notifyLevel == pkg.NotifyLevelFail { + if statusThreshold == pkg.MetricsStatusThresholdBoth { // either scrutiny or smart failures should trigger an email requiredDeviceStatus = pkg.DeviceStatusSet(pkg.DeviceStatusFailedSmart, pkg.DeviceStatusFailedScrutiny) requiredAttrStatus = pkg.AttributeStatusSet(pkg.AttributeStatusFailedSmart, pkg.AttributeStatusFailedScrutiny) - } else if notifyLevel == pkg.NotifyLevelFailSmart { + } else if statusThreshold == pkg.MetricsStatusThresholdSmart { //only smart failures requiredDeviceStatus = pkg.DeviceStatusFailedSmart requiredAttrStatus = pkg.AttributeStatusFailedSmart @@ -53,9 +55,9 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev // 2. check if the attributes that are failing should be filtered (non-critical) // 3. for any unfiltered attribute, store the failure reason (Smart or Scrutiny) - if notifyFilterAttributes == pkg.NotifyFilterAttributesCritical { + if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical { hasFailingCriticalAttr := false - var statusFailingCrtiticalAttr pkg.AttributeStatus + var statusFailingCriticalAttr pkg.AttributeStatus for attrId, attrData := range smartAttrs.Attributes { //find failing attribute @@ -64,7 +66,7 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev } // merge the status's of all critical attributes - statusFailingCrtiticalAttr = pkg.AttributeStatusSet(statusFailingCrtiticalAttr, attrData.GetStatus()) + statusFailingCriticalAttr = pkg.AttributeStatusSet(statusFailingCriticalAttr, attrData.GetStatus()) //found a failing attribute, see if its critical if device.IsScsi() && thresholds.ScsiMetadata[attrId].Critical { @@ -89,7 +91,7 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLev return false } else { // check if any of the critical attributes have a status that we're looking for - return pkg.AttributeStatusHas(statusFailingCrtiticalAttr, requiredAttrStatus) + return pkg.AttributeStatusHas(statusFailingCriticalAttr, requiredAttrStatus) } } else { diff --git a/webapp/backend/pkg/notify/notify_test.go b/webapp/backend/pkg/notify/notify_test.go index aadb5f9..b891ede 100644 --- a/webapp/backend/pkg/notify/notify_test.go +++ b/webapp/backend/pkg/notify/notify_test.go @@ -15,56 +15,56 @@ func TestShouldNotify_MustSkipPassingDevices(t *testing.T) { DeviceStatus: pkg.DeviceStatusPassed, } smartAttrs := measurements.Smart{} - notifyLevel := pkg.NotifyLevelFail - notifyFilterAttributes := pkg.NotifyFilterAttributesAll + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll //assert - require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } -func TestShouldNotify_NotifyLevelFail_FailingSmartDevice(t *testing.T) { +func TestShouldNotify_MetricsStatusThresholdBoth_FailingSmartDevice(t *testing.T) { t.Parallel() //setup device := models.Device{ DeviceStatus: pkg.DeviceStatusFailedSmart, } smartAttrs := measurements.Smart{} - notifyLevel := pkg.NotifyLevelFail - notifyFilterAttributes := pkg.NotifyFilterAttributesAll + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll //assert - require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } -func TestShouldNotify_NotifyLevelFailSmart_FailingSmartDevice(t *testing.T) { +func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing.T) { t.Parallel() //setup device := models.Device{ DeviceStatus: pkg.DeviceStatusFailedSmart, } smartAttrs := measurements.Smart{} - notifyLevel := pkg.NotifyLevelFailSmart - notifyFilterAttributes := pkg.NotifyFilterAttributesAll + statusThreshold := pkg.MetricsStatusThresholdSmart + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll //assert - require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } -func TestShouldNotify_NotifyLevelFailScrutiny_FailingSmartDevice(t *testing.T) { +func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testing.T) { t.Parallel() //setup device := models.Device{ DeviceStatus: pkg.DeviceStatusFailedSmart, } smartAttrs := measurements.Smart{} - notifyLevel := pkg.NotifyLevelFailScrutiny - notifyFilterAttributes := pkg.NotifyFilterAttributesAll + statusThreshold := pkg.MetricsStatusThresholdScrutiny + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll //assert - require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } -func TestShouldNotify_NotifyFilterAttributesCritical_WithCriticalAttrs(t *testing.T) { +func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t *testing.T) { t.Parallel() //setup device := models.Device{ @@ -75,14 +75,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithCriticalAttrs(t *testin Status: pkg.AttributeStatusFailedSmart, }, }} - notifyLevel := pkg.NotifyLevelFail - notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical //assert - require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } -func TestShouldNotify_NotifyFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) { +func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) { t.Parallel() //setup device := models.Device{ @@ -96,14 +96,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithMultipleCriticalAttrs(t Status: pkg.AttributeStatusFailedScrutiny, }, }} - notifyLevel := pkg.NotifyLevelFail - notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical //assert - require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } -func TestShouldNotify_NotifyFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) { +func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) { t.Parallel() //setup device := models.Device{ @@ -114,14 +114,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithNoCriticalAttrs(t *test Status: pkg.AttributeStatusFailedSmart, }, }} - notifyLevel := pkg.NotifyLevelFail - notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical //assert - require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } -func TestShouldNotify_NotifyFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) { +func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) { t.Parallel() //setup device := models.Device{ @@ -132,14 +132,14 @@ func TestShouldNotify_NotifyFilterAttributesCritical_WithNoFailingCriticalAttrs( Status: pkg.AttributeStatusPassed, }, }} - notifyLevel := pkg.NotifyLevelFail - notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical //assert - require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } -func TestShouldNotify_NotifyFilterAttributesCritical_NotifyLevelFailSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) { +func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresholdSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) { t.Parallel() //setup device := models.Device{ @@ -153,9 +153,9 @@ func TestShouldNotify_NotifyFilterAttributesCritical_NotifyLevelFailSmart_WithCr Status: pkg.AttributeStatusFailedScrutiny, }, }} - notifyLevel := pkg.NotifyLevelFailSmart - notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + statusThreshold := pkg.MetricsStatusThresholdSmart + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical //assert - require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) + require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } diff --git a/webapp/backend/pkg/web/handler/get_settings.go b/webapp/backend/pkg/web/handler/get_settings.go index ca66082..b6969ea 100644 --- a/webapp/backend/pkg/web/handler/get_settings.go +++ b/webapp/backend/pkg/web/handler/get_settings.go @@ -11,7 +11,7 @@ func GetSettings(c *gin.Context) { logger := c.MustGet("LOGGER").(logrus.FieldLogger) deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) - settings, err := deviceRepo.GetSettings(c) + settings, err := deviceRepo.LoadSettings(c) if err != nil { logger.Errorln("An error occurred while retrieving settings", err) c.JSON(http.StatusInternalServerError, gin.H{"success": false}) diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index d27f66b..0d4b74c 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -67,7 +67,12 @@ func UploadDeviceMetrics(c *gin.Context) { } //check for error - if notify.ShouldNotify(updatedDevice, smartData, appConfig.GetString("notify.level"), appConfig.GetString("notify.filter_attributes")) { + if notify.ShouldNotify( + updatedDevice, + smartData, + pkg.MetricsStatusThreshold(appConfig.GetInt("dbsetting.metrics.status.threshold")), + pkg.MetricsStatusFilterAttributes(appConfig.GetInt("dbsetting.metrics.status.filter_attributes")), + ) { //send notifications liveNotify := notify.New( diff --git a/webapp/backend/pkg/web/middleware/logger.go b/webapp/backend/pkg/web/middleware/logger.go index dc988bb..7568efe 100644 --- a/webapp/backend/pkg/web/middleware/logger.go +++ b/webapp/backend/pkg/web/middleware/logger.go @@ -32,7 +32,7 @@ func LoggerMiddleware(logger logrus.FieldLogger) gin.HandlerFunc { hostname, err := os.Hostname() if err != nil { - hostname = "unknow" + hostname = "unknown" } return func(c *gin.Context) { diff --git a/webapp/backend/pkg/web/middleware/repository.go b/webapp/backend/pkg/web/middleware/repository.go index 3fe58d2..f545a33 100644 --- a/webapp/backend/pkg/web/middleware/repository.go +++ b/webapp/backend/pkg/web/middleware/repository.go @@ -1,6 +1,7 @@ package middleware import ( + "context" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/database" "github.com/gin-gonic/gin" @@ -14,6 +15,14 @@ func RepositoryMiddleware(appConfig config.Interface, globalLogger logrus.FieldL panic(err) } + // ensure the settings have been loaded into the app config during startup. + _, err = deviceRepo.LoadSettings(context.Background()) + if err != nil { + panic(err) + } + + //settings.UpdateSettingEntries() + //TODO: determine where we can call defer deviceRepo.Close() return func(c *gin.Context) { c.Set("DEVICE_REPOSITORY", deviceRepo) From c0f1dfdb0b3e3ccaaf26368002da0060fb5698c7 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 20 Jul 2022 21:55:21 -0700 Subject: [PATCH 07/28] fixing config mock. --- webapp/backend/pkg/config/config.go | 9 +++ webapp/backend/pkg/config/interface.go | 1 + webapp/backend/pkg/config/mock/mock_config.go | 71 +++++++++++++++++++ .../database/scrutiny_repository_settings.go | 15 ++-- webapp/backend/pkg/web/server_test.go | 54 ++++++++++---- 5 files changed, 128 insertions(+), 22 deletions(-) diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index ed9ed93..3d689f3 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -9,6 +9,8 @@ import ( "strings" ) +const DBSETTING_SUBKEY = "dbsetting" + // When initializing this class the following methods must be called: // Config.New // Config.Init @@ -70,6 +72,13 @@ func (c *configuration) SubKeys(key string) []string { return c.Sub(key).AllKeys() } +func (c *configuration) Sub(key string) Interface { + config := configuration{ + Viper: c.Viper.Sub(key), + } + return &config +} + func (c *configuration) ReadConfig(configFilePath string) error { //make sure that we specify that this is the correct config path (for eventual WriteConfig() calls) c.SetConfigFile(configFilePath) diff --git a/webapp/backend/pkg/config/interface.go b/webapp/backend/pkg/config/interface.go index 51f8400..d041dc2 100644 --- a/webapp/backend/pkg/config/interface.go +++ b/webapp/backend/pkg/config/interface.go @@ -14,6 +14,7 @@ type Interface interface { SetDefault(key string, value interface{}) MergeConfigMap(cfg map[string]interface{}) error + Sub(key string) Interface AllSettings() map[string]interface{} AllKeys() []string SubKeys(key string) []string diff --git a/webapp/backend/pkg/config/mock/mock_config.go b/webapp/backend/pkg/config/mock/mock_config.go index 8e54f6f..1b61b2c 100644 --- a/webapp/backend/pkg/config/mock/mock_config.go +++ b/webapp/backend/pkg/config/mock/mock_config.go @@ -7,6 +7,7 @@ package mock_config import ( reflect "reflect" + config "github.com/analogj/scrutiny/webapp/backend/pkg/config" gomock "github.com/golang/mock/gomock" viper "github.com/spf13/viper" ) @@ -34,6 +35,20 @@ func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { return m.recorder } +// AllKeys mocks base method. +func (m *MockInterface) AllKeys() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllKeys") + ret0, _ := ret[0].([]string) + return ret0 +} + +// AllKeys indicates an expected call of AllKeys. +func (mr *MockInterfaceMockRecorder) AllKeys() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllKeys", reflect.TypeOf((*MockInterface)(nil).AllKeys)) +} + // AllSettings mocks base method. func (m *MockInterface) AllSettings() map[string]interface{} { m.ctrl.T.Helper() @@ -90,6 +105,20 @@ func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key) } +// GetInt64 mocks base method. +func (m *MockInterface) GetInt64(key string) int64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInt64", key) + ret0, _ := ret[0].(int64) + return ret0 +} + +// GetInt64 indicates an expected call of GetInt64. +func (mr *MockInterfaceMockRecorder) GetInt64(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt64", reflect.TypeOf((*MockInterface)(nil).GetInt64), key) +} + // GetString mocks base method. func (m *MockInterface) GetString(key string) string { m.ctrl.T.Helper() @@ -146,6 +175,20 @@ func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key) } +// MergeConfigMap mocks base method. +func (m *MockInterface) MergeConfigMap(cfg map[string]interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MergeConfigMap", cfg) + ret0, _ := ret[0].(error) + return ret0 +} + +// MergeConfigMap indicates an expected call of MergeConfigMap. +func (mr *MockInterfaceMockRecorder) MergeConfigMap(cfg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MergeConfigMap", reflect.TypeOf((*MockInterface)(nil).MergeConfigMap), cfg) +} + // ReadConfig mocks base method. func (m *MockInterface) ReadConfig(configFilePath string) error { m.ctrl.T.Helper() @@ -184,6 +227,34 @@ func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value) } +// Sub mocks base method. +func (m *MockInterface) Sub(key string) config.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sub", key) + ret0, _ := ret[0].(config.Interface) + return ret0 +} + +// Sub indicates an expected call of Sub. +func (mr *MockInterfaceMockRecorder) Sub(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sub", reflect.TypeOf((*MockInterface)(nil).Sub), key) +} + +// SubKeys mocks base method. +func (m *MockInterface) SubKeys(key string) []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SubKeys", key) + ret0, _ := ret[0].([]string) + return ret0 +} + +// SubKeys indicates an expected call of SubKeys. +func (mr *MockInterfaceMockRecorder) SubKeys(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubKeys", reflect.TypeOf((*MockInterface)(nil).SubKeys), key) +} + // UnmarshalKey mocks base method. func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error { m.ctrl.T.Helper() diff --git a/webapp/backend/pkg/database/scrutiny_repository_settings.go b/webapp/backend/pkg/database/scrutiny_repository_settings.go index 1889b8f..329b375 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_settings.go +++ b/webapp/backend/pkg/database/scrutiny_repository_settings.go @@ -3,12 +3,11 @@ package database import ( "context" "fmt" + "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/mitchellh/mapstructure" ) -const DBSETTING_SUBKEY = "dbsetting" - // LoadSettings will retrieve settings from the database, store them in the AppConfig object, and return a Settings struct func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Settings, error) { settingsEntries := []models.SettingEntry{} @@ -18,18 +17,18 @@ func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Setting // store retrieved settings in the AppConfig obj for _, settingsEntry := range settingsEntries { - configKey := fmt.Sprintf("%s.%s", DBSETTING_SUBKEY, settingsEntry.SettingKeyName) + configKey := fmt.Sprintf("%s.%s", config.DBSETTING_SUBKEY, settingsEntry.SettingKeyName) if settingsEntry.SettingDataType == "numeric" { - sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueNumeric) + sr.appConfig.Set(configKey, settingsEntry.SettingValueNumeric) } else if settingsEntry.SettingDataType == "string" { - sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueString) + sr.appConfig.Set(configKey, settingsEntry.SettingValueString) } } // unmarshal the dbsetting object data to a settings object. var settings models.Settings - err := sr.appConfig.UnmarshalKey(DBSETTING_SUBKEY, &settings) + err := sr.appConfig.UnmarshalKey(config.DBSETTING_SUBKEY, &settings) if err != nil { return nil, err } @@ -48,7 +47,7 @@ func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models. return err } settingsWrapperMap := map[string]interface{}{} - settingsWrapperMap[DBSETTING_SUBKEY] = *settingsMap + settingsWrapperMap[config.DBSETTING_SUBKEY] = *settingsMap err = sr.appConfig.MergeConfigMap(settingsWrapperMap) if err != nil { return err @@ -62,7 +61,7 @@ func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models. //update settingsEntries for ndx, settingsEntry := range settingsEntries { - configKey := fmt.Sprintf("%s.%s", DBSETTING_SUBKEY, settingsEntry.SettingKeyName) + configKey := fmt.Sprintf("%s.%s", config.DBSETTING_SUBKEY, settingsEntry.SettingKeyName) if settingsEntry.SettingDataType == "numeric" { settingsEntries[ndx].SettingValueNumeric = sr.appConfig.GetInt(configKey) diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index b64617c..28b78cc 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -89,6 +89,8 @@ func (suite *ServerTestSuite) TestHealthRoute() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes() fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes() fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -130,6 +132,8 @@ func (suite *ServerTestSuite) TestRegisterDevicesRoute() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes() fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes() fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -170,6 +174,8 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -186,8 +192,9 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() { } else { fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes() } - fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail) - fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) ae := web.AppEngine{ Config: fakeConfig, @@ -219,10 +226,13 @@ func (suite *ServerTestSuite) TestPopulateMultiple() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) //fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") fakeConfig.EXPECT().GetStringSlice("notify.urls").Return([]string{}).AnyTimes() - fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail) - fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -319,6 +329,8 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -330,8 +342,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"}) - fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail) - fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -361,6 +374,8 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -372,8 +387,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"}) - fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail) - fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -403,6 +419,8 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -414,8 +432,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///usr/bin/env"}) - fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail) - fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -445,6 +464,8 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -456,8 +477,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"}) - fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail) - fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -486,6 +508,8 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -497,8 +521,10 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{}) - fakeConfig.EXPECT().GetString("notify.level").AnyTimes().Return(pkg.NotifyLevelFail) - fakeConfig.EXPECT().GetString("notify.filter_attributes").AnyTimes().Return(pkg.NotifyFilterAttributesAll) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() From 54e2cacb006ebb895d0cfb781c10676f725cd99c Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 23 Jul 2022 09:32:56 -0700 Subject: [PATCH 08/28] move frontend settings into the DB (for consistent settings handling). Flattened settings object. --- webapp/backend/pkg/config/config.go | 2 +- .../migrations/m20220716214900/setting.go | 2 +- .../scrutiny_repository_migrations.go | 43 +++++++++++++++--- .../database/scrutiny_repository_settings.go | 10 ++--- webapp/backend/pkg/models/settings.go | 16 ++++--- .../pkg/web/handler/upload_device_metrics.go | 5 ++- webapp/backend/pkg/web/server_test.go | 44 ++++++++++--------- 7 files changed, 79 insertions(+), 43 deletions(-) diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index 3d689f3..3961373 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -9,7 +9,7 @@ import ( "strings" ) -const DBSETTING_SUBKEY = "dbsetting" +const DB_USER_SETTINGS_SUBKEY = "user" // When initializing this class the following methods must be called: // Config.New diff --git a/webapp/backend/pkg/database/migrations/m20220716214900/setting.go b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go index 8e38845..9c1f746 100644 --- a/webapp/backend/pkg/database/migrations/m20220716214900/setting.go +++ b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go @@ -12,6 +12,6 @@ type Setting struct { SettingKeyDescription string `json:"setting_key_description"` SettingDataType string `json:"setting_data_type"` - SettingValueNumeric int64 `json:"setting_value_numeric"` + SettingValueNumeric int `json:"setting_value_numeric"` SettingValueString string `json:"setting_value_string"` } diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 26f01fd..3be1071 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -290,22 +290,53 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { var defaultSettings = []m20220716214900.Setting{ { - SettingKeyName: "metrics.notify.level", + SettingKeyName: "theme", + SettingKeyDescription: "Frontend theme ('light' | 'dark' | 'system')", + SettingDataType: "string", + SettingValueString: "system", // options: 'light' | 'dark' | 'system' + }, + { + SettingKeyName: "layout", + SettingKeyDescription: "Frontend layout ('material')", + SettingDataType: "string", + SettingValueString: "material", + }, + { + SettingKeyName: "dashboardDisplay", + SettingKeyDescription: "Frontend device display title ('name' | 'serial_id' | 'uuid' | 'label')", + SettingDataType: "string", + SettingValueString: "name", + }, + { + SettingKeyName: "dashboardSort", + SettingKeyDescription: "Frontend device sort by ('status' | 'title' | 'age')", + SettingDataType: "string", + SettingValueString: "status", + }, + { + SettingKeyName: "temperatureUnit", + SettingKeyDescription: "Frontend temperature unit ('celsius' | 'fahrenheit')", + SettingDataType: "string", + SettingValueString: "celsius", + }, + + { + SettingKeyName: "metrics.notifyLevel", SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)", SettingDataType: "numeric", - SettingValueNumeric: int64(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn' + SettingValueNumeric: int(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn' }, { - SettingKeyName: "metrics.status.filter_attributes", + SettingKeyName: "metrics.statusFilterAttributes", SettingKeyDescription: "Determines which attributes should impact device status", SettingDataType: "numeric", - SettingValueNumeric: int64(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or 'critical' + SettingValueNumeric: int(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or 'critical' }, { - SettingKeyName: "metrics.status.threshold", + SettingKeyName: "metrics.statusThreshold", SettingKeyDescription: "Determines which threshold should impact device status", SettingDataType: "numeric", - SettingValueNumeric: int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both' + SettingValueNumeric: int(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both' }, } return tx.Create(&defaultSettings).Error diff --git a/webapp/backend/pkg/database/scrutiny_repository_settings.go b/webapp/backend/pkg/database/scrutiny_repository_settings.go index 329b375..7e874b3 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_settings.go +++ b/webapp/backend/pkg/database/scrutiny_repository_settings.go @@ -17,7 +17,7 @@ func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Setting // store retrieved settings in the AppConfig obj for _, settingsEntry := range settingsEntries { - configKey := fmt.Sprintf("%s.%s", config.DBSETTING_SUBKEY, settingsEntry.SettingKeyName) + configKey := fmt.Sprintf("%s.%s", config.DB_USER_SETTINGS_SUBKEY, settingsEntry.SettingKeyName) if settingsEntry.SettingDataType == "numeric" { sr.appConfig.Set(configKey, settingsEntry.SettingValueNumeric) @@ -28,7 +28,7 @@ func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Setting // unmarshal the dbsetting object data to a settings object. var settings models.Settings - err := sr.appConfig.UnmarshalKey(config.DBSETTING_SUBKEY, &settings) + err := sr.appConfig.UnmarshalKey(config.DB_USER_SETTINGS_SUBKEY, &settings) if err != nil { return nil, err } @@ -36,7 +36,7 @@ func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Setting } // testing -// curl -d '{"metrics": { "notify": { "level": 5 }, "status": { "filter_attributes": 5, "threshold": 5 } }}' -H "Content-Type: application/json" -X POST http://localhost:9090/api/settings +// curl -d '{"metrics": { "notifyLevel": 5, "statusFilterAttributes": 5, "statusThreshold": 5 }}' -H "Content-Type: application/json" -X POST http://localhost:9090/api/settings // SaveSettings will update settings in AppConfig object, then save the settings to the database. func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error { @@ -47,7 +47,7 @@ func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models. return err } settingsWrapperMap := map[string]interface{}{} - settingsWrapperMap[config.DBSETTING_SUBKEY] = *settingsMap + settingsWrapperMap[config.DB_USER_SETTINGS_SUBKEY] = *settingsMap err = sr.appConfig.MergeConfigMap(settingsWrapperMap) if err != nil { return err @@ -61,7 +61,7 @@ func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models. //update settingsEntries for ndx, settingsEntry := range settingsEntries { - configKey := fmt.Sprintf("%s.%s", config.DBSETTING_SUBKEY, settingsEntry.SettingKeyName) + configKey := fmt.Sprintf("%s.%s", config.DB_USER_SETTINGS_SUBKEY, settingsEntry.SettingKeyName) if settingsEntry.SettingDataType == "numeric" { settingsEntries[ndx].SettingValueNumeric = sr.appConfig.GetInt(configKey) diff --git a/webapp/backend/pkg/models/settings.go b/webapp/backend/pkg/models/settings.go index ec29681..49860f4 100644 --- a/webapp/backend/pkg/models/settings.go +++ b/webapp/backend/pkg/models/settings.go @@ -8,13 +8,15 @@ package models //} type Settings struct { + Theme string `json:"theme" mapstructure:"theme"` + Layout string `json:"layout" mapstructure:"layout"` + DashboardDisplay string `json:"dashboardDisplay" mapstructure:"dashboardDisplay"` + DashboardSort string `json:"dashboardSort" mapstructure:"dashboardSort"` + TemperatureUnit string `json:"temperatureUnit" mapstructure:"temperatureUnit"` + Metrics struct { - Notify struct { - Level int `json:"level" mapstructure:"level"` - } `json:"notify" mapstructure:"notify"` - Status struct { - FilterAttributes int `json:"filter_attributes" mapstructure:"filter_attributes"` - Threshold int `json:"threshold" mapstructure:"threshold"` - } `json:"status" mapstructure:"status"` + NotifyLevel int `json:"notifyLevel" mapstructure:"notifyLevel"` + StatusFilterAttributes int `json:"statusFilterAttributes" mapstructure:"statusFilterAttributes"` + StatusThreshold int `json:"statusThreshold" mapstructure:"statusThreshold"` } `json:"metrics" mapstructure:"metrics"` } diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index 0d4b74c..a80e4ed 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -1,6 +1,7 @@ package handler import ( + "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/database" @@ -70,8 +71,8 @@ func UploadDeviceMetrics(c *gin.Context) { if notify.ShouldNotify( updatedDevice, smartData, - pkg.MetricsStatusThreshold(appConfig.GetInt("dbsetting.metrics.status.threshold")), - pkg.MetricsStatusFilterAttributes(appConfig.GetInt("dbsetting.metrics.status.filter_attributes")), + pkg.MetricsStatusThreshold(appConfig.GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY))), + pkg.MetricsStatusFilterAttributes(appConfig.GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY))), ) { //send notifications diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index 28b78cc..f2c2d07 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -3,7 +3,9 @@ package web_test import ( "bytes" "encoding/json" + "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg" + "github.com/analogj/scrutiny/webapp/backend/pkg/config" mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" @@ -192,9 +194,9 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() { } else { fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes() } - fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) ae := web.AppEngine{ Config: fakeConfig, @@ -230,9 +232,9 @@ func (suite *ServerTestSuite) TestPopulateMultiple() { fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) //fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") fakeConfig.EXPECT().GetStringSlice("notify.urls").Return([]string{}).AnyTimes() - fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -342,9 +344,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"}) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -387,9 +389,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"}) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -432,9 +434,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///usr/bin/env"}) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -477,9 +479,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"}) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -521,9 +523,9 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{}) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.notify.level").AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.filter_attributes").AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt("dbsetting.metrics.status.threshold").AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. From 7e672e8b8ed2a7a6056cc144421ce17ff5d986ed Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 23 Jul 2022 10:19:15 -0700 Subject: [PATCH 09/28] adding tests for config.MergeConfigMap functionality. (Set vs SetDefault). Converted all settings keys to snakecase. --- webapp/backend/pkg/config/config_test.go | 34 +++++++++++++++ webapp/backend/pkg/constants.go | 6 --- .../scrutiny_repository_migrations.go | 12 +++--- .../database/scrutiny_repository_settings.go | 12 +++--- webapp/backend/pkg/models/settings.go | 12 +++--- .../pkg/web/handler/upload_device_metrics.go | 4 +- webapp/backend/pkg/web/server_test.go | 42 +++++++++---------- 7 files changed, 75 insertions(+), 47 deletions(-) create mode 100644 webapp/backend/pkg/config/config_test.go diff --git a/webapp/backend/pkg/config/config_test.go b/webapp/backend/pkg/config/config_test.go new file mode 100644 index 0000000..f734d50 --- /dev/null +++ b/webapp/backend/pkg/config/config_test.go @@ -0,0 +1,34 @@ +package config + +import ( + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + "testing" +) + +func Test_MergeConfigMap(t *testing.T) { + //setup + testConfig := configuration{ + Viper: viper.New(), + } + testConfig.Set("user.dashboard_display", "hello") + testConfig.SetDefault("user.layout", "hello") + + mergeSettings := map[string]interface{}{ + "user": map[string]interface{}{ + "dashboard_display": "dashboard_display", + "layout": "layout", + }, + } + //test + err := testConfig.MergeConfigMap(mergeSettings) + + //verify + require.NoError(t, err) + + // if using Set, the MergeConfigMap functionality will not override + // if using SetDefault, the MergeConfigMap will override correctly + require.Equal(t, "hello", testConfig.GetString("user.dashboard_display")) + require.Equal(t, "layout", testConfig.GetString("user.layout")) + +} diff --git a/webapp/backend/pkg/constants.go b/webapp/backend/pkg/constants.go index d05bb15..a82c9c3 100644 --- a/webapp/backend/pkg/constants.go +++ b/webapp/backend/pkg/constants.go @@ -63,9 +63,3 @@ const ( //shortcut MetricsStatusThresholdBoth MetricsStatusThreshold = 3 ) - -// Deprecated -const NotifyFilterAttributesAll = "all" - -// Deprecated -const NotifyLevelFail = "fail" diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 3be1071..a6f1b68 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -302,38 +302,38 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { SettingValueString: "material", }, { - SettingKeyName: "dashboardDisplay", + SettingKeyName: "dashboard_display", SettingKeyDescription: "Frontend device display title ('name' | 'serial_id' | 'uuid' | 'label')", SettingDataType: "string", SettingValueString: "name", }, { - SettingKeyName: "dashboardSort", + SettingKeyName: "dashboard_sort", SettingKeyDescription: "Frontend device sort by ('status' | 'title' | 'age')", SettingDataType: "string", SettingValueString: "status", }, { - SettingKeyName: "temperatureUnit", + SettingKeyName: "temperature_unit", SettingKeyDescription: "Frontend temperature unit ('celsius' | 'fahrenheit')", SettingDataType: "string", SettingValueString: "celsius", }, { - SettingKeyName: "metrics.notifyLevel", + SettingKeyName: "metrics.notify_level", SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)", SettingDataType: "numeric", SettingValueNumeric: int(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn' }, { - SettingKeyName: "metrics.statusFilterAttributes", + SettingKeyName: "metrics.status_filter_attributes", SettingKeyDescription: "Determines which attributes should impact device status", SettingDataType: "numeric", SettingValueNumeric: int(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or 'critical' }, { - SettingKeyName: "metrics.statusThreshold", + SettingKeyName: "metrics.status_threshold", SettingKeyDescription: "Determines which threshold should impact device status", SettingDataType: "numeric", SettingValueNumeric: int(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both' diff --git a/webapp/backend/pkg/database/scrutiny_repository_settings.go b/webapp/backend/pkg/database/scrutiny_repository_settings.go index 7e874b3..918a9f4 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_settings.go +++ b/webapp/backend/pkg/database/scrutiny_repository_settings.go @@ -6,6 +6,7 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/mitchellh/mapstructure" + "strings" ) // LoadSettings will retrieve settings from the database, store them in the AppConfig object, and return a Settings struct @@ -20,9 +21,9 @@ func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Setting configKey := fmt.Sprintf("%s.%s", config.DB_USER_SETTINGS_SUBKEY, settingsEntry.SettingKeyName) if settingsEntry.SettingDataType == "numeric" { - sr.appConfig.Set(configKey, settingsEntry.SettingValueNumeric) + sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueNumeric) } else if settingsEntry.SettingDataType == "string" { - sr.appConfig.Set(configKey, settingsEntry.SettingValueString) + sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueString) } } @@ -36,10 +37,9 @@ func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Setting } // testing -// curl -d '{"metrics": { "notifyLevel": 5, "statusFilterAttributes": 5, "statusThreshold": 5 }}' -H "Content-Type: application/json" -X POST http://localhost:9090/api/settings +// curl -d '{"metrics": { "notify_level": 5, "status_filter_attributes": 5, "status_threshold": 5 }}' -H "Content-Type: application/json" -X POST http://localhost:9090/api/settings // SaveSettings will update settings in AppConfig object, then save the settings to the database. func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error { - //save the entries to the appconfig settingsMap := &map[string]interface{}{} err := mapstructure.Decode(settings, &settingsMap) @@ -52,7 +52,7 @@ func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models. if err != nil { return err } - + sr.logger.Debugf("after merge settings: %v", sr.appConfig.AllSettings()) //retrieve current settings from the database settingsEntries := []models.SettingEntry{} if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil { @@ -61,7 +61,7 @@ func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models. //update settingsEntries for ndx, settingsEntry := range settingsEntries { - configKey := fmt.Sprintf("%s.%s", config.DB_USER_SETTINGS_SUBKEY, settingsEntry.SettingKeyName) + configKey := fmt.Sprintf("%s.%s", config.DB_USER_SETTINGS_SUBKEY, strings.ToLower(settingsEntry.SettingKeyName)) if settingsEntry.SettingDataType == "numeric" { settingsEntries[ndx].SettingValueNumeric = sr.appConfig.GetInt(configKey) diff --git a/webapp/backend/pkg/models/settings.go b/webapp/backend/pkg/models/settings.go index 49860f4..48ba2d5 100644 --- a/webapp/backend/pkg/models/settings.go +++ b/webapp/backend/pkg/models/settings.go @@ -10,13 +10,13 @@ package models type Settings struct { Theme string `json:"theme" mapstructure:"theme"` Layout string `json:"layout" mapstructure:"layout"` - DashboardDisplay string `json:"dashboardDisplay" mapstructure:"dashboardDisplay"` - DashboardSort string `json:"dashboardSort" mapstructure:"dashboardSort"` - TemperatureUnit string `json:"temperatureUnit" mapstructure:"temperatureUnit"` + DashboardDisplay string `json:"dashboard_display" mapstructure:"dashboard_display"` + DashboardSort string `json:"dashboard_sort" mapstructure:"dashboard_sort"` + TemperatureUnit string `json:"temperature_unit" mapstructure:"temperature_unit"` Metrics struct { - NotifyLevel int `json:"notifyLevel" mapstructure:"notifyLevel"` - StatusFilterAttributes int `json:"statusFilterAttributes" mapstructure:"statusFilterAttributes"` - StatusThreshold int `json:"statusThreshold" mapstructure:"statusThreshold"` + NotifyLevel int `json:"notify_level" mapstructure:"notify_level"` + StatusFilterAttributes int `json:"status_filter_attributes" mapstructure:"status_filter_attributes"` + StatusThreshold int `json:"status_threshold" mapstructure:"status_threshold"` } `json:"metrics" mapstructure:"metrics"` } diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index a80e4ed..82d5850 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -71,8 +71,8 @@ func UploadDeviceMetrics(c *gin.Context) { if notify.ShouldNotify( updatedDevice, smartData, - pkg.MetricsStatusThreshold(appConfig.GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY))), - pkg.MetricsStatusFilterAttributes(appConfig.GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY))), + pkg.MetricsStatusThreshold(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY))), + pkg.MetricsStatusFilterAttributes(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY))), ) { //send notifications diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index f2c2d07..83d345b 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -194,9 +194,9 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() { } else { fakeConfig.EXPECT().GetString("web.influxdb.host").Return("localhost").AnyTimes() } - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) ae := web.AppEngine{ Config: fakeConfig, @@ -232,9 +232,9 @@ func (suite *ServerTestSuite) TestPopulateMultiple() { fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) //fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") fakeConfig.EXPECT().GetStringSlice("notify.urls").Return([]string{}).AnyTimes() - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetString("web.listen.basepath").Return(suite.Basepath).AnyTimes() @@ -344,9 +344,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"}) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -389,9 +389,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"}) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -434,9 +434,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///usr/bin/env"}) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -479,9 +479,9 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"}) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. @@ -523,9 +523,9 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{}) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notifyLevel", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusFilterAttributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) - fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.statusThreshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) + fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusThresholdBoth)) if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. From 94594db20a674c55f1ba794b3e23f45545c6e238 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 23 Jul 2022 11:43:12 -0700 Subject: [PATCH 10/28] on settings save, return the new settings. update the frontend to persist settings to the database. Using ScrutinyConfigService instead of TreoConfigService. Using snake case settings in frontend. Make sure we're using AppConfig type where possible. --- .../backend/pkg/web/handler/save_settings.go | 3 +- webapp/backend/pkg/web/server_test.go | 18 ++-- webapp/frontend/src/app/app.module.ts | 34 ++++---- .../src/app/core/config/app.config.ts | 51 +++++++++-- .../app/core/config/scrutiny-config.module.ts | 33 ++++++++ .../core/config/scrutiny-config.service.ts | 84 +++++++++++++++++++ .../dashboard-device.component.html | 5 +- .../dashboard-device.component.spec.ts | 6 +- .../dashboard-device.component.ts | 6 +- .../dashboard-settings.component.ts | 22 ++--- .../src/app/layout/layout.component.ts | 44 +++++----- .../dashboard/dashboard.component.html | 6 +- .../modules/dashboard/dashboard.component.ts | 12 +-- .../app/modules/detail/detail.component.html | 4 +- .../app/modules/detail/detail.component.ts | 6 +- 15 files changed, 245 insertions(+), 89 deletions(-) create mode 100644 webapp/frontend/src/app/core/config/scrutiny-config.module.ts create mode 100644 webapp/frontend/src/app/core/config/scrutiny-config.service.ts diff --git a/webapp/backend/pkg/web/handler/save_settings.go b/webapp/backend/pkg/web/handler/save_settings.go index d466169..16de020 100644 --- a/webapp/backend/pkg/web/handler/save_settings.go +++ b/webapp/backend/pkg/web/handler/save_settings.go @@ -28,6 +28,7 @@ func SaveSettings(c *gin.Context) { } c.JSON(http.StatusOK, gin.H{ - "success": true, + "success": true, + "settings": settings, }) } diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index 83d345b..beea762 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -91,7 +91,7 @@ func (suite *ServerTestSuite) TestHealthRoute() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes() fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes() @@ -134,7 +134,7 @@ func (suite *ServerTestSuite) TestRegisterDevicesRoute() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes() fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes() @@ -176,7 +176,7 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) @@ -228,7 +228,7 @@ func (suite *ServerTestSuite) TestPopulateMultiple() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) //fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") fakeConfig.EXPECT().GetStringSlice("notify.urls").Return([]string{}).AnyTimes() @@ -331,7 +331,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) @@ -376,7 +376,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) @@ -421,7 +421,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) @@ -466,7 +466,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) @@ -510,7 +510,7 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() { mockCtrl := gomock.NewController(suite.T()) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().Set(gomock.Any(), gomock.Any()).AnyTimes() + fakeConfig.EXPECT().SetDefault(gomock.Any(), gomock.Any()).AnyTimes() fakeConfig.EXPECT().UnmarshalKey(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) diff --git a/webapp/frontend/src/app/app.module.ts b/webapp/frontend/src/app/app.module.ts index 904ee15..8d0d060 100644 --- a/webapp/frontend/src/app/app.module.ts +++ b/webapp/frontend/src/app/app.module.ts @@ -1,22 +1,22 @@ -import { NgModule, enableProdMode } from '@angular/core'; -import { BrowserModule } from '@angular/platform-browser'; -import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; -import { ExtraOptions, PreloadAllModules, RouterModule } from '@angular/router'; -import { APP_BASE_HREF } from '@angular/common'; -import { MarkdownModule } from 'ngx-markdown'; -import { TreoModule } from '@treo'; -import { TreoConfigModule } from '@treo/services/config'; -import { TreoMockApiModule } from '@treo/lib/mock-api'; -import { CoreModule } from 'app/core/core.module'; -import { appConfig } from 'app/core/config/app.config'; -import { mockDataServices } from 'app/data/mock'; -import { LayoutModule } from 'app/layout/layout.module'; -import { AppComponent } from 'app/app.component'; -import { appRoutes, getAppBaseHref } from 'app/app.routing'; +import {enableProdMode, NgModule} from '@angular/core'; +import {BrowserModule} from '@angular/platform-browser'; +import {BrowserAnimationsModule} from '@angular/platform-browser/animations'; +import {ExtraOptions, PreloadAllModules, RouterModule} from '@angular/router'; +import {APP_BASE_HREF} from '@angular/common'; +import {MarkdownModule} from 'ngx-markdown'; +import {TreoModule} from '@treo'; +import {ScrutinyConfigModule} from 'app/core/config/scrutiny-config.module'; +import {TreoMockApiModule} from '@treo/lib/mock-api'; +import {CoreModule} from 'app/core/core.module'; +import {appConfig} from 'app/core/config/app.config'; +import {mockDataServices} from 'app/data/mock'; +import {LayoutModule} from 'app/layout/layout.module'; +import {AppComponent} from 'app/app.component'; +import {appRoutes, getAppBaseHref} from 'app/app.routing'; const routerConfig: ExtraOptions = { scrollPositionRestoration: 'enabled', - preloadingStrategy : PreloadAllModules + preloadingStrategy: PreloadAllModules }; let dev = [ @@ -41,7 +41,7 @@ if (process.env.NODE_ENV === 'production') { // Treo & Treo Mock API TreoModule, - TreoConfigModule.forRoot(appConfig), + ScrutinyConfigModule.forRoot(appConfig), ...dev, // Core diff --git a/webapp/frontend/src/app/core/config/app.config.ts b/webapp/frontend/src/app/core/config/app.config.ts index 74143c5..f26dc01 100644 --- a/webapp/frontend/src/app/core/config/app.config.ts +++ b/webapp/frontend/src/app/core/config/app.config.ts @@ -10,19 +10,47 @@ export type DashboardSort = 'status' | 'title' | 'age' export type TemperatureUnit = 'celsius' | 'fahrenheit' + +enum MetricsNotifyLevel { + Warn = 1, + Fail = 2 +} + +enum MetricsStatusFilterAttributes { + All = 0, + Critical = 1 +} + +enum MetricsStatusThreshold { + Smart = 1, + Scrutiny = 2, + + // shortcut + Both = 3 +} + /** * AppConfig interface. Update this interface to strictly type your config * object. */ export interface AppConfig { - theme: Theme; - layout: Layout; + theme?: Theme; + layout?: Layout; // Dashboard options - dashboardDisplay: DashboardDisplay; - dashboardSort: DashboardSort; + dashboard_display?: DashboardDisplay; + dashboard_sort?: DashboardSort; + + temperature_unit?: TemperatureUnit; + + // Settings from Scrutiny API + + metrics?: { + notify_level?: MetricsNotifyLevel + status_filter_attributes?: MetricsStatusFilterAttributes + status_threshold?: MetricsStatusThreshold + } - temperatureUnit: TemperatureUnit; } /** @@ -34,12 +62,17 @@ export interface AppConfig { * "ConfigService". */ export const appConfig: AppConfig = { - theme : 'light', + theme: 'light', layout: 'material', - dashboardDisplay: 'name', - dashboardSort: 'status', + dashboard_display: 'name', + dashboard_sort: 'status', - temperatureUnit: 'celsius', + temperature_unit: 'celsius', + metrics: { + notify_level: MetricsNotifyLevel.Fail, + status_filter_attributes: MetricsStatusFilterAttributes.All, + status_threshold: MetricsStatusThreshold.Both + } }; diff --git a/webapp/frontend/src/app/core/config/scrutiny-config.module.ts b/webapp/frontend/src/app/core/config/scrutiny-config.module.ts new file mode 100644 index 0000000..3dd5bc8 --- /dev/null +++ b/webapp/frontend/src/app/core/config/scrutiny-config.module.ts @@ -0,0 +1,33 @@ +import {ModuleWithProviders, NgModule} from '@angular/core'; +import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service'; +import {TREO_APP_CONFIG} from '@treo/services/config/config.constants'; + +@NgModule() +export class ScrutinyConfigModule { + /** + * Constructor + * + * @param {ScrutinyConfigService} _scrutinyConfigService + */ + constructor( + private _scrutinyConfigService: ScrutinyConfigService + ) { + } + + /** + * forRoot method for setting user configuration + * + * @param config + */ + static forRoot(config: any): ModuleWithProviders { + return { + ngModule: ScrutinyConfigModule, + providers: [ + { + provide: TREO_APP_CONFIG, + useValue: config + } + ] + }; + } +} diff --git a/webapp/frontend/src/app/core/config/scrutiny-config.service.ts b/webapp/frontend/src/app/core/config/scrutiny-config.service.ts new file mode 100644 index 0000000..4c6d7b9 --- /dev/null +++ b/webapp/frontend/src/app/core/config/scrutiny-config.service.ts @@ -0,0 +1,84 @@ +import {Inject, Injectable} from '@angular/core'; +import {HttpClient} from '@angular/common/http'; +import {TREO_APP_CONFIG} from '@treo/services/config/config.constants'; +import {BehaviorSubject, Observable} from 'rxjs'; +import {getBasePath} from '../../app.routing'; +import {map, tap} from 'rxjs/operators'; +import {AppConfig} from './app.config'; +import {merge} from 'lodash'; + +@Injectable({ + providedIn: 'root' +}) +export class ScrutinyConfigService { + // Private + private _config: BehaviorSubject; + private _defaultConfig: AppConfig; + + constructor( + private _httpClient: HttpClient, + @Inject(TREO_APP_CONFIG) defaultConfig: AppConfig + ) { + // Set the private defaults + this._defaultConfig = defaultConfig + this._config = new BehaviorSubject(null); + } + + + // ----------------------------------------------------------------------------------------------------- + // @ Accessors + // ----------------------------------------------------------------------------------------------------- + + /** + * Setter & getter for config + */ + set config(value: AppConfig) { + // get the current config, merge the new values, and then submit. (setTheme only sets a single key, not the whole obj) + const mergedSettings = merge({}, this._config.getValue(), value); + + console.log('saving settings...', mergedSettings) + this._httpClient.post(getBasePath() + '/api/settings', mergedSettings).pipe( + map((response: any) => { + console.log('settings resp') + return response.settings + }), + tap((settings: AppConfig) => { + this._config.next(settings); + return settings + }) + ).subscribe(resp => { + console.log('updated settings', resp) + }) + } + + get config$(): Observable { + if (this._config.getValue()) { + console.log('using cached settings:', this._config.getValue()) + return this._config.asObservable() + } else { + console.log('retrieving settings') + return this._httpClient.get(getBasePath() + '/api/settings').pipe( + map((response: any) => { + return response.settings + }), + tap((settings: AppConfig) => { + this._config.next(settings); + return this._config.asObservable() + }) + ); + } + + } + + // ----------------------------------------------------------------------------------------------------- + // @ Public methods + // ----------------------------------------------------------------------------------------------------- + + /** + * Resets the config to the default + */ + reset(): void { + // Set the config + this.config = this._defaultConfig + } +} diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html index e7b0ffd..774df02 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html @@ -15,7 +15,7 @@
{{deviceSummary.device | deviceTitle:config.dashboardDisplay}} + class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceSummary.device | deviceTitle:config.dashboard_display}}
Last Updated on {{deviceSummary.smart.collector_date | date:'MMMM dd, yyyy - HH:mm' }}
@@ -51,7 +51,8 @@
Temperature
-
{{ deviceSummary.smart?.temp | temperature:config.temperatureUnit:true }}
+
{{ deviceSummary.smart?.temp | temperature:config.temperature_unit:true }}
--
diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts index 7b334bb..2c21c89 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts @@ -9,13 +9,14 @@ import {MatMenuModule} from '@angular/material/menu'; import {TREO_APP_CONFIG} from '@treo/services/config/config.constants'; import {DeviceSummaryModel} from 'app/core/models/device-summary-model'; import * as moment from 'moment'; +import {HttpClientTestingModule} from '@angular/common/http/testing'; describe('DashboardDeviceComponent', () => { let component: DashboardDeviceComponent; let fixture: ComponentFixture; const matDialogSpy = jasmine.createSpyObj('MatDialog', ['open']); - // const configServiceSpy = jasmine.createSpyObj('TreoConfigService', ['config$']); + // const configServiceSpy = jasmine.createSpyObj('ScrutinyConfigService', ['config$']); beforeEach(async(() => { @@ -25,10 +26,11 @@ describe('DashboardDeviceComponent', () => { MatIconModule, MatMenuModule, SharedModule, + HttpClientTestingModule, ], providers: [ {provide: MatDialog, useValue: matDialogSpy}, - {provide: TREO_APP_CONFIG, useValue: {dashboardDisplay: 'name'}} + {provide: TREO_APP_CONFIG, useValue: {dashboard_display: 'name'}} ], declarations: [DashboardDeviceComponent] }) diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts index 4fb7d7a..6262c4f 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts @@ -2,7 +2,7 @@ import {Component, EventEmitter, Input, OnInit, Output} from '@angular/core'; import * as moment from 'moment'; import {takeUntil} from 'rxjs/operators'; import {AppConfig} from 'app/core/config/app.config'; -import {TreoConfigService} from '@treo/services/config'; +import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service'; import {Subject} from 'rxjs'; import humanizeDuration from 'humanize-duration' import {MatDialog} from '@angular/material/dialog'; @@ -18,7 +18,7 @@ import {DeviceSummaryModel} from 'app/core/models/device-summary-model'; export class DashboardDeviceComponent implements OnInit { constructor( - private _configService: TreoConfigService, + private _configService: ScrutinyConfigService, public dialog: MatDialog, ) { // Set the private defaults @@ -82,7 +82,7 @@ export class DashboardDeviceComponent implements OnInit { // width: '250px', data: { wwn: this.deviceWWN, - title: DeviceTitlePipe.deviceTitleWithFallback(this.deviceSummary.device, this.config.dashboardDisplay) + title: DeviceTitlePipe.deviceTitleWithFallback(this.deviceSummary.device, this.config.dashboard_display) } }); diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts index 70a0978..39110d8 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts @@ -1,6 +1,6 @@ import {Component, OnInit} from '@angular/core'; -import {AppConfig} from 'app/core/config/app.config'; -import {TreoConfigService} from '@treo/services/config'; +import {AppConfig, DashboardDisplay, DashboardSort, TemperatureUnit, Theme} from 'app/core/config/app.config'; +import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service'; import {Subject} from 'rxjs'; import {takeUntil} from 'rxjs/operators'; @@ -20,7 +20,7 @@ export class DashboardSettingsComponent implements OnInit { private _unsubscribeAll: Subject; constructor( - private _configService: TreoConfigService, + private _configService: ScrutinyConfigService, ) { // Set the private defaults this._unsubscribeAll = new Subject(); @@ -33,9 +33,9 @@ export class DashboardSettingsComponent implements OnInit { .subscribe((config: AppConfig) => { // Store the config - this.dashboardDisplay = config.dashboardDisplay; - this.dashboardSort = config.dashboardSort; - this.temperatureUnit = config.temperatureUnit; + this.dashboardDisplay = config.dashboard_display; + this.dashboardSort = config.dashboard_sort; + this.temperatureUnit = config.temperature_unit; this.theme = config.theme; }); @@ -43,11 +43,11 @@ export class DashboardSettingsComponent implements OnInit { } saveSettings(): void { - const newSettings = { - dashboardDisplay: this.dashboardDisplay, - dashboardSort: this.dashboardSort, - temperatureUnit: this.temperatureUnit, - theme: this.theme + const newSettings: AppConfig = { + dashboard_display: this.dashboardDisplay as DashboardDisplay, + dashboard_sort: this.dashboardSort as DashboardSort, + temperature_unit: this.temperatureUnit as TemperatureUnit, + theme: this.theme as Theme } this._configService.config = newSettings console.log(`Saved Settings: ${JSON.stringify(newSettings)}`) diff --git a/webapp/frontend/src/app/layout/layout.component.ts b/webapp/frontend/src/app/layout/layout.component.ts index 6a3a68b..8e567a6 100644 --- a/webapp/frontend/src/app/layout/layout.component.ts +++ b/webapp/frontend/src/app/layout/layout.component.ts @@ -1,22 +1,21 @@ -import { Component, Inject, OnDestroy, OnInit, ViewEncapsulation } from '@angular/core'; -import { DOCUMENT } from '@angular/common'; -import { ActivatedRoute, NavigationEnd, Router } from '@angular/router'; -import { MatSlideToggleChange } from '@angular/material/slide-toggle'; -import { Subject } from 'rxjs'; -import { filter, takeUntil } from 'rxjs/operators'; -import { TreoConfigService } from '@treo/services/config'; -import { TreoDrawerService } from '@treo/components/drawer'; -import { Layout } from 'app/layout/layout.types'; -import { AppConfig, Theme } from 'app/core/config/app.config'; +import {Component, Inject, OnDestroy, OnInit, ViewEncapsulation} from '@angular/core'; +import {DOCUMENT} from '@angular/common'; +import {ActivatedRoute, NavigationEnd, Router} from '@angular/router'; +import {MatSlideToggleChange} from '@angular/material/slide-toggle'; +import {Subject} from 'rxjs'; +import {filter, takeUntil} from 'rxjs/operators'; +import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service'; +import {TreoDrawerService} from '@treo/components/drawer'; +import {Layout} from 'app/layout/layout.types'; +import {AppConfig, Theme} from 'app/core/config/app.config'; @Component({ - selector : 'layout', - templateUrl : './layout.component.html', - styleUrls : ['./layout.component.scss'], + selector: 'layout', + templateUrl: './layout.component.html', + styleUrls: ['./layout.component.scss'], encapsulation: ViewEncapsulation.None }) -export class LayoutComponent implements OnInit, OnDestroy -{ +export class LayoutComponent implements OnInit, OnDestroy { config: AppConfig; layout: Layout; theme: Theme; @@ -29,14 +28,14 @@ export class LayoutComponent implements OnInit, OnDestroy * Constructor * * @param {ActivatedRoute} _activatedRoute - * @param {TreoConfigService} _treoConfigService + * @param {ScrutinyConfigService} _scrutinyConfigService * @param {TreoDrawerService} _treoDrawerService * @param {DOCUMENT} _document * @param {Router} _router */ constructor( private _activatedRoute: ActivatedRoute, - private _treoConfigService: TreoConfigService, + private _scrutinyConfigService: ScrutinyConfigService, private _treoDrawerService: TreoDrawerService, @Inject(DOCUMENT) private _document: any, private _router: Router @@ -59,7 +58,7 @@ export class LayoutComponent implements OnInit, OnDestroy ngOnInit(): void { // Subscribe to config changes - this._treoConfigService.config$ + this._scrutinyConfigService.config$ .pipe(takeUntil(this._unsubscribeAll)) .subscribe((config: AppConfig) => { @@ -180,18 +179,17 @@ export class LayoutComponent implements OnInit, OnDestroy * * @param layout */ - setLayout(layout: string): void - { + setLayout(layout: Layout): void { // Clear the 'layout' query param to allow layout changes this._router.navigate([], { - queryParams : { + queryParams: { layout: null }, queryParamsHandling: 'merge' }).then(() => { // Set the config - this._treoConfigService.config = {layout}; + this._scrutinyConfigService.config = {layout}; }); } @@ -202,6 +200,6 @@ export class LayoutComponent implements OnInit, OnDestroy */ setTheme(change: MatSlideToggleChange): void { - this._treoConfigService.config = {theme: change.checked ? 'dark' : 'light'}; + this._scrutinyConfigService.config = {theme: change.checked ? 'dark' : 'light'}; } } diff --git a/webapp/frontend/src/app/modules/dashboard/dashboard.component.html b/webapp/frontend/src/app/modules/dashboard/dashboard.component.html index d370ab5..f106131 100644 --- a/webapp/frontend/src/app/modules/dashboard/dashboard.component.html +++ b/webapp/frontend/src/app/modules/dashboard/dashboard.component.html @@ -51,7 +51,11 @@

{{hostId.key}}

- +
diff --git a/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts b/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts index 7352e98..f790891 100644 --- a/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts +++ b/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts @@ -14,7 +14,7 @@ import {DashboardService} from 'app/modules/dashboard/dashboard.service'; import {MatDialog} from '@angular/material/dialog'; import {DashboardSettingsComponent} from 'app/layout/common/dashboard-settings/dashboard-settings.component'; import {AppConfig} from 'app/core/config/app.config'; -import {TreoConfigService} from '@treo/services/config'; +import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service'; import {Router} from '@angular/router'; import {TemperaturePipe} from 'app/shared/temperature.pipe'; import {DeviceTitlePipe} from 'app/shared/device-title.pipe'; @@ -43,13 +43,13 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy * Constructor * * @param {DashboardService} _dashboardService - * @param {TreoConfigService} _configService + * @param {ScrutinyConfigService} _configService * @param {MatDialog} dialog * @param {Router} router */ constructor( private _dashboardService: DashboardService, - private _configService: TreoConfigService, + private _configService: ScrutinyConfigService, public dialog: MatDialog, private router: Router, ) @@ -150,7 +150,7 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy continue } - const deviceName = DeviceTitlePipe.deviceTitleWithFallback(deviceSummary.device, this.config.dashboardDisplay) + const deviceName = DeviceTitlePipe.deviceTitleWithFallback(deviceSummary.device, this.config.dashboard_display) const deviceSeriesMetadata = { name: deviceName, @@ -161,7 +161,7 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy const newDate = new Date(tempHistory.date); deviceSeriesMetadata.data.push({ x: newDate, - y: TemperaturePipe.formatTemperature(tempHistory.temp, this.config.temperatureUnit, false) + y: TemperaturePipe.formatTemperature(tempHistory.temp, this.config.temperature_unit, false) }) } deviceTemperatureSeries.push(deviceSeriesMetadata) @@ -212,7 +212,7 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy y : { formatter: (value) => { - return TemperaturePipe.formatTemperature(value, this.config.temperatureUnit, true) as string; + return TemperaturePipe.formatTemperature(value, this.config.temperature_unit, true) as string; } } }, diff --git a/webapp/frontend/src/app/modules/detail/detail.component.html b/webapp/frontend/src/app/modules/detail/detail.component.html index da8ad7a..e96a493 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.html +++ b/webapp/frontend/src/app/modules/detail/detail.component.html @@ -4,7 +4,7 @@
-

Drive Details - {{device | deviceTitle:config.dashboardDisplay}}

+

Drive Details - {{device | deviceTitle:config.dashboard_display}}

Dive into S.M.A.R.T data
@@ -126,7 +126,7 @@
Powered On
-
{{smart_results[0]?.temp | temperature:config.temperatureUnit:true}}
+
{{smart_results[0]?.temp | temperature:config.temperature_unit:true}}
Temperature
diff --git a/webapp/frontend/src/app/modules/detail/detail.component.ts b/webapp/frontend/src/app/modules/detail/detail.component.ts index 0e29652..1353c51 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.ts +++ b/webapp/frontend/src/app/modules/detail/detail.component.ts @@ -8,7 +8,7 @@ import {MatDialog} from '@angular/material/dialog'; import {MatSort} from '@angular/material/sort'; import {MatTableDataSource} from '@angular/material/table'; import {Subject} from 'rxjs'; -import {TreoConfigService} from '@treo/services/config'; +import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service'; import {animate, state, style, transition, trigger} from '@angular/animations'; import {formatDate} from '@angular/common'; import {takeUntil} from 'rxjs/operators'; @@ -44,13 +44,13 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy { * * @param {DetailService} _detailService * @param {MatDialog} dialog - * @param {TreoConfigService} _configService + * @param {ScrutinyConfigService} _configService * @param {string} locale */ constructor( private _detailService: DetailService, public dialog: MatDialog, - private _configService: TreoConfigService, + private _configService: ScrutinyConfigService, @Inject(LOCALE_ID) public locale: string ) { // Set the private defaults From 7a68a68e76f4e690e8e2f26eef604fa25ddbd390 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 23 Jul 2022 16:11:49 -0700 Subject: [PATCH 11/28] frontend, determine the device status by checking against the configured thresholds. --- .../src/app/core/config/app.config.ts | 6 +- .../dashboard-device.component.html | 13 ++-- .../dashboard-device.component.ts | 10 ++- .../dashboard-settings.component.html | 72 +++++-------------- .../dashboard-settings.component.ts | 21 +++++- 5 files changed, 56 insertions(+), 66 deletions(-) diff --git a/webapp/frontend/src/app/core/config/app.config.ts b/webapp/frontend/src/app/core/config/app.config.ts index f26dc01..b4a6114 100644 --- a/webapp/frontend/src/app/core/config/app.config.ts +++ b/webapp/frontend/src/app/core/config/app.config.ts @@ -11,17 +11,17 @@ export type DashboardSort = 'status' | 'title' | 'age' export type TemperatureUnit = 'celsius' | 'fahrenheit' -enum MetricsNotifyLevel { +export enum MetricsNotifyLevel { Warn = 1, Fail = 2 } -enum MetricsStatusFilterAttributes { +export enum MetricsStatusFilterAttributes { All = 0, Critical = 1 } -enum MetricsStatusThreshold { +export enum MetricsStatusThreshold { Smart = 1, Scrutiny = 2, diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html index 774df02..bd7b4a1 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html @@ -1,15 +1,15 @@ -
@@ -46,7 +46,8 @@
Status
-
{{ deviceStatusString(deviceSummary.device.device_status) | titlecase}}
+
{{ deviceStatusString(deviceSummary) | titlecase}}
No Data
diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts index 6262c4f..a8de9d5 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts @@ -68,7 +68,15 @@ export class DashboardDeviceComponent implements OnInit { } } - deviceStatusString(deviceStatus: number): string { + deviceStatusString(deviceSummary: DeviceSummaryModel): string { + // no smart data, so treat the device status as unknown + if (!deviceSummary.smart) { + return 'unknown' + } + + // determine the device status, by comparing it against the allowed threshold + // tslint:disable-next-line:no-bitwise + const deviceStatus = deviceSummary.device.device_status & this.config.metrics.status_threshold if (deviceStatus === 0) { return 'passed' } else { diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html index a10d550..06cf800 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html @@ -43,63 +43,27 @@ Fahrenheit -
-
- - +
+ + Device Status - Filter Attributes + + All + Critical + + +
-
- - Critical Error Threshold - - - - Critical Warning Threshold - - -
- -
- - Error Threshold - - - - Warning Threshold - - -
- - - - -
- - Critical Error Threshold - - - - Critical Warning Threshold - - -
- - - -
- - Critical Error Threshold - - - - Critical Warning Threshold - - -
- - +
+ + Device Status - Thresholds + + Smart + Scrutiny + Both + +
diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts index 39110d8..21e8e8e 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts @@ -1,5 +1,13 @@ import {Component, OnInit} from '@angular/core'; -import {AppConfig, DashboardDisplay, DashboardSort, TemperatureUnit, Theme} from 'app/core/config/app.config'; +import { + AppConfig, + DashboardDisplay, + DashboardSort, + MetricsStatusFilterAttributes, + MetricsStatusThreshold, + TemperatureUnit, + Theme +} from 'app/core/config/app.config'; import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service'; import {Subject} from 'rxjs'; import {takeUntil} from 'rxjs/operators'; @@ -15,6 +23,8 @@ export class DashboardSettingsComponent implements OnInit { dashboardSort: string; temperatureUnit: string; theme: string; + statusThreshold: number; + statusFilterAttributes: number; // Private private _unsubscribeAll: Subject; @@ -38,6 +48,9 @@ export class DashboardSettingsComponent implements OnInit { this.temperatureUnit = config.temperature_unit; this.theme = config.theme; + this.statusFilterAttributes = config.metrics.status_filter_attributes; + this.statusThreshold = config.metrics.status_threshold; + }); } @@ -47,7 +60,11 @@ export class DashboardSettingsComponent implements OnInit { dashboard_display: this.dashboardDisplay as DashboardDisplay, dashboard_sort: this.dashboardSort as DashboardSort, temperature_unit: this.temperatureUnit as TemperatureUnit, - theme: this.theme as Theme + theme: this.theme as Theme, + metrics: { + status_filter_attributes: this.statusFilterAttributes as MetricsStatusFilterAttributes, + status_threshold: this.statusThreshold as MetricsStatusThreshold + } } this._configService.config = newSettings console.log(`Saved Settings: ${JSON.stringify(newSettings)}`) From e41ee47371bb51bd927a29863274c59df3cf5445 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 23 Jul 2022 16:21:53 -0700 Subject: [PATCH 12/28] filter attributes after notify --- .../dashboard-settings.component.html | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html index 06cf800..cde830a 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html @@ -47,21 +47,21 @@
- Device Status - Filter Attributes - - All - Critical + Device Status - Thresholds + + Smart + Scrutiny + Both
- Device Status - Thresholds - - Smart - Scrutiny - Both + Notify - Filter Attributes + + All + Critical
From 2e768fb49108ffe60f2552c17dda03feeb954196 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Mon, 25 Jul 2022 07:46:44 -0700 Subject: [PATCH 13/28] adding tests. Make sure that device status depends on the configured threshold. --- .../dashboard-device.component.spec.ts | 180 +++++++++++++++++- .../dashboard-device.component.ts | 10 +- 2 files changed, 178 insertions(+), 12 deletions(-) diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts index 2c21c89..3c6cc67 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts @@ -10,6 +10,10 @@ import {TREO_APP_CONFIG} from '@treo/services/config/config.constants'; import {DeviceSummaryModel} from 'app/core/models/device-summary-model'; import * as moment from 'moment'; import {HttpClientTestingModule} from '@angular/common/http/testing'; +import {HttpClient} from '@angular/common/http'; +import {ScrutinyConfigService} from 'app/core/config/scrutiny-config.service'; +import {of} from 'rxjs'; +import {MetricsStatusThreshold} from 'app/core/config/app.config'; describe('DashboardDeviceComponent', () => { let component: DashboardDeviceComponent; @@ -17,9 +21,14 @@ describe('DashboardDeviceComponent', () => { const matDialogSpy = jasmine.createSpyObj('MatDialog', ['open']); // const configServiceSpy = jasmine.createSpyObj('ScrutinyConfigService', ['config$']); - + let configService: ScrutinyConfigService; + let httpClientSpy: jasmine.SpyObj; beforeEach(async(() => { + + httpClientSpy = jasmine.createSpyObj('HttpClient', ['get']); + configService = new ScrutinyConfigService(httpClientSpy, {}); + TestBed.configureTestingModule({ imports: [ MatButtonModule, @@ -30,7 +39,8 @@ describe('DashboardDeviceComponent', () => { ], providers: [ {provide: MatDialog, useValue: matDialogSpy}, - {provide: TREO_APP_CONFIG, useValue: {dashboard_display: 'name'}} + {provide: TREO_APP_CONFIG, useValue: {dashboard_display: 'name', metrics: {status_threshold: 3}}}, + {provide: ScrutinyConfigService, useValue: configService} ], declarations: [DashboardDeviceComponent] }) @@ -50,25 +60,53 @@ describe('DashboardDeviceComponent', () => { describe('#classDeviceLastUpdatedOn()', () => { it('if non-zero device status, should be red', () => { + httpClientSpy.get.and.returnValue(of({ + settings: { + metrics: { + status_threshold: MetricsStatusThreshold.Both, + } + } + })); + component.ngOnInit() // component.deviceSummary = summary.data.summary['0x5000c500673e6b5f'] as DeviceSummaryModel expect(component.classDeviceLastUpdatedOn({ device: { - device_status: 2 - } + device_status: 2, + }, + smart: { + collector_date: moment().subtract(13, 'days').toISOString() + }, } as DeviceSummaryModel)).toBe('text-red') }); it('if non-zero device status, should be red', () => { - // component.deviceSummary = summary.data.summary['0x5000c500673e6b5f'] as DeviceSummaryModel + httpClientSpy.get.and.returnValue(of({ + settings: { + metrics: { + status_threshold: MetricsStatusThreshold.Both, + } + } + })); + component.ngOnInit() expect(component.classDeviceLastUpdatedOn({ device: { device_status: 2 - } + }, + smart: { + collector_date: moment().subtract(13, 'days').toISOString() + }, } as DeviceSummaryModel)).toBe('text-red') }); it('if healthy device status and updated in the last two weeks, should be green', () => { - // component.deviceSummary = summary.data.summary['0x5000c500673e6b5f'] as DeviceSummaryModel + httpClientSpy.get.and.returnValue(of({ + settings: { + metrics: { + status_threshold: MetricsStatusThreshold.Both, + } + } + })); + component.ngOnInit() expect(component.classDeviceLastUpdatedOn({ device: { device_status: 0 @@ -80,7 +118,14 @@ describe('DashboardDeviceComponent', () => { }); it('if healthy device status and updated more than two weeks ago, but less than 1 month, should be yellow', () => { - // component.deviceSummary = summary.data.summary['0x5000c500673e6b5f'] as DeviceSummaryModel + httpClientSpy.get.and.returnValue(of({ + settings: { + metrics: { + status_threshold: MetricsStatusThreshold.Both, + } + } + })); + component.ngOnInit() expect(component.classDeviceLastUpdatedOn({ device: { device_status: 0 @@ -92,7 +137,14 @@ describe('DashboardDeviceComponent', () => { }); it('if healthy device status and updated more 1 month ago, should be red', () => { - // component.deviceSummary = summary.data.summary['0x5000c500673e6b5f'] as DeviceSummaryModel + httpClientSpy.get.and.returnValue(of({ + settings: { + metrics: { + status_threshold: MetricsStatusThreshold.Both, + } + } + })); + component.ngOnInit() expect(component.classDeviceLastUpdatedOn({ device: { device_status: 0 @@ -104,4 +156,114 @@ describe('DashboardDeviceComponent', () => { }); }) + + describe('#deviceStatusString()', () => { + + it('if healthy device, should be passing', () => { + httpClientSpy.get.and.returnValue(of({ + settings: { + metrics: { + status_threshold: MetricsStatusThreshold.Both, + } + } + })); + component.ngOnInit() + expect(component.deviceStatusString({ + device: { + device_status: 0 + }, + smart: { + collector_date: moment().subtract(13, 'days').toISOString() + }, + } as DeviceSummaryModel)).toBe('passed') + }); + + it('if device with no smart data, should be unknown', () => { + httpClientSpy.get.and.returnValue(of({ + settings: { + metrics: { + status_threshold: MetricsStatusThreshold.Both, + } + } + })); + component.ngOnInit() + expect(component.deviceStatusString({ + device: { + device_status: 0 + }, + } as DeviceSummaryModel)).toBe('unknown') + }); + + const testCases = [ + { + 'deviceStatus': 1, + 'threshold': MetricsStatusThreshold.Smart, + 'result': 'failed' + }, + { + 'deviceStatus': 1, + 'threshold': MetricsStatusThreshold.Scrutiny, + 'result': 'passed' + }, + { + 'deviceStatus': 1, + 'threshold': MetricsStatusThreshold.Both, + 'result': 'failed' + }, + + { + 'deviceStatus': 2, + 'threshold': MetricsStatusThreshold.Smart, + 'result': 'passed' + }, + { + 'deviceStatus': 2, + 'threshold': MetricsStatusThreshold.Scrutiny, + 'result': 'failed' + }, + { + 'deviceStatus': 2, + 'threshold': MetricsStatusThreshold.Both, + 'result': 'failed' + }, + + { + 'deviceStatus': 3, + 'threshold': MetricsStatusThreshold.Smart, + 'result': 'failed' + }, + { + 'deviceStatus': 3, + 'threshold': MetricsStatusThreshold.Scrutiny, + 'result': 'failed' + }, + { + 'deviceStatus': 3, + 'threshold': MetricsStatusThreshold.Both, + 'result': 'failed' + } + + ] + + testCases.forEach((test, index) => { + it(`if device with status (${test.deviceStatus}) and threshold (${test.threshold}), should be ${test.result}`, () => { + httpClientSpy.get.and.returnValue(of({ + settings: { + metrics: { + status_threshold: test.threshold, + } + } + })); + component.ngOnInit() + expect(component.deviceStatusString({ + device: { + device_status: test.deviceStatus + }, + smart: { + collector_date: moment().subtract(13, 'days').toISOString() + }, + } as DeviceSummaryModel)).toBe(test.result) + }); + }); + }) }); diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts index a8de9d5..254f8c1 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts @@ -10,6 +10,8 @@ import {DashboardDeviceDeleteDialogComponent} from 'app/layout/common/dashboard- import {DeviceTitlePipe} from 'app/shared/device-title.pipe'; import {DeviceSummaryModel} from 'app/core/models/device-summary-model'; +export type deviceStatusName = 'unknown' | 'passed' | 'failed' + @Component({ selector: 'app-dashboard-device', templateUrl: './dashboard-device.component.html', @@ -50,9 +52,10 @@ export class DashboardDeviceComponent implements OnInit { // ----------------------------------------------------------------------------------------------------- classDeviceLastUpdatedOn(deviceSummary: DeviceSummaryModel): string { - if (deviceSummary.device.device_status !== 0) { + const deviceStatus = this.deviceStatusString(deviceSummary) + if (deviceStatus === 'failed') { return 'text-red' // if the device has failed, always highlight in red - } else if (deviceSummary.device.device_status === 0 && deviceSummary.smart) { + } else if (deviceStatus === 'passed') { if (moment().subtract(14, 'days').isBefore(deviceSummary.smart.collector_date)) { // this device was updated in the last 2 weeks. return 'text-green' @@ -68,7 +71,8 @@ export class DashboardDeviceComponent implements OnInit { } } - deviceStatusString(deviceSummary: DeviceSummaryModel): string { + + deviceStatusString(deviceSummary: DeviceSummaryModel): deviceStatusName { // no smart data, so treat the device status as unknown if (!deviceSummary.smart) { return 'unknown' From 5cd441da7ba446757d6ca64ae78bba142e223571 Mon Sep 17 00:00:00 2001 From: Matthew Kobayashi Date: Fri, 29 Jul 2022 09:33:55 +1000 Subject: [PATCH 14/28] Add udev troubleshooting doc --- docs/TROUBLESHOOTING_UDEV.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docs/TROUBLESHOOTING_UDEV.md diff --git a/docs/TROUBLESHOOTING_UDEV.md b/docs/TROUBLESHOOTING_UDEV.md new file mode 100644 index 0000000..619bf0c --- /dev/null +++ b/docs/TROUBLESHOOTING_UDEV.md @@ -0,0 +1,17 @@ +# Operating systems without udev + +Some operating systems do not come with `udev` out of the box, for example Alpine Linux. In these instances you will not be able to bind `/run/udev` to the container for sharing device metadata. Some operating systems offer `udev` as a package that can be installed separately, or an alternative (such as `eudev` in the case of Alpine Linux) that provides the same functionality. + +To install `eudev` in Alpine Linux (run as root): + +``` +apk add eudev +``` + +Once your `udev` implementation is installed, create `/run/udev` with the following command: + +``` +udevadm trigger +``` + +On Alpine Linux, this also has the benefit of creating symlinks to device serial numbers in `/dev/disk/by-id`. From ce2f990eb1b112f4f816c91dbcf4e849ab4b44d4 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Fri, 29 Jul 2022 07:11:57 -0700 Subject: [PATCH 15/28] consolidate device status to string logic in DeviceStatusPipe. Ensure device status takes into account new settings. --- .../dashboard-device.component.html | 12 +- .../dashboard-device.component.spec.ts | 111 ------------- .../dashboard-device.component.ts | 25 +-- .../app/modules/detail/detail.component.html | 5 +- .../app/modules/detail/detail.component.ts | 2 + .../src/app/shared/device-status.pipe.spec.ts | 148 +++++++++++++++++- .../src/app/shared/device-status.pipe.ts | 78 +++++++-- 7 files changed, 222 insertions(+), 159 deletions(-) diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html index bd7b4a1..7dfe40c 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html @@ -1,15 +1,15 @@ -
@@ -47,7 +47,7 @@
Status
{{ deviceStatusString(deviceSummary) | titlecase}}
+ *ngIf="deviceSummary.smart?.collector_date; else unknownStatus">{{ deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, config.metrics.status_threshold) | titlecase}}
No Data
diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts index 3c6cc67..6e578a1 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.spec.ts @@ -155,115 +155,4 @@ describe('DashboardDeviceComponent', () => { } as DeviceSummaryModel)).toBe('text-red') }); }) - - - describe('#deviceStatusString()', () => { - - it('if healthy device, should be passing', () => { - httpClientSpy.get.and.returnValue(of({ - settings: { - metrics: { - status_threshold: MetricsStatusThreshold.Both, - } - } - })); - component.ngOnInit() - expect(component.deviceStatusString({ - device: { - device_status: 0 - }, - smart: { - collector_date: moment().subtract(13, 'days').toISOString() - }, - } as DeviceSummaryModel)).toBe('passed') - }); - - it('if device with no smart data, should be unknown', () => { - httpClientSpy.get.and.returnValue(of({ - settings: { - metrics: { - status_threshold: MetricsStatusThreshold.Both, - } - } - })); - component.ngOnInit() - expect(component.deviceStatusString({ - device: { - device_status: 0 - }, - } as DeviceSummaryModel)).toBe('unknown') - }); - - const testCases = [ - { - 'deviceStatus': 1, - 'threshold': MetricsStatusThreshold.Smart, - 'result': 'failed' - }, - { - 'deviceStatus': 1, - 'threshold': MetricsStatusThreshold.Scrutiny, - 'result': 'passed' - }, - { - 'deviceStatus': 1, - 'threshold': MetricsStatusThreshold.Both, - 'result': 'failed' - }, - - { - 'deviceStatus': 2, - 'threshold': MetricsStatusThreshold.Smart, - 'result': 'passed' - }, - { - 'deviceStatus': 2, - 'threshold': MetricsStatusThreshold.Scrutiny, - 'result': 'failed' - }, - { - 'deviceStatus': 2, - 'threshold': MetricsStatusThreshold.Both, - 'result': 'failed' - }, - - { - 'deviceStatus': 3, - 'threshold': MetricsStatusThreshold.Smart, - 'result': 'failed' - }, - { - 'deviceStatus': 3, - 'threshold': MetricsStatusThreshold.Scrutiny, - 'result': 'failed' - }, - { - 'deviceStatus': 3, - 'threshold': MetricsStatusThreshold.Both, - 'result': 'failed' - } - - ] - - testCases.forEach((test, index) => { - it(`if device with status (${test.deviceStatus}) and threshold (${test.threshold}), should be ${test.result}`, () => { - httpClientSpy.get.and.returnValue(of({ - settings: { - metrics: { - status_threshold: test.threshold, - } - } - })); - component.ngOnInit() - expect(component.deviceStatusString({ - device: { - device_status: test.deviceStatus - }, - smart: { - collector_date: moment().subtract(13, 'days').toISOString() - }, - } as DeviceSummaryModel)).toBe(test.result) - }); - }); - }) }); diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts index 254f8c1..e29957e 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.ts @@ -9,8 +9,7 @@ import {MatDialog} from '@angular/material/dialog'; import {DashboardDeviceDeleteDialogComponent} from 'app/layout/common/dashboard-device-delete-dialog/dashboard-device-delete-dialog.component'; import {DeviceTitlePipe} from 'app/shared/device-title.pipe'; import {DeviceSummaryModel} from 'app/core/models/device-summary-model'; - -export type deviceStatusName = 'unknown' | 'passed' | 'failed' +import {DeviceStatusPipe} from 'app/shared/device-status.pipe'; @Component({ selector: 'app-dashboard-device', @@ -37,6 +36,8 @@ export class DashboardDeviceComponent implements OnInit { readonly humanizeDuration = humanizeDuration; + deviceStatusForModelWithThreshold = DeviceStatusPipe.deviceStatusForModelWithThreshold + ngOnInit(): void { // Subscribe to config changes this._configService.config$ @@ -52,7 +53,7 @@ export class DashboardDeviceComponent implements OnInit { // ----------------------------------------------------------------------------------------------------- classDeviceLastUpdatedOn(deviceSummary: DeviceSummaryModel): string { - const deviceStatus = this.deviceStatusString(deviceSummary) + const deviceStatus = DeviceStatusPipe.deviceStatusForModelWithThreshold(deviceSummary.device, !!deviceSummary.smart, this.config.metrics.status_threshold) if (deviceStatus === 'failed') { return 'text-red' // if the device has failed, always highlight in red } else if (deviceStatus === 'passed') { @@ -71,24 +72,6 @@ export class DashboardDeviceComponent implements OnInit { } } - - deviceStatusString(deviceSummary: DeviceSummaryModel): deviceStatusName { - // no smart data, so treat the device status as unknown - if (!deviceSummary.smart) { - return 'unknown' - } - - // determine the device status, by comparing it against the allowed threshold - // tslint:disable-next-line:no-bitwise - const deviceStatus = deviceSummary.device.device_status & this.config.metrics.status_threshold - if (deviceStatus === 0) { - return 'passed' - } else { - return 'failed' - } - } - - openDeleteDialog(): void { const dialogRef = this.dialog.open(DashboardDeviceDeleteDialogComponent, { // width: '250px', diff --git a/webapp/frontend/src/app/modules/detail/detail.component.html b/webapp/frontend/src/app/modules/detail/detail.component.html index e96a493..d2ff848 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.html +++ b/webapp/frontend/src/app/modules/detail/detail.component.html @@ -56,12 +56,13 @@
- {{device?.device_status | deviceStatus}} + {{device | deviceStatus:!!smart_results:config.metrics.status_threshold:true}}
Status
diff --git a/webapp/frontend/src/app/modules/detail/detail.component.ts b/webapp/frontend/src/app/modules/detail/detail.component.ts index 1353c51..ce21642 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.ts +++ b/webapp/frontend/src/app/modules/detail/detail.component.ts @@ -16,6 +16,7 @@ import {DeviceModel} from 'app/core/models/device-model'; import {SmartModel} from 'app/core/models/measurements/smart-model'; import {SmartAttributeModel} from 'app/core/models/measurements/smart-attribute-model'; import {AttributeMetadataModel} from 'app/core/models/thresholds/attribute-metadata-model'; +import {DeviceStatusPipe} from 'app/shared/device-status.pipe'; // from Constants.go - these must match const AttributeStatusPassed = 0 @@ -89,6 +90,7 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy { readonly humanizeDuration = humanizeDuration; + deviceStatusForModelWithThreshold = DeviceStatusPipe.deviceStatusForModelWithThreshold // ----------------------------------------------------------------------------------------------------- // @ Lifecycle hooks // ----------------------------------------------------------------------------------------------------- diff --git a/webapp/frontend/src/app/shared/device-status.pipe.spec.ts b/webapp/frontend/src/app/shared/device-status.pipe.spec.ts index 23bc958..57d9e7c 100644 --- a/webapp/frontend/src/app/shared/device-status.pipe.spec.ts +++ b/webapp/frontend/src/app/shared/device-status.pipe.spec.ts @@ -1,8 +1,146 @@ -import { DeviceStatusPipe } from './device-status.pipe'; +import {DeviceStatusPipe} from './device-status.pipe'; +import {MetricsStatusThreshold} from '../core/config/app.config'; +import {DeviceModel} from '../core/models/device-model'; describe('DeviceStatusPipe', () => { - it('create an instance', () => { - const pipe = new DeviceStatusPipe(); - expect(pipe).toBeTruthy(); - }); + it('create an instance', () => { + const pipe = new DeviceStatusPipe(); + expect(pipe).toBeTruthy(); + }); + + describe('#deviceStatusForModelWithThreshold', () => { + it('if healthy device, should be passing', () => { + expect(DeviceStatusPipe.deviceStatusForModelWithThreshold( + {device_status: 0} as DeviceModel, + true, + MetricsStatusThreshold.Both + )).toBe('passed') + }); + + it('if device with no smart data, should be unknown', () => { + expect(DeviceStatusPipe.deviceStatusForModelWithThreshold( + {device_status: 0} as DeviceModel, + false, + MetricsStatusThreshold.Both + )).toBe('unknown') + }); + + const testCases = [ + { + 'deviceStatus': 10000, // invalid status + 'hasSmartResults': false, + 'threshold': MetricsStatusThreshold.Smart, + 'includeReason': false, + 'result': 'unknown' + }, + + { + 'deviceStatus': 1, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Smart, + 'includeReason': false, + 'result': 'failed' + }, + { + 'deviceStatus': 1, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Scrutiny, + 'includeReason': false, + 'result': 'passed' + }, + { + 'deviceStatus': 1, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Both, + 'includeReason': false, + 'result': 'failed' + }, + + { + 'deviceStatus': 2, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Smart, + 'includeReason': false, + 'result': 'passed' + }, + { + 'deviceStatus': 2, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Scrutiny, + 'includeReason': false, + 'result': 'failed' + }, + { + 'deviceStatus': 2, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Both, + 'includeReason': false, + 'result': 'failed' + }, + + { + 'deviceStatus': 3, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Smart, + 'includeReason': false, + 'result': 'failed' + }, + { + 'deviceStatus': 3, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Scrutiny, + 'includeReason': false, + 'result': 'failed' + }, + { + 'deviceStatus': 3, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Both, + 'includeReason': false, + 'result': 'failed' + }, + + { + 'deviceStatus': 3, + 'hasSmartResults': false, + 'threshold': MetricsStatusThreshold.Smart, + 'includeReason': true, + 'result': 'unknown' + }, + { + 'deviceStatus': 3, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Smart, + 'includeReason': true, + 'result': 'failed: smart' + }, + { + 'deviceStatus': 3, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Scrutiny, + 'includeReason': true, + 'result': 'failed: scrutiny' + }, + { + 'deviceStatus': 3, + 'hasSmartResults': true, + 'threshold': MetricsStatusThreshold.Both, + 'includeReason': true, + 'result': 'failed: both' + } + + + ] + + testCases.forEach((test, index) => { + it(`if device with status (${test.deviceStatus}), hasSmartResults(${test.hasSmartResults}) and threshold (${test.threshold}), should be ${test.result}`, () => { + expect(DeviceStatusPipe.deviceStatusForModelWithThreshold( + {device_status: test.deviceStatus} as DeviceModel, + test.hasSmartResults, + test.threshold, + test.includeReason + )).toBe(test.result) + }); + }); + }); }); diff --git a/webapp/frontend/src/app/shared/device-status.pipe.ts b/webapp/frontend/src/app/shared/device-status.pipe.ts index 42261c6..68a692d 100644 --- a/webapp/frontend/src/app/shared/device-status.pipe.ts +++ b/webapp/frontend/src/app/shared/device-status.pipe.ts @@ -1,21 +1,71 @@ -import { Pipe, PipeTransform } from '@angular/core'; +import {Pipe, PipeTransform} from '@angular/core'; +import {MetricsStatusThreshold} from '../core/config/app.config'; +import {DeviceModel} from '../core/models/device-model'; + +const DEVICE_STATUS_NAMES: { [key: number]: string } = { + 0: 'passed', + 1: 'failed', + 2: 'failed', + 3: 'failed' +}; + +const DEVICE_STATUS_NAMES_WITH_REASON: { [key: number]: string } = { + 0: 'passed', + 1: 'failed: smart', + 2: 'failed: scrutiny', + 3: 'failed: both' +}; + @Pipe({ - name: 'deviceStatus' + name: 'deviceStatus' }) export class DeviceStatusPipe implements PipeTransform { - transform(deviceStatusFlag: number): string { - if(deviceStatusFlag === 0){ - return 'passed' - } else if(deviceStatusFlag === 3){ - return 'failed: both' - } else if(deviceStatusFlag === 2) { - return 'failed: scrutiny' - } else if(deviceStatusFlag === 1) { - return 'failed: smart' - } - return 'unknown' - } + + static deviceStatusForModelWithThreshold( + deviceModel: DeviceModel, + hasSmartResults: boolean = true, + threshold: MetricsStatusThreshold = MetricsStatusThreshold.Both, + includeReason: boolean = false + ): string { + // no smart data, so treat the device status as unknown + if (!hasSmartResults) { + return 'unknown' + } + + let statusNameLookup = DEVICE_STATUS_NAMES + if (includeReason) { + statusNameLookup = DEVICE_STATUS_NAMES_WITH_REASON + } + // determine the device status, by comparing it against the allowed threshold + // tslint:disable-next-line:no-bitwise + const deviceStatus = deviceModel.device_status & threshold + return statusNameLookup[deviceStatus] + } + + // static deviceStatusForModelWithThreshold(deviceModel: DeviceModel | any, threshold: MetricsStatusThreshold): string { + // // tslint:disable-next-line:no-bitwise + // const deviceStatus = deviceModel?.device_status & threshold + // if(deviceStatus === 0){ + // return 'passed' + // } else if(deviceStatus === 3){ + // return 'failed: both' + // } else if(deviceStatus === 2) { + // return 'failed: scrutiny' + // } else if(deviceStatus === 1) { + // return 'failed: smart' + // } + // return 'unknown' + // } + + transform( + deviceModel: DeviceModel, + hasSmartResults: boolean = true, + threshold: MetricsStatusThreshold = MetricsStatusThreshold.Both, + includeReason: boolean = false + ): string { + return DeviceStatusPipe.deviceStatusForModelWithThreshold(deviceModel, hasSmartResults, threshold, includeReason) + } } From 3f272b36d4c6d88b6a8efecc5eda8c3f3afea101 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 30 Jul 2022 08:50:23 -0700 Subject: [PATCH 16/28] adding setting to allow users to customize between binary vs SI/Metric units in UI. fixes #330 --- .../migrations/m20220716214900/setting.go | 1 + .../scrutiny_repository_migrations.go | 6 ++ .../database/scrutiny_repository_settings.go | 6 +- webapp/backend/pkg/models/setting_entry.go | 1 + webapp/backend/pkg/models/settings.go | 1 + .../src/app/core/config/app.config.ts | 4 + .../dashboard-device.component.html | 3 +- .../dashboard-settings.component.html | 10 ++- .../dashboard-settings.component.ts | 3 + .../app/modules/detail/detail.component.html | 2 +- .../src/app/shared/file-size.pipe.spec.ts | 60 ++++++++++--- .../frontend/src/app/shared/file-size.pipe.ts | 84 ++++--------------- 12 files changed, 100 insertions(+), 81 deletions(-) diff --git a/webapp/backend/pkg/database/migrations/m20220716214900/setting.go b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go index 9c1f746..70d8d5e 100644 --- a/webapp/backend/pkg/database/migrations/m20220716214900/setting.go +++ b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go @@ -14,4 +14,5 @@ type Setting struct { SettingValueNumeric int `json:"setting_value_numeric"` SettingValueString string `json:"setting_value_string"` + SettingValueBool bool `json:"setting_value_bool"` } diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index a6f1b68..99eb1f3 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -319,6 +319,12 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { SettingDataType: "string", SettingValueString: "celsius", }, + { + SettingKeyName: "file_size_si_units", + SettingKeyDescription: "File size in SI units (true | false)", + SettingDataType: "bool", + SettingValueBool: false, + }, { SettingKeyName: "metrics.notify_level", diff --git a/webapp/backend/pkg/database/scrutiny_repository_settings.go b/webapp/backend/pkg/database/scrutiny_repository_settings.go index 918a9f4..d92ce9b 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_settings.go +++ b/webapp/backend/pkg/database/scrutiny_repository_settings.go @@ -24,6 +24,8 @@ func (sr *scrutinyRepository) LoadSettings(ctx context.Context) (*models.Setting sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueNumeric) } else if settingsEntry.SettingDataType == "string" { sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueString) + } else if settingsEntry.SettingDataType == "bool" { + sr.appConfig.SetDefault(configKey, settingsEntry.SettingValueBool) } } @@ -67,11 +69,13 @@ func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models. settingsEntries[ndx].SettingValueNumeric = sr.appConfig.GetInt(configKey) } else if settingsEntry.SettingDataType == "string" { settingsEntries[ndx].SettingValueString = sr.appConfig.GetString(configKey) + } else if settingsEntry.SettingDataType == "bool" { + settingsEntries[ndx].SettingValueBool = sr.appConfig.GetBool(configKey) } // store in database. //TODO: this should be `sr.gormClient.Updates(&settingsEntries).Error` - err := sr.gormClient.Model(&models.SettingEntry{}).Where([]uint{settingsEntry.ID}).Select("setting_value_numeric", "setting_value_string").Updates(settingsEntries[ndx]).Error + err := sr.gormClient.Model(&models.SettingEntry{}).Where([]uint{settingsEntry.ID}).Select("setting_value_numeric", "setting_value_string", "setting_value_bool").Updates(settingsEntries[ndx]).Error if err != nil { return err } diff --git a/webapp/backend/pkg/models/setting_entry.go b/webapp/backend/pkg/models/setting_entry.go index 48d2c4c..2ac78f6 100644 --- a/webapp/backend/pkg/models/setting_entry.go +++ b/webapp/backend/pkg/models/setting_entry.go @@ -15,6 +15,7 @@ type SettingEntry struct { SettingValueNumeric int `json:"setting_value_numeric"` SettingValueString string `json:"setting_value_string"` + SettingValueBool bool `json:"setting_value_bool"` } func (s SettingEntry) TableName() string { diff --git a/webapp/backend/pkg/models/settings.go b/webapp/backend/pkg/models/settings.go index 48ba2d5..f06db84 100644 --- a/webapp/backend/pkg/models/settings.go +++ b/webapp/backend/pkg/models/settings.go @@ -13,6 +13,7 @@ type Settings struct { DashboardDisplay string `json:"dashboard_display" mapstructure:"dashboard_display"` DashboardSort string `json:"dashboard_sort" mapstructure:"dashboard_sort"` TemperatureUnit string `json:"temperature_unit" mapstructure:"temperature_unit"` + FileSizeSIUnits bool `json:"file_size_si_units" mapstructure:"file_size_si_units"` Metrics struct { NotifyLevel int `json:"notify_level" mapstructure:"notify_level"` diff --git a/webapp/frontend/src/app/core/config/app.config.ts b/webapp/frontend/src/app/core/config/app.config.ts index b4a6114..92f0451 100644 --- a/webapp/frontend/src/app/core/config/app.config.ts +++ b/webapp/frontend/src/app/core/config/app.config.ts @@ -43,6 +43,8 @@ export interface AppConfig { temperature_unit?: TemperatureUnit; + file_size_si_units?: boolean; + // Settings from Scrutiny API metrics?: { @@ -69,6 +71,8 @@ export const appConfig: AppConfig = { dashboard_sort: 'status', temperature_unit: 'celsius', + file_size_si_units: false, + metrics: { notify_level: MetricsNotifyLevel.Fail, status_filter_attributes: MetricsStatusFilterAttributes.All, diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html index 7dfe40c..43e8964 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html @@ -58,7 +58,8 @@
Capacity
-
{{ deviceSummary.device.capacity | fileSize}}
+
{{ deviceSummary.device.capacity | fileSize:config.file_size_si_units}}
Powered On
diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html index cde830a..750d54d 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html @@ -37,12 +37,20 @@
- Temperature Display Unit + Temperature Celsius Fahrenheit + + + File Size + + SI Units (GB) + Binary Units (GiB) + +
diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts index 21e8e8e..6bc5f2a 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts @@ -22,6 +22,7 @@ export class DashboardSettingsComponent implements OnInit { dashboardDisplay: string; dashboardSort: string; temperatureUnit: string; + fileSizeSIUnits: boolean; theme: string; statusThreshold: number; statusFilterAttributes: number; @@ -46,6 +47,7 @@ export class DashboardSettingsComponent implements OnInit { this.dashboardDisplay = config.dashboard_display; this.dashboardSort = config.dashboard_sort; this.temperatureUnit = config.temperature_unit; + this.fileSizeSIUnits = config.file_size_si_units; this.theme = config.theme; this.statusFilterAttributes = config.metrics.status_filter_attributes; @@ -60,6 +62,7 @@ export class DashboardSettingsComponent implements OnInit { dashboard_display: this.dashboardDisplay as DashboardDisplay, dashboard_sort: this.dashboardSort as DashboardSort, temperature_unit: this.temperatureUnit as TemperatureUnit, + file_size_si_units: this.fileSizeSIUnits, theme: this.theme as Theme, metrics: { status_filter_attributes: this.statusFilterAttributes as MetricsStatusFilterAttributes, diff --git a/webapp/frontend/src/app/modules/detail/detail.component.html b/webapp/frontend/src/app/modules/detail/detail.component.html index d2ff848..5c94592 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.html +++ b/webapp/frontend/src/app/modules/detail/detail.component.html @@ -107,7 +107,7 @@
Firmware Version
-
{{device?.capacity | fileSize}}
+
{{device?.capacity | fileSize:config.file_size_si_units}}
Capacity
diff --git a/webapp/frontend/src/app/shared/file-size.pipe.spec.ts b/webapp/frontend/src/app/shared/file-size.pipe.spec.ts index 14973cf..0f2127a 100644 --- a/webapp/frontend/src/app/shared/file-size.pipe.spec.ts +++ b/webapp/frontend/src/app/shared/file-size.pipe.spec.ts @@ -1,4 +1,4 @@ -import { FileSizePipe } from './file-size.pipe'; +import {FileSizePipe} from './file-size.pipe'; describe('FileSizePipe', () => { it('create an instance', () => { @@ -10,23 +10,61 @@ describe('FileSizePipe', () => { const testCases = [ { 'bytes': 1500, - 'precision': undefined, - 'result': '1 KB' - },{ - 'bytes': 2_100_000_000, - 'precision': undefined, - 'result': '2.0 GB', - },{ + 'si': false, + 'result': '1.5 KiB' + }, + { 'bytes': 1500, - 'precision': 2, - 'result': '1.46 KB', + 'si': true, + 'result': '1.5 kB' + }, + { + 'bytes': 5000, + 'si': false, + 'result': '4.9 KiB', + }, + { + 'bytes': 5000, + 'si': true, + 'result': '5.0 kB', + }, + { + 'bytes': 999_949, + 'si': false, + 'result': '976.5 KiB', + }, + { + 'bytes': 999_949, + 'si': true, + 'result': '999.9 kB', + }, + { + 'bytes': 999_950, + 'si': true, + 'result': '1.0 MB', + }, + { + 'bytes': 1_551_859_712, + 'si': false, + 'result': '1.4 GiB', + }, + { + 'bytes': 2_100_000_000, + 'si': false, + 'result': '2.0 GiB', + }, + { + 'bytes': 2_100_000_000, + 'si': true, + 'result': '2.1 GB', } ] + testCases.forEach((test, index) => { it(`should correctly format bytes ${test.bytes}. (testcase: ${index + 1})`, () => { // test const pipe = new FileSizePipe(); - const formatted = pipe.transform(test.bytes, test.precision) + const formatted = pipe.transform(test.bytes, test.si) expect(formatted).toEqual(test.result); }); }) diff --git a/webapp/frontend/src/app/shared/file-size.pipe.ts b/webapp/frontend/src/app/shared/file-size.pipe.ts index e6cbc7b..14fdf0c 100644 --- a/webapp/frontend/src/app/shared/file-size.pipe.ts +++ b/webapp/frontend/src/app/shared/file-size.pipe.ts @@ -1,75 +1,27 @@ -/** - * @license - * Copyright (c) 2019 Jonathan Catmull. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -import { Pipe, PipeTransform } from '@angular/core'; +import {Pipe, PipeTransform} from '@angular/core'; -type unit = 'bytes' | 'KB' | 'MB' | 'GB' | 'TB' | 'PB'; -type unitPrecisionMap = { - [u in unit]: number; -}; - -const defaultPrecisionMap: unitPrecisionMap = { - bytes: 0, - KB: 0, - MB: 1, - GB: 1, - TB: 2, - PB: 2 -}; - -/* - * Convert bytes into largest possible unit. - * Takes an precision argument that can be a number or a map for each unit. - * Usage: - * bytes | fileSize:precision - * @example - * // returns 1 KB - * {{ 1500 | fileSize }} - * @example - * // returns 2.1 GB - * {{ 2100000000 | fileSize }} - * @example - * // returns 1.46 KB - * {{ 1500 | fileSize:2 }} - */ -@Pipe({ name: 'fileSize' }) +@Pipe({name: 'fileSize'}) export class FileSizePipe implements PipeTransform { - private readonly units: unit[] = ['bytes', 'KB', 'MB', 'GB', 'TB', 'PB']; - transform(bytes: number = 0, precision: number | unitPrecisionMap = defaultPrecisionMap): string { - if (isNaN(parseFloat(String(bytes))) || !isFinite(bytes)) return '?'; + transform(bytes: number = 0, si = false, dp = 1): string { + const thresh = si ? 1000 : 1024; - let unitIndex = 0; - - while (bytes >= 1024) { - bytes /= 1024; - unitIndex++; + if (Math.abs(bytes) < thresh) { + return bytes + ' B'; } - const unit = this.units[unitIndex]; + const units = si + ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] + : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; + let u = -1; + const r = 10 ** dp; - if (typeof precision === 'number') { - return `${bytes.toFixed(+precision)} ${unit}`; - } - return `${bytes.toFixed(precision[unit])} ${unit}`; + do { + bytes /= thresh; + ++u; + } while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1); + + + return bytes.toFixed(dp) + ' ' + units[u]; } } From 3205e3d022a1840f73ae46f6e5e7d6b7a06850a2 Mon Sep 17 00:00:00 2001 From: KF5JWC <43411836+KF5JWC@users.noreply.github.com> Date: Sun, 31 Jul 2022 00:07:04 -0500 Subject: [PATCH 17/28] Update INSTALL_SYNOLOGY_COLLECTOR.md Typo: Created and loaded config into `conf/`, but specifies `config/` in argument --- docs/INSTALL_SYNOLOGY_COLLECTOR.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/INSTALL_SYNOLOGY_COLLECTOR.md b/docs/INSTALL_SYNOLOGY_COLLECTOR.md index acd80d0..e9c7a05 100644 --- a/docs/INSTALL_SYNOLOGY_COLLECTOR.md +++ b/docs/INSTALL_SYNOLOGY_COLLECTOR.md @@ -91,7 +91,7 @@ wget https://raw.githubusercontent.com/smartmontools/smartmontools/master/smartm ``` #!/bin/bash -/volume1/\@Entware/scrutiny/bin/scrutiny-collector-metrics-linux-arm64 run --config /volume1/\@Entware/scrutiny/config/collector.yaml +/volume1/\@Entware/scrutiny/bin/scrutiny-collector-metrics-linux-arm64 run --config /volume1/\@Entware/scrutiny/conf/collector.yaml ``` ## Set up Synology to run a scheduled task. @@ -131,4 +131,4 @@ Frequency: ## Troubleshooting -If you have any issues with your devices being detected, or incorrect data, please take a look at [TROUBLESHOOTING_DEVICE_COLLECTOR.md](./TROUBLESHOOTING_DEVICE_COLLECTOR.md) \ No newline at end of file +If you have any issues with your devices being detected, or incorrect data, please take a look at [TROUBLESHOOTING_DEVICE_COLLECTOR.md](./TROUBLESHOOTING_DEVICE_COLLECTOR.md) From 41c9daa93914aa44b0467e6db47847b3f04b3bf7 Mon Sep 17 00:00:00 2001 From: KF5JWC <43411836+KF5JWC@users.noreply.github.com> Date: Mon, 1 Aug 2022 15:07:28 -0500 Subject: [PATCH 18/28] Make `run_collect.sh` executable Synology task will fail when not executable: ``` /bin/bash: /volume1/@Entware/scrutiny/bin/run_collect.sh: Permission denied ``` --- docs/INSTALL_SYNOLOGY_COLLECTOR.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/INSTALL_SYNOLOGY_COLLECTOR.md b/docs/INSTALL_SYNOLOGY_COLLECTOR.md index acd80d0..345c148 100644 --- a/docs/INSTALL_SYNOLOGY_COLLECTOR.md +++ b/docs/INSTALL_SYNOLOGY_COLLECTOR.md @@ -94,6 +94,10 @@ wget https://raw.githubusercontent.com/smartmontools/smartmontools/master/smartm /volume1/\@Entware/scrutiny/bin/scrutiny-collector-metrics-linux-arm64 run --config /volume1/\@Entware/scrutiny/config/collector.yaml ``` +**Make `run_collect.sh` executable** + +`chmod +x /volume1/\@Entware/scrutiny/bin/run_collect.sh` + ## Set up Synology to run a scheduled task. Log in to DSM and do the following: @@ -131,4 +135,4 @@ Frequency: ## Troubleshooting -If you have any issues with your devices being detected, or incorrect data, please take a look at [TROUBLESHOOTING_DEVICE_COLLECTOR.md](./TROUBLESHOOTING_DEVICE_COLLECTOR.md) \ No newline at end of file +If you have any issues with your devices being detected, or incorrect data, please take a look at [TROUBLESHOOTING_DEVICE_COLLECTOR.md](./TROUBLESHOOTING_DEVICE_COLLECTOR.md) From fa8f86ab7b4d8d960f8997e51d1c78cfb0d97762 Mon Sep 17 00:00:00 2001 From: Matthew Kobayashi Date: Wed, 3 Aug 2022 11:02:51 +1000 Subject: [PATCH 19/28] Add missing setup command --- docs/TROUBLESHOOTING_UDEV.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/TROUBLESHOOTING_UDEV.md b/docs/TROUBLESHOOTING_UDEV.md index 619bf0c..89dce0d 100644 --- a/docs/TROUBLESHOOTING_UDEV.md +++ b/docs/TROUBLESHOOTING_UDEV.md @@ -6,6 +6,7 @@ To install `eudev` in Alpine Linux (run as root): ``` apk add eudev +setup-udev ``` Once your `udev` implementation is installed, create `/run/udev` with the following command: From a1b010850382bf0a9cdb08f1752f74951b8de339 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Tue, 2 Aug 2022 22:14:23 -0700 Subject: [PATCH 20/28] Added PRAGMA settings support when connecting to SQLITE db. When a transaction cannot lock the database, because it is already locked by another one, SQLite by default throws an error: database is locked. This behavior is usually not appropriate when concurrent access is needed, typically when multiple processes write to the same database. PRAGMA busy_timeout lets you set a timeout or a handler for these events. When setting a timeout, SQLite will try the transaction multiple times within this timeout. https://rsqlite.r-dbi.org/reference/sqlitesetbusyhandler retrying for 30000 milliseconds, 30seconds - this would be unreasonable for a distributed multi-tenant application, but should be fine for local usage. added mechanism for global settings (PRAGMA and DB level instructions). fixes #341 --- .../pkg/database/scrutiny_repository.go | 28 ++++++++++++++++++- .../scrutiny_repository_migrations.go | 24 ++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/webapp/backend/pkg/database/scrutiny_repository.go b/webapp/backend/pkg/database/scrutiny_repository.go index 81f2316..521ba7d 100644 --- a/webapp/backend/pkg/database/scrutiny_repository.go +++ b/webapp/backend/pkg/database/scrutiny_repository.go @@ -62,7 +62,20 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field // Gorm/SQLite setup //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location")) - database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{ + + // When a transaction cannot lock the database, because it is already locked by another one, + // SQLite by default throws an error: database is locked. This behavior is usually not appropriate when + // concurrent access is needed, typically when multiple processes write to the same database. + // PRAGMA busy_timeout lets you set a timeout or a handler for these events. When setting a timeout, + // SQLite will try the transaction multiple times within this timeout. + // fixes #341 + // https://rsqlite.r-dbi.org/reference/sqlitesetbusyhandler + // retrying for 30000 milliseconds, 30seconds - this would be unreasonable for a distributed multi-tenant application, + // but should be fine for local usage. + pragmaStr := sqlitePragmaString(map[string]string{ + "busy_timeout": "30000", + }) + database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")+pragmaStr), &gorm.Config{ //TODO: figure out how to log database queries again. //Logger: logger DisableForeignKeyConstraintWhenMigrating: true, @@ -450,3 +463,16 @@ func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []str } return []string{DURATION_KEY_WEEK} } + +func sqlitePragmaString(pragmas map[string]string) string { + q := url.Values{} + for key, val := range pragmas { + q.Add("_pragma", key+"="+val) + } + + queryStr := q.Encode() + if len(queryStr) > 0 { + return "?" + queryStr + } + return "" +} diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 99eb1f3..015428c 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -355,6 +355,30 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { return err } sr.logger.Infoln("Database migration completed successfully") + + //these migrations cannot be done within a transaction, so they are done as a separate group, with `UseTransaction = false` + sr.logger.Infoln("SQLite global configuration migrations starting. Please wait....") + globalMigrateOptions := gormigrate.DefaultOptions + globalMigrateOptions.UseTransaction = false + gm := gormigrate.New(sr.gormClient, globalMigrateOptions, []*gormigrate.Migration{ + { + ID: "g20220802211500", + Migrate: func(tx *gorm.DB) error { + //shrink the Database (maybe necessary after 20220503113100) + if err := tx.Exec("VACUUM;").Error; err != nil { + return err + } + return nil + }, + }, + }) + + if err := gm.Migrate(); err != nil { + sr.logger.Errorf("SQLite global configuration migrations failed with error. \n Please open a github issue at https://github.com/AnalogJ/scrutiny and attach a copy of your scrutiny.db file. \n %v", err) + return err + } + sr.logger.Infoln("SQLite global configuration migrations completed successfully") + return nil } From 2d6f60abaa68dc4560aa426874cf04e528121b62 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 3 Aug 2022 18:23:58 -0700 Subject: [PATCH 21/28] attrHistory needs to be reversed, so the newest data is on the right fixes #339 --- webapp/frontend/src/app/modules/detail/detail.component.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webapp/frontend/src/app/modules/detail/detail.component.ts b/webapp/frontend/src/app/modules/detail/detail.component.ts index ce21642..e87b75f 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.ts +++ b/webapp/frontend/src/app/modules/detail/detail.component.ts @@ -351,7 +351,9 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy { attributes[attrId].chartData = [ { name: 'chart-line-sparkline', - data: attrHistory + // attrHistory needs to be reversed, so the newest data is on the right + // fixes #339 + data: attrHistory.reverse() } ] } From 9a4a8de34183609e39c878fe5bec06e6dc02599f Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 3 Aug 2022 18:38:59 -0700 Subject: [PATCH 22/28] make sure the settings dialog width is 600px for readability. --- .../frontend/src/app/modules/dashboard/dashboard.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts b/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts index f790891..70b40f6 100644 --- a/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts +++ b/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts @@ -237,7 +237,7 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy } openDialog(): void { - const dialogRef = this.dialog.open(DashboardSettingsComponent); + const dialogRef = this.dialog.open(DashboardSettingsComponent, {width: '600px',}); dialogRef.afterClosed().subscribe(result => { console.log(`Dialog result: ${result}`); From d41d535ab71532e9ba20a6bec69c8f90f34d3bb6 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 3 Aug 2022 20:55:34 -0700 Subject: [PATCH 23/28] make sure that the device host id is provided in notifications (if available). fixes #337 --- docs/TROUBLESHOOTING_NOTIFICATIONS.md | 1 + webapp/backend/pkg/notify/notify.go | 61 ++++++++++++----- webapp/backend/pkg/notify/notify_test.go | 83 ++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 18 deletions(-) diff --git a/docs/TROUBLESHOOTING_NOTIFICATIONS.md b/docs/TROUBLESHOOTING_NOTIFICATIONS.md index 2baba6a..2f2abee 100644 --- a/docs/TROUBLESHOOTING_NOTIFICATIONS.md +++ b/docs/TROUBLESHOOTING_NOTIFICATIONS.md @@ -21,5 +21,6 @@ SCRUTINY_DEVICE_NAME - eg. /dev/sda SCRUTINY_DEVICE_TYPE - ATA/SCSI/NVMe SCRUTINY_DEVICE_SERIAL - eg. WDDJ324KSO SCRUTINY_MESSAGE - eg. "Scrutiny SMART error notification for device: %s\nFailure Type: %s\nDevice Name: %s\nDevice Serial: %s\nDevice Type: %s\nDate: %s" +SCRUTINY_HOST_ID - (optional) eg. "my-custom-host-id" ``` diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index 657b007..3dbe661 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -101,12 +101,13 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThr } } -// TODO: include host and/or user label for device. +// TODO: include user label for device. type Payload struct { - DeviceType string `json:"device_type"` //ATA/SCSI/NVMe - DeviceName string `json:"device_name"` //dev/sda - DeviceSerial string `json:"device_serial"` //WDDJ324KSO - Test bool `json:"test"` // false + HostId string `json:"host_id,omitempty"` //host id (optional) + DeviceType string `json:"device_type"` //ATA/SCSI/NVMe + DeviceName string `json:"device_name"` //dev/sda + DeviceSerial string `json:"device_serial"` //WDDJ324KSO + Test bool `json:"test"` // false //private, populated during init (marked as Public for JSON serialization) Date string `json:"date"` //populated by Send function. @@ -115,8 +116,9 @@ type Payload struct { Message string `json:"message"` } -func NewPayload(device models.Device, test bool) Payload { +func NewPayload(device models.Device, test bool, currentTime ...time.Time) Payload { payload := Payload{ + HostId: strings.TrimSpace(device.HostId), DeviceType: device.DeviceType, DeviceName: device.DeviceName, DeviceSerial: device.SerialNumber, @@ -124,7 +126,13 @@ func NewPayload(device models.Device, test bool) Payload { } //validate that the Payload is populated - sendDate := time.Now() + var sendDate time.Time + if currentTime != nil && len(currentTime) > 0 { + sendDate = currentTime[0] + } else { + sendDate = time.Now() + } + payload.Date = sendDate.Format(time.RFC3339) payload.FailureType = payload.GenerateFailureType(device.DeviceStatus) payload.Subject = payload.GenerateSubject() @@ -148,25 +156,39 @@ func (p *Payload) GenerateFailureType(deviceStatus pkg.DeviceStatus) string { func (p *Payload) GenerateSubject() string { //generate a detailed failure message - return fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName) + var subject string + if len(p.HostId) > 0 { + subject = fmt.Sprintf("Scrutiny SMART error (%s) detected on [host]device: [%s]%s", p.FailureType, p.HostId, p.DeviceName) + } else { + subject = fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName) + } + return subject } func (p *Payload) GenerateMessage() string { //generate a detailed failure message - message := fmt.Sprintf( - `Scrutiny SMART error notification for device: %s -Failure Type: %s -Device Name: %s -Device Serial: %s -Device Type: %s -Date: %s`, p.DeviceName, p.FailureType, p.DeviceName, p.DeviceSerial, p.DeviceType, p.Date) + messageParts := []string{} - if p.Test { - message = "TEST NOTIFICATION:\n" + message + messageParts = append(messageParts, fmt.Sprintf("Scrutiny SMART error notification for device: %s", p.DeviceName)) + if len(p.HostId) > 0 { + messageParts = append(messageParts, fmt.Sprintf("Host Id: %s", p.HostId)) } - return message + messageParts = append(messageParts, + fmt.Sprintf("Failure Type: %s", p.FailureType), + fmt.Sprintf("Device Name: %s", p.DeviceName), + fmt.Sprintf("Device Serial: %s", p.DeviceSerial), + fmt.Sprintf("Device Type: %s", p.DeviceType), + "", + fmt.Sprintf("Date: %s", p.Date), + ) + + if p.Test { + messageParts = append([]string{"TEST NOTIFICATION:"}, messageParts...) + } + + return strings.Join(messageParts, "\n") } func New(logger logrus.FieldLogger, appconfig config.Interface, device models.Device, test bool) Notify { @@ -287,6 +309,9 @@ func (n *Notify) SendScriptNotification(scriptUrl string) error { copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_TYPE=%s", n.Payload.DeviceType)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_SERIAL=%s", n.Payload.DeviceSerial)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.Message)) + if len(n.Payload.HostId) > 0 { + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_HOST_ID=%s", n.Payload.HostId)) + } err := utils.CmdExec(scriptPath, []string{}, "", copyEnv, "") if err != nil { n.Logger.Errorf("An error occurred while executing script %s: %v", scriptPath, err) diff --git a/webapp/backend/pkg/notify/notify_test.go b/webapp/backend/pkg/notify/notify_test.go index b891ede..c76a924 100644 --- a/webapp/backend/pkg/notify/notify_test.go +++ b/webapp/backend/pkg/notify/notify_test.go @@ -1,11 +1,13 @@ package notify import ( + "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" "github.com/stretchr/testify/require" "testing" + "time" ) func TestShouldNotify_MustSkipPassingDevices(t *testing.T) { @@ -159,3 +161,84 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresho //assert require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) } + +func TestNewPayload(t *testing.T) { + t.Parallel() + + //setup + device := models.Device{ + SerialNumber: "FAKEWDDJ324KSO", + DeviceType: pkg.DeviceProtocolAta, + DeviceName: "/dev/sda", + DeviceStatus: pkg.DeviceStatusFailedScrutiny, + } + currentTime := time.Now() + //test + + payload := NewPayload(device, false, currentTime) + + //assert + require.Equal(t, "Scrutiny SMART error (ScrutinyFailure) detected on device: /dev/sda", payload.Subject) + require.Equal(t, fmt.Sprintf(`Scrutiny SMART error notification for device: /dev/sda +Failure Type: ScrutinyFailure +Device Name: /dev/sda +Device Serial: FAKEWDDJ324KSO +Device Type: ATA + +Date: %s`, currentTime.Format(time.RFC3339)), payload.Message) +} + +func TestNewPayload_TestMode(t *testing.T) { + t.Parallel() + + //setup + device := models.Device{ + SerialNumber: "FAKEWDDJ324KSO", + DeviceType: pkg.DeviceProtocolAta, + DeviceName: "/dev/sda", + DeviceStatus: pkg.DeviceStatusFailedScrutiny, + } + currentTime := time.Now() + //test + + payload := NewPayload(device, true, currentTime) + + //assert + require.Equal(t, "Scrutiny SMART error (EmailTest) detected on device: /dev/sda", payload.Subject) + require.Equal(t, fmt.Sprintf(`TEST NOTIFICATION: +Scrutiny SMART error notification for device: /dev/sda +Failure Type: EmailTest +Device Name: /dev/sda +Device Serial: FAKEWDDJ324KSO +Device Type: ATA + +Date: %s`, currentTime.Format(time.RFC3339)), payload.Message) +} + +func TestNewPayload_WithHostId(t *testing.T) { + t.Parallel() + + //setup + device := models.Device{ + SerialNumber: "FAKEWDDJ324KSO", + DeviceType: pkg.DeviceProtocolAta, + DeviceName: "/dev/sda", + DeviceStatus: pkg.DeviceStatusFailedScrutiny, + HostId: "custom-host", + } + currentTime := time.Now() + //test + + payload := NewPayload(device, false, currentTime) + + //assert + require.Equal(t, "Scrutiny SMART error (ScrutinyFailure) detected on [host]device: [custom-host]/dev/sda", payload.Subject) + require.Equal(t, fmt.Sprintf(`Scrutiny SMART error notification for device: /dev/sda +Host Id: custom-host +Failure Type: ScrutinyFailure +Device Name: /dev/sda +Device Serial: FAKEWDDJ324KSO +Device Type: ATA + +Date: %s`, currentTime.Format(time.RFC3339)), payload.Message) +} From f823127825c638c640f2c4a18e2bf8d906a281b3 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 3 Aug 2022 22:51:44 -0700 Subject: [PATCH 24/28] simplify logger creation (move logic into a function in `main` packages) Ensure logger creation is consistent between Web and Collector Create logger in main, pass down to downstream functions (like gin) In debug mode, print a copy of AppConfig Better debugging for logger. --- .../collector-metrics/collector-metrics.go | 50 ++++++++++++------- webapp/backend/cmd/scrutiny/scrutiny.go | 42 +++++++++++++++- .../backend/pkg/web/handler/delete_device.go | 2 +- .../pkg/web/handler/get_device_details.go | 2 +- .../pkg/web/handler/get_devices_summary.go | 2 +- .../get_devices_summary_temp_history.go | 2 +- .../backend/pkg/web/handler/get_settings.go | 2 +- .../pkg/web/handler/register_devices.go | 2 +- .../backend/pkg/web/handler/save_settings.go | 2 +- .../pkg/web/handler/send_test_notification.go | 2 +- .../pkg/web/handler/upload_device_metrics.go | 2 +- webapp/backend/pkg/web/middleware/logger.go | 2 +- webapp/backend/pkg/web/server.go | 31 +++--------- 13 files changed, 89 insertions(+), 54 deletions(-) diff --git a/collector/cmd/collector-metrics/collector-metrics.go b/collector/cmd/collector-metrics/collector-metrics.go index a170f16..39d0a49 100644 --- a/collector/cmd/collector-metrics/collector-metrics.go +++ b/collector/cmd/collector-metrics/collector-metrics.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "github.com/analogj/scrutiny/collector/pkg/collector" "github.com/analogj/scrutiny/collector/pkg/config" @@ -120,26 +121,16 @@ OPTIONS: config.Set("api.endpoint", apiEndpoint) } - collectorLogger := logrus.WithFields(logrus.Fields{ - "type": "metrics", - }) - - if level, err := logrus.ParseLevel(config.GetString("log.level")); err == nil { - logrus.SetLevel(level) - } else { - logrus.SetLevel(logrus.InfoLevel) - } - - if config.IsSet("log.file") && len(config.GetString("log.file")) > 0 { - logFile, err := os.OpenFile(config.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - logrus.Errorf("Failed to open log file %s for output: %s", config.GetString("log.file"), err) - return err - } + collectorLogger, logFile, err := CreateLogger(config) + if logFile != nil { defer logFile.Close() - logrus.SetOutput(io.MultiWriter(os.Stderr, logFile)) + } + if err != nil { + return err } + settingsData, err := json.MarshalIndent(config.AllSettings(), "", "\t") + collectorLogger.Debug(string(settingsData), err) metricCollector, err := collector.CreateMetricsCollector( config, collectorLogger, @@ -192,5 +183,28 @@ OPTIONS: if err != nil { log.Fatal(color.HiRedString("ERROR: %v", err)) } - +} + +func CreateLogger(appConfig config.Interface) (*logrus.Entry, *os.File, error) { + logger := logrus.WithFields(logrus.Fields{ + "type": "metrics", + }) + + if level, err := logrus.ParseLevel(appConfig.GetString("log.level")); err == nil { + logger.Logger.SetLevel(level) + } else { + logger.Logger.SetLevel(logrus.InfoLevel) + } + + var logFile *os.File + var err error + if appConfig.IsSet("log.file") && len(appConfig.GetString("log.file")) > 0 { + logFile, err = os.OpenFile(appConfig.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + logger.Logger.Errorf("Failed to open log file %s for output: %s", appConfig.GetString("log.file"), err) + return nil, logFile, err + } + logger.Logger.SetOutput(io.MultiWriter(os.Stderr, logFile)) + } + return logger, logFile, nil } diff --git a/webapp/backend/cmd/scrutiny/scrutiny.go b/webapp/backend/cmd/scrutiny/scrutiny.go index 103fd6b..ae22677 100644 --- a/webapp/backend/cmd/scrutiny/scrutiny.go +++ b/webapp/backend/cmd/scrutiny/scrutiny.go @@ -1,12 +1,15 @@ package main import ( + "encoding/json" "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/errors" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/analogj/scrutiny/webapp/backend/pkg/web" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" + "io" + "log" "os" "time" @@ -107,7 +110,18 @@ OPTIONS: config.Set("log.file", c.String("log-file")) } - webServer := web.AppEngine{Config: config} + webLogger, logFile, err := CreateLogger(config) + if logFile != nil { + defer logFile.Close() + } + if err != nil { + return err + } + + settingsData, err := json.Marshal(config.AllSettings()) + webLogger.Debug(string(settingsData), err) + + webServer := web.AppEngine{Config: config, Logger: webLogger} return webServer.Start() }, @@ -140,3 +154,27 @@ OPTIONS: } } + +func CreateLogger(appConfig config.Interface) (*logrus.Entry, *os.File, error) { + logger := logrus.WithFields(logrus.Fields{ + "type": "web", + }) + //set default log level + if level, err := logrus.ParseLevel(appConfig.GetString("log.level")); err == nil { + logger.Logger.SetLevel(level) + } else { + logger.Logger.SetLevel(logrus.InfoLevel) + } + + var logFile *os.File + var err error + if appConfig.IsSet("log.file") && len(appConfig.GetString("log.file")) > 0 { + logFile, err = os.OpenFile(appConfig.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + logger.Logger.Errorf("Failed to open log file %s for output: %s", appConfig.GetString("log.file"), err) + return nil, logFile, err + } + logger.Logger.SetOutput(io.MultiWriter(os.Stderr, logFile)) + } + return logger, logFile, nil +} diff --git a/webapp/backend/pkg/web/handler/delete_device.go b/webapp/backend/pkg/web/handler/delete_device.go index 63336c9..f8a507d 100644 --- a/webapp/backend/pkg/web/handler/delete_device.go +++ b/webapp/backend/pkg/web/handler/delete_device.go @@ -8,7 +8,7 @@ import ( ) func DeleteDevice(c *gin.Context) { - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) err := deviceRepo.DeleteDevice(c, c.Param("wwn")) diff --git a/webapp/backend/pkg/web/handler/get_device_details.go b/webapp/backend/pkg/web/handler/get_device_details.go index 864b842..b4e24ee 100644 --- a/webapp/backend/pkg/web/handler/get_device_details.go +++ b/webapp/backend/pkg/web/handler/get_device_details.go @@ -9,7 +9,7 @@ import ( ) func GetDeviceDetails(c *gin.Context) { - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) device, err := deviceRepo.GetDeviceDetails(c, c.Param("wwn")) diff --git a/webapp/backend/pkg/web/handler/get_devices_summary.go b/webapp/backend/pkg/web/handler/get_devices_summary.go index 56e3eb5..a256f4b 100644 --- a/webapp/backend/pkg/web/handler/get_devices_summary.go +++ b/webapp/backend/pkg/web/handler/get_devices_summary.go @@ -8,7 +8,7 @@ import ( ) func GetDevicesSummary(c *gin.Context) { - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) summary, err := deviceRepo.GetSummary(c) diff --git a/webapp/backend/pkg/web/handler/get_devices_summary_temp_history.go b/webapp/backend/pkg/web/handler/get_devices_summary_temp_history.go index 631b9ec..b822dc9 100644 --- a/webapp/backend/pkg/web/handler/get_devices_summary_temp_history.go +++ b/webapp/backend/pkg/web/handler/get_devices_summary_temp_history.go @@ -8,7 +8,7 @@ import ( ) func GetDevicesSummaryTempHistory(c *gin.Context) { - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) durationKey, exists := c.GetQuery("duration_key") diff --git a/webapp/backend/pkg/web/handler/get_settings.go b/webapp/backend/pkg/web/handler/get_settings.go index b6969ea..82a1bc6 100644 --- a/webapp/backend/pkg/web/handler/get_settings.go +++ b/webapp/backend/pkg/web/handler/get_settings.go @@ -8,7 +8,7 @@ import ( ) func GetSettings(c *gin.Context) { - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) settings, err := deviceRepo.LoadSettings(c) diff --git a/webapp/backend/pkg/web/handler/register_devices.go b/webapp/backend/pkg/web/handler/register_devices.go index cb0c59b..38132ad 100644 --- a/webapp/backend/pkg/web/handler/register_devices.go +++ b/webapp/backend/pkg/web/handler/register_devices.go @@ -13,7 +13,7 @@ import ( // This function is run everytime a collector is about to start a run. It can be used to update device metadata. func RegisterDevices(c *gin.Context) { deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) var collectorDeviceWrapper models.DeviceWrapper err := c.BindJSON(&collectorDeviceWrapper) diff --git a/webapp/backend/pkg/web/handler/save_settings.go b/webapp/backend/pkg/web/handler/save_settings.go index 16de020..6706aaa 100644 --- a/webapp/backend/pkg/web/handler/save_settings.go +++ b/webapp/backend/pkg/web/handler/save_settings.go @@ -9,7 +9,7 @@ import ( ) func SaveSettings(c *gin.Context) { - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) var settings models.Settings diff --git a/webapp/backend/pkg/web/handler/send_test_notification.go b/webapp/backend/pkg/web/handler/send_test_notification.go index dae55ab..4d8e56d 100644 --- a/webapp/backend/pkg/web/handler/send_test_notification.go +++ b/webapp/backend/pkg/web/handler/send_test_notification.go @@ -13,7 +13,7 @@ import ( // Send test notification func SendTestNotification(c *gin.Context) { appConfig := c.MustGet("CONFIG").(config.Interface) - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) testNotify := notify.New( logger, diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index 82d5850..f58d6ed 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -14,7 +14,7 @@ import ( func UploadDeviceMetrics(c *gin.Context) { //db := c.MustGet("DB").(*gorm.DB) - logger := c.MustGet("LOGGER").(logrus.FieldLogger) + logger := c.MustGet("LOGGER").(*logrus.Entry) appConfig := c.MustGet("CONFIG").(config.Interface) //influxWriteDb := c.MustGet("INFLUXDB_WRITE").(*api.WriteAPIBlocking) deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) diff --git a/webapp/backend/pkg/web/middleware/logger.go b/webapp/backend/pkg/web/middleware/logger.go index 7568efe..8540d5f 100644 --- a/webapp/backend/pkg/web/middleware/logger.go +++ b/webapp/backend/pkg/web/middleware/logger.go @@ -28,7 +28,7 @@ import ( var timeFormat = "02/Jan/2006:15:04:05 -0700" // Logger is the logrus logger handler -func LoggerMiddleware(logger logrus.FieldLogger) gin.HandlerFunc { +func LoggerMiddleware(logger *logrus.Entry) gin.HandlerFunc { hostname, err := os.Hostname() if err != nil { diff --git a/webapp/backend/pkg/web/server.go b/webapp/backend/pkg/web/server.go index 0ef8bc8..bb82405 100644 --- a/webapp/backend/pkg/web/server.go +++ b/webapp/backend/pkg/web/server.go @@ -9,18 +9,17 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/web/middleware" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" - "io" "net/http" - "os" "path/filepath" "strings" ) type AppEngine struct { Config config.Interface + Logger *logrus.Entry } -func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine { +func (ae *AppEngine) Setup(logger *logrus.Entry) *gin.Engine { r := gin.New() r.Use(middleware.LoggerMiddleware(logger)) @@ -36,6 +35,10 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine { api := base.Group("/api") { api.GET("/health", func(c *gin.Context) { + //TODO: + // check if the /web folder is populated. + // check if access to influxdb + // check if access to sqlitedb. c.JSON(http.StatusOK, gin.H{ "success": true, }) @@ -77,26 +80,6 @@ func (ae *AppEngine) Start() error { gin.SetMode(gin.DebugMode) } - logger := logrus.New() - //set default log level - logLevel, err := logrus.ParseLevel(ae.Config.GetString("log.level")) - if err != nil { - return err - } - logger.SetLevel(logLevel) - //set the log file if present - if len(ae.Config.GetString("log.file")) != 0 { - logFile, err := os.OpenFile(ae.Config.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644) - defer logFile.Close() - if err != nil { - logrus.Errorf("Failed to open log file %s for output: %s", ae.Config.GetString("log.file"), err) - return err - } - - //configure the logrus default - logger.SetOutput(io.MultiWriter(os.Stderr, logFile)) - } - //check if the database parent directory exists, fail here rather than in a handler. if !utils.FileExists(filepath.Dir(ae.Config.GetString("web.database.location"))) { return errors.ConfigValidationError(fmt.Sprintf( @@ -104,7 +87,7 @@ func (ae *AppEngine) Start() error { filepath.Dir(ae.Config.GetString("web.database.location")))) } - r := ae.Setup(logger) + r := ae.Setup(ae.Logger) return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port"))) } From 51f59e4fcdb719310037ebbf4b8e9d4be1ab8abc Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 3 Aug 2022 22:59:19 -0700 Subject: [PATCH 25/28] docs, added an explanation for why influxdb is required. --- docs/TROUBLESHOOTING_INFLUXDB.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/TROUBLESHOOTING_INFLUXDB.md b/docs/TROUBLESHOOTING_INFLUXDB.md index 22b52c6..faa9d4d 100644 --- a/docs/TROUBLESHOOTING_INFLUXDB.md +++ b/docs/TROUBLESHOOTING_INFLUXDB.md @@ -1,7 +1,19 @@ # InfluxDB Troubleshooting -## Installation -InfluxDB is a required dependency for Scrutiny v0.4.0+. +## Why?? + +Scrutiny has many features, but the relevant one to this conversation is the "S.M.A.R.T metric tracking for historical +trends". Basically Scrutiny not only shows you the current SMART values, but how they've changed over weeks, months (or +even years). + +To efficiently handle that data at scale (and to make my life easier as a developer) I decided to add InfluxDB as a +dependency. It's a dedicated timeseries database, as opposed to the general purpose sqlite DB I used before. I also did +a bunch of testing and analysis before I made the change. With InfluxDB the memory footprint for Scrutiny (at idle) is ~ +100mb, which is still fairly reasonable. + +## Installation + +InfluxDB is a required dependency for Scrutiny v0.4.0+. https://docs.influxdata.com/influxdb/v2.2/install/ From 59e2e928a8c430b9913fa9c5c98faf6d6afec04a Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 3 Aug 2022 23:12:09 -0700 Subject: [PATCH 26/28] remove the notify.level and notify.filter_attributes values from the example.scrutiny.yaml, since they are no longer allowed. --- example.scrutiny.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml index b73b711..c93725e 100644 --- a/example.scrutiny.yaml +++ b/example.scrutiny.yaml @@ -73,8 +73,6 @@ log: # - "join://shoutrrr:api-key@join/?devices=device1[,device2, ...][&icon=icon][&title=title]" # - "script:///file/path/on/disk" # - "https://www.example.com/path" -# filter_attributes: 'all' # options: 'all' or 'critical' -# level: 'fail' # options: 'fail', 'fail_scrutiny', 'fail_smart' ######################################################################################################################## # FEATURES COMING SOON From 15d3206f6f765e89535d44ecf1c5cc1c49c0fd44 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 4 Aug 2022 07:30:14 -0700 Subject: [PATCH 27/28] remove settings dialog from Details page. --- webapp/frontend/src/app/modules/detail/detail.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/frontend/src/app/modules/detail/detail.component.html b/webapp/frontend/src/app/modules/detail/detail.component.html index 5c94592..0f37778 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.html +++ b/webapp/frontend/src/app/modules/detail/detail.component.html @@ -17,7 +17,7 @@ Export