gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP settings system.

Browse Source 
- updated dbdiagrams schema
- [BREAKING] force failure if `notify.filter_attributes` or `notify.level` is set
- added Settings table (and default values during migration)
- Added Save Settings and Get Settings functions.
- Added web API endpoints for getting and saving settings.
- Deprecated old Notify* constants. Created new MetricsStatus* and MetricsNotifyLevel constants.
This commit is contained in:
Jason Kulatunga
2022-07-17 10:32:28 -07:00
parent dd0c3e6fba
commit 99af2b8b16
14 changed files with 312 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/INSTALL_NAS.md.go → docs/INSTALL_NAS.md
View File
docs/dbdiagram.io.txt
+70 -44
View File
@@ -1,62 +1,88 @@
// SQLite Table(s)
Table device {
created_at timestamp
wwn varchar [pk]
Table Device {
//GORM attributes, see: http://gorm.io/docs/conventions.html
CreatedAt time
UpdatedAt time
DeletedAt time
//user provided
label varchar
host_id varchar
WWN string
// smartctl provided
device_name varchar
manufacturer varchar
model_name varchar
interface_type varchar
interface_speed varchar
serial_number varchar
firmware varchar
rotational_speed varchar
capacity varchar
form_factor varchar
smart_support varchar
device_protocol varchar
device_type varchar
DeviceName string
DeviceUUID string
DeviceSerialID string
DeviceLabel string
Manufacturer string
ModelName string
InterfaceType string
InterfaceSpeed string
SerialNumber string
Firmware string
RotationSpeed int
Capacity int64
FormFactor string
SmartSupport bool
DeviceProtocol string//protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
DeviceType string//device type is used for querying with -d/t flag, should only be used by collector.
// User provided metadata
Label string
HostId string
// Data set by Scrutiny
DeviceStatus enum
}
Table Setting {
//GORM attributes, see: http://gorm.io/docs/conventions.html
SettingKeyName string
SettingKeyDescription string
SettingDataType string
SettingValueNumeric int64
SettingValueString string
}
// InfluxDB Tables
Table device_temperature {
//timestamp
created_at timestamp
//tags (indexed & queryable)
device_wwn varchar [pk]
//fields
temp bigint
}
Table SmartTemperature {
Date time
DeviceWWN string //(tag)
Temp int64
}
Table smart_ata_results {
//timestamp
created_at timestamp
Table Smart {
Date time
DeviceWWN string //(tag)
DeviceProtocol string
//tags (indexed & queryable)
device_wwn varchar [pk]
smart_status varchar
scrutiny_status varchar
//Metrics (fields)
Temp int64
PowerOnHours int64
PowerCycleCount int64
//Smart Status
Status enum
//fields
temp bigint
power_on_hours bigint
power_cycle_count bigint
//SMART Attributes (fields)
Attr_ID_AttributeId int
Attr_ID_Value int64
Attr_ID_Threshold int64
Attr_ID_Worst int64
Attr_ID_RawValue int64
Attr_ID_RawString string
Attr_ID_WhenFailed string
//Generated data
Attr_ID_TransformedValue int64
Attr_ID_Status enum
Attr_ID_StatusReason string
Attr_ID_FailureRate float64
}
Ref: device.wwn < smart_ata_results.device_wwn
Ref: Device.WWN < Smart.DeviceWWN
Ref: Device.WWN < SmartTemperature.DeviceWWN
webapp/backend/pkg/config/config.go
+13 -33
View File
@@ -2,7 +2,6 @@ package config
import (
"github.com/analogj/go-util/utils"
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
"github.com/spf13/viper"
"log"
@@ -39,8 +38,6 @@ func (c *configuration) Init() error {
c.SetDefault("log.file", "")
c.SetDefault("notify.urls", []string{})
c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll)
c.SetDefault("notify.level", pkg.NotifyLevelFail)
c.SetDefault("web.influxdb.scheme", "http")
c.SetDefault("web.influxdb.host", "localhost")
@@ -55,17 +52,6 @@ func (c *configuration) Init() error {
//c.SetDefault("disks.include", []string{})
//c.SetDefault("disks.exclude", []string{})
//c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh")
//c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh")
//c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh")
//c.SetDefault("collect.metric.enable", true)
//c.SetDefault("collect.metric.command", "-a -o on -S on")
//c.SetDefault("collect.long.enable", true)
//c.SetDefault("collect.long.command", "-a -o on -S on")
//c.SetDefault("collect.short.enable", true)
//c.SetDefault("collect.short.command", "-a -o on -S on")
//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
c.SetConfigType("yaml")
//c.SetConfigName("drawbridge")
@@ -77,7 +63,7 @@ func (c *configuration) Init() error {
c.AutomaticEnv()
//CLI options will be added via the `Set()` function
return nil
return c.ValidateConfig()
}
func (c *configuration) ReadConfig(configFilePath string) error {
@@ -120,24 +106,18 @@ func (c *configuration) ReadConfig(configFilePath string) error {
// This function ensures that the merged config works correctly.
func (c *configuration) ValidateConfig() error {
////deserialize Questions
//questionsMap := map[string]Question{}
//err := c.UnmarshalKey("questions", &questionsMap)
//
//if err != nil {
// log.Printf("questions could not be deserialized correctly. %v", err)
// return err
//}
//
//for _, v := range questionsMap {
//
// typeContent, ok := v.Schema["type"].(string)
// if !ok || len(typeContent) == 0 {
// return errors.QuestionSyntaxError("`type` is required for questions")
// }
//}
//
//
//the following keys are deprecated, and no longer supported
/*
- notify.filter_attributes (replaced by metrics.status.filter_attributes SETTING)
- notify.level (replaced by metrics.notify.level and metrics.status.threshold SETTING)
*/
//TODO add docs and upgrade doc.
if c.IsSet("notify.filter_attributes") {
return errors.ConfigValidationError("`notify.filter_attributes` configuration option is deprecated. Replaced by option in Dashboard Settings page")
}
if c.IsSet("notify.level") {
return errors.ConfigValidationError("`notify.level` configuration option is deprecated. Replaced by option in Dashboard Settings page")
}
return nil
}
webapp/backend/pkg/constants.go
+45 -9
View File
@@ -4,17 +4,11 @@ const DeviceProtocolAta = "ATA"
const DeviceProtocolScsi = "SCSI"
const DeviceProtocolNvme = "NVMe"
const NotifyFilterAttributesAll = "all"
const NotifyFilterAttributesCritical = "critical"
const NotifyLevelFail = "fail"
const NotifyLevelFailScrutiny = "fail_scrutiny"
const NotifyLevelFailSmart = "fail_smart"
//go:generate stringer -type=AttributeStatus
// AttributeStatus bitwise flag, 1,2,4,8,16,32,etc
type AttributeStatus uint8
const (
// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
AttributeStatusPassed AttributeStatus = 0
AttributeStatusFailedSmart AttributeStatus = 1
AttributeStatusWarningScrutiny AttributeStatus = 2
@@ -30,9 +24,10 @@ func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^
func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 }
//go:generate stringer -type=DeviceStatus
// DeviceStatus bitwise flag, 1,2,4,8,16,32,etc
type DeviceStatus uint8
const (
// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
DeviceStatusPassed DeviceStatus = 0
DeviceStatusFailedSmart DeviceStatus = 1
DeviceStatusFailedScrutiny DeviceStatus = 2
@@ -42,3 +37,44 @@ func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus { return b | flag }
func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus { return b &^ flag }
func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
func DeviceStatusHas(b, flag DeviceStatus) bool { return b&flag != 0 }
// Metrics Specific Filtering & Threshold Constants
type MetricsNotifyLevel int64
const (
MetricsNotifyLevelWarn MetricsNotifyLevel = 1
MetricsNotifyLevelFail MetricsNotifyLevel = 2
)
type MetricsStatusFilterAttributes int64
const (
MetricsStatusFilterAttributesAll MetricsStatusFilterAttributes = 0
MetricsStatusFilterAttributesCritical MetricsStatusFilterAttributes = 1
)
// MetricsStatusThreshold bitwise flag, 1,2,4,8,16,32,etc
type MetricsStatusThreshold int64
const (
MetricsStatusThresholdSmart MetricsStatusThreshold = 1
MetricsStatusThresholdScrutiny MetricsStatusThreshold = 2
//shortcut
MetricsStatusThresholdBoth MetricsStatusThreshold = 3
)
// Deprecated
const NotifyFilterAttributesAll = "all"
// Deprecated
const NotifyFilterAttributesCritical = "critical"
// Deprecated
const NotifyLevelFail = "fail"
// Deprecated
const NotifyLevelFailScrutiny = "fail_scrutiny"
// Deprecated
const NotifyLevelFailSmart = "fail_smart"
webapp/backend/pkg/database/interface.go
+3
View File
@@ -28,4 +28,7 @@ type DeviceRepo interface {
GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
GetSettings(ctx context.Context) (*models.Settings, error)
SaveSettings(ctx context.Context, settings models.Settings) error
}
webapp/backend/pkg/database/migrations/m20220716214900/setting.go
+3 -2
View File
@@ -8,8 +8,9 @@ type Setting struct {
//GORM attributes, see: http://gorm.io/docs/conventions.html
gorm.Model
SettingKeyName string `json:"setting_key_name"`
SettingDataType string `json:"setting_data_type"`
SettingKeyName string `json:"setting_key_name"`
SettingKeyDescription string `json:"setting_key_description"`
SettingDataType string `json:"setting_data_type"`
SettingValueNumeric int64 `json:"setting_value_numeric"`
SettingValueString string `json:"setting_value_string"`
webapp/backend/pkg/database/scrutiny_repository_migrations.go
+28 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100"
@@ -281,7 +282,33 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
Migrate: func(tx *gorm.DB) error {
// adding the settings table.
return tx.AutoMigrate(m20220716214900.Setting{})
err := tx.AutoMigrate(m20220716214900.Setting{})
if err != nil {
return err
}
//add defaults.
var defaultSettings = []m20220716214900.Setting{
{
SettingKeyName: "metrics.notify.level",
SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)",
SettingDataType: "numeric",
SettingValueNumeric: int64(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn'
},
{
SettingKeyName: "metrics.status.filter_attributes",
SettingKeyDescription: "Determines which attributes should impact device status",
SettingDataType: "numeric",
SettingValueNumeric: int64(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or 'critical'
},
{
SettingKeyName: "metrics.status.threshold",
SettingKeyDescription: "Determines which threshold should impact device status",
SettingDataType: "string",
SettingValueNumeric: int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both'
},
}
return tx.Create(&defaultSettings).Error
},
},
})
webapp/backend/pkg/database/scrutiny_repository_settings.go
+33
View File
@@ -0,0 +1,33 @@
package database
import (
"context"
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
)
func (sr *scrutinyRepository) GetSettings(ctx context.Context) (*models.Settings, error) {
settingsEntries := []models.SettingEntry{}
if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
return nil, fmt.Errorf("Could not get settings from DB: %v", err)
}
settings := models.Settings{}
settings.PopulateFromSettingEntries(settingsEntries)
return &settings, nil
}
func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error {
//get current settings
settingsEntries := []models.SettingEntry{}
if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
return fmt.Errorf("Could not get settings from DB: %v", err)
}
// override with values from settings object
settingsEntries = settings.UpdateSettingEntries(settingsEntries)
// store in database.
return sr.gormClient.Updates(&settingsEntries).Error
}
webapp/backend/pkg/models/setting.go
-16
View File
@@ -1,16 +0,0 @@
package models
import (
"gorm.io/gorm"
)
type Setting struct {
//GORM attributes, see: http://gorm.io/docs/conventions.html
gorm.Model
SettingKeyName string `json:"setting_key_name"`
SettingDataType string `json:"setting_data_type"`
SettingValueNumeric int64 `json:"setting_value_numeric"`
SettingValueString string `json:"setting_value_string"`
}
webapp/backend/pkg/models/setting_entry.go
+22
View File
@@ -0,0 +1,22 @@
package models
import (
"gorm.io/gorm"
)
// SettingEntry matches a setting row in the database
type SettingEntry struct {
//GORM attributes, see: http://gorm.io/docs/conventions.html
gorm.Model
SettingKeyName string `json:"setting_key_name" gorm:"unique;not null"`
SettingKeyDescription string `json:"setting_key_description"`
SettingDataType string `json:"setting_data_type"`
SettingValueNumeric int64 `json:"setting_value_numeric"`
SettingValueString string `json:"setting_value_string"`
}
func (s SettingEntry) TableName() string {
return "settings"
}
webapp/backend/pkg/models/settings.go
+35
View File
@@ -0,0 +1,35 @@
package models
import "github.com/analogj/scrutiny/webapp/backend/pkg"
// Settings is made up of parsed SettingEntry objects retrieved from the database
type Settings struct {
MetricsNotifyLevel pkg.MetricsNotifyLevel `json:"metrics_notify_level"`
MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics_status_filter_attributes"`
MetricsStatusThreshold pkg.MetricsStatusThreshold `json:"metrics_status_threshold"`
}
func (s *Settings) PopulateFromSettingEntries(entries []SettingEntry) {
for _, entry := range entries {
if entry.SettingKeyName == "metrics.notify.level" {
s.MetricsNotifyLevel = pkg.MetricsNotifyLevel(entry.SettingValueNumeric)
} else if entry.SettingKeyName == "metrics.status.filter_attributes" {
s.MetricsStatusFilterAttributes = pkg.MetricsStatusFilterAttributes(entry.SettingValueNumeric)
} else if entry.SettingKeyName == "metrics.status.threshold" {
s.MetricsStatusThreshold = pkg.MetricsStatusThreshold(entry.SettingValueNumeric)
}
}
}
func (s *Settings) UpdateSettingEntries(entries []SettingEntry) []SettingEntry {
for _, entry := range entries {
if entry.SettingKeyName == "metrics.notify.level" {
entry.SettingValueNumeric = int64(s.MetricsNotifyLevel)
} else if entry.SettingKeyName == "metrics.status.filter_attributes" {
entry.SettingValueNumeric = int64(s.MetricsStatusFilterAttributes)
} else if entry.SettingKeyName == "metrics.status.threshold" {
entry.SettingValueNumeric = int64(s.MetricsStatusThreshold)
}
}
return entries
}
webapp/backend/pkg/web/handler/get_settings.go
+25
View File
@@ -0,0 +1,25 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"net/http"
)
func GetSettings(c *gin.Context) {
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
settings, err := deviceRepo.GetSettings(c)
if err != nil {
logger.Errorln("An error occurred while retrieving settings", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"settings": settings,
})
}
webapp/backend/pkg/web/handler/save_settings.go
+33
View File
@@ -0,0 +1,33 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"net/http"
)
func SaveSettings(c *gin.Context) {
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
var settings models.Settings
err := c.BindJSON(&settings)
if err != nil {
logger.Errorln("Cannot parse updated settings", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
err = deviceRepo.SaveSettings(c, settings)
if err != nil {
logger.Errorln("An error occurred while saving settings", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
})
}
webapp/backend/pkg/web/server.go
+2
View File
@@ -50,6 +50,8 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details
api.DELETE("/device/:wwn", handler.DeleteDevice) //used by UI to delete device
api.GET("/settings", handler.GetSettings) //used to get settings
api.POST("/settings", handler.SaveSettings) //used to save settings
}
}