ensure that users can filter their notifications by:
- failing attribute type (Critical vs All) - failure reason (Smart, Scrutiny, Both) fixes #300
This commit is contained in:
Jason Kulatunga
@@ -73,6 +73,8 @@ log:
|
||||
# - "join://shoutrrr:api-key@join/?devices=device1[,device2, ...][&icon=icon][&title=title]"
|
||||
# - "script:///file/path/on/disk"
|
||||
# - "https://www.example.com/path"
|
||||
# filter_attributes: 'all' # options: 'all' or 'critical'
|
||||
# level: 'fail' # options: 'fail', 'fail_scrutiny', 'fail_smart'
|
||||
|
||||
########################################################################################################################
|
||||
# FEATURES COMING SOON
|
||||
|
||||
@@ -2,6 +2,7 @@ package config
|
||||
|
||||
import (
|
||||
"github.com/analogj/go-util/utils"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
|
||||
"github.com/spf13/viper"
|
||||
"log"
|
||||
@@ -38,6 +39,8 @@ func (c *configuration) Init() error {
|
||||
c.SetDefault("log.file", "")
|
||||
|
||||
c.SetDefault("notify.urls", []string{})
|
||||
c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll)
|
||||
c.SetDefault("notify.level", pkg.NotifyLevelFail)
|
||||
|
||||
c.SetDefault("web.influxdb.scheme", "http")
|
||||
c.SetDefault("web.influxdb.host", "localhost")
|
||||
|
||||
@@ -4,6 +4,13 @@ const DeviceProtocolAta = "ATA"
|
||||
const DeviceProtocolScsi = "SCSI"
|
||||
const DeviceProtocolNvme = "NVMe"
|
||||
|
||||
const NotifyFilterAttributesAll = "all"
|
||||
const NotifyFilterAttributesCritical = "critical"
|
||||
|
||||
const NotifyLevelFail = "fail"
|
||||
const NotifyLevelFailScrutiny = "fail_scrutiny"
|
||||
const NotifyLevelFailSmart = "fail_smart"
|
||||
|
||||
type AttributeStatus uint8
|
||||
|
||||
const (
|
||||
|
||||
@@ -6,7 +6,11 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/analogj/go-util/utils"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
|
||||
"github.com/containrrr/shoutrrr"
|
||||
shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -14,28 +18,130 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const NotifyFailureTypeEmailTest = "EmailTest"
|
||||
const NotifyFailureTypeSmartPrefail = "SmartPreFailure"
|
||||
const NotifyFailureTypeBothFailure = "SmartFailure" //SmartFailure always takes precedence when Scrutiny & Smart failed.
|
||||
const NotifyFailureTypeSmartFailure = "SmartFailure"
|
||||
const NotifyFailureTypeSmartErrorLog = "SmartErrorLog"
|
||||
const NotifyFailureTypeSmartSelfTest = "SmartSelfTestLog"
|
||||
const NotifyFailureTypeScrutinyFailure = "ScrutinyFailure"
|
||||
|
||||
// ShouldNotify check if the error Message should be filtered (level mismatch or filtered_attributes)
|
||||
func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLevel string, notifyFilterAttributes string) bool {
|
||||
// 1. check if the device is healthy
|
||||
if device.DeviceStatus == pkg.DeviceStatusPassed {
|
||||
return false
|
||||
}
|
||||
|
||||
// setup constants for comparison
|
||||
var requiredDeviceStatus pkg.DeviceStatus
|
||||
var requiredAttrStatus pkg.AttributeStatus
|
||||
if notifyLevel == pkg.NotifyLevelFail {
|
||||
// either scrutiny or smart failures should trigger an email
|
||||
requiredDeviceStatus = pkg.DeviceStatusSet(pkg.DeviceStatusFailedSmart, pkg.DeviceStatusFailedScrutiny)
|
||||
requiredAttrStatus = pkg.AttributeStatusSet(pkg.AttributeStatusFailedSmart, pkg.AttributeStatusFailedScrutiny)
|
||||
} else if notifyLevel == pkg.NotifyLevelFailSmart {
|
||||
//only smart failures
|
||||
requiredDeviceStatus = pkg.DeviceStatusFailedSmart
|
||||
requiredAttrStatus = pkg.AttributeStatusFailedSmart
|
||||
} else {
|
||||
requiredDeviceStatus = pkg.DeviceStatusFailedScrutiny
|
||||
requiredAttrStatus = pkg.AttributeStatusFailedScrutiny
|
||||
}
|
||||
|
||||
// 2. check if the attributes that are failing should be filtered (non-critical)
|
||||
// 3. for any unfiltered attribute, store the failure reason (Smart or Scrutiny)
|
||||
if notifyFilterAttributes == pkg.NotifyFilterAttributesCritical {
|
||||
hasFailingCriticalAttr := false
|
||||
var statusFailingCrtiticalAttr pkg.AttributeStatus
|
||||
|
||||
for attrId, attrData := range smartAttrs.Attributes {
|
||||
//find failing attribute
|
||||
if attrData.GetStatus() == pkg.AttributeStatusPassed {
|
||||
continue //skip all passing attributes
|
||||
}
|
||||
|
||||
// merge the status's of all critical attributes
|
||||
statusFailingCrtiticalAttr = pkg.AttributeStatusSet(statusFailingCrtiticalAttr, attrData.GetStatus())
|
||||
|
||||
//found a failing attribute, see if its critical
|
||||
if device.IsScsi() && thresholds.ScsiMetadata[attrId].Critical {
|
||||
hasFailingCriticalAttr = true
|
||||
} else if device.IsNvme() && thresholds.NmveMetadata[attrId].Critical {
|
||||
hasFailingCriticalAttr = true
|
||||
} else {
|
||||
//this is ATA
|
||||
attrIdInt, err := strconv.Atoi(attrId)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if thresholds.AtaMetadata[attrIdInt].Critical {
|
||||
hasFailingCriticalAttr = true
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if !hasFailingCriticalAttr {
|
||||
//no critical attributes are failing, and notifyFilterAttributes == "critical"
|
||||
return false
|
||||
} else {
|
||||
// check if any of the critical attributes have a status that we're looking for
|
||||
return pkg.AttributeStatusHas(statusFailingCrtiticalAttr, requiredAttrStatus)
|
||||
}
|
||||
|
||||
} else {
|
||||
// 2. SKIP - we are processing every attribute.
|
||||
// 3. check if the device failure level matches the wanted failure level.
|
||||
return pkg.DeviceStatusHas(device.DeviceStatus, requiredDeviceStatus)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: include host and/or user label for device.
|
||||
type Payload struct {
|
||||
Date string `json:"date"` //populated by Send function.
|
||||
FailureType string `json:"failure_type"` //EmailTest, SmartFail, ScrutinyFail
|
||||
DeviceType string `json:"device_type"` //ATA/SCSI/NVMe
|
||||
DeviceName string `json:"device_name"` //dev/sda
|
||||
DeviceSerial string `json:"device_serial"` //WDDJ324KSO
|
||||
Test bool `json:"test"` // false
|
||||
|
||||
//should not be populated
|
||||
Subject string `json:"subject"`
|
||||
Message string `json:"message"`
|
||||
//private, populated during init (marked as Public for JSON serialization)
|
||||
Date string `json:"date"` //populated by Send function.
|
||||
FailureType string `json:"failure_type"` //EmailTest, BothFail, SmartFail, ScrutinyFail
|
||||
Subject string `json:"subject"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func NewPayload(device models.Device, test bool) Payload {
|
||||
payload := Payload{
|
||||
DeviceType: device.DeviceType,
|
||||
DeviceName: device.DeviceName,
|
||||
DeviceSerial: device.SerialNumber,
|
||||
Test: test,
|
||||
}
|
||||
|
||||
//validate that the Payload is populated
|
||||
sendDate := time.Now()
|
||||
payload.Date = sendDate.Format(time.RFC3339)
|
||||
payload.FailureType = payload.GenerateFailureType(device.DeviceStatus)
|
||||
payload.Subject = payload.GenerateSubject()
|
||||
payload.Message = payload.GenerateMessage()
|
||||
return payload
|
||||
}
|
||||
|
||||
func (p *Payload) GenerateFailureType(deviceStatus pkg.DeviceStatus) string {
|
||||
//generate a failure type, given Test and DeviceStatus
|
||||
if p.Test {
|
||||
return NotifyFailureTypeEmailTest // must be an email test if "Test" is true
|
||||
}
|
||||
if pkg.DeviceStatusHas(deviceStatus, pkg.DeviceStatusFailedSmart) && pkg.DeviceStatusHas(deviceStatus, pkg.DeviceStatusFailedScrutiny) {
|
||||
return NotifyFailureTypeBothFailure //both failed
|
||||
} else if pkg.DeviceStatusHas(deviceStatus, pkg.DeviceStatusFailedSmart) {
|
||||
return NotifyFailureTypeSmartFailure //only SMART failed
|
||||
} else {
|
||||
return NotifyFailureTypeScrutinyFailure //only Scrutiny failed
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Payload) GenerateSubject() string {
|
||||
@@ -61,6 +167,14 @@ Date: %s`, p.DeviceName, p.FailureType, p.DeviceName, p.DeviceSerial, p.DeviceTy
|
||||
return message
|
||||
}
|
||||
|
||||
func New(logger logrus.FieldLogger, appconfig config.Interface, device models.Device, test bool) Notify {
|
||||
return Notify{
|
||||
Logger: logger,
|
||||
Config: appconfig,
|
||||
Payload: NewPayload(device, test),
|
||||
}
|
||||
}
|
||||
|
||||
type Notify struct {
|
||||
Logger logrus.FieldLogger
|
||||
Config config.Interface
|
||||
@@ -68,11 +182,6 @@ type Notify struct {
|
||||
}
|
||||
|
||||
func (n *Notify) Send() error {
|
||||
//validate that the Payload is populated
|
||||
sendDate := time.Now()
|
||||
n.Payload.Date = sendDate.Format(time.RFC3339)
|
||||
n.Payload.Subject = n.Payload.GenerateSubject()
|
||||
n.Payload.Message = n.Payload.GenerateMessage()
|
||||
|
||||
//retrieve list of notification endpoints from config file
|
||||
configUrls := n.Config.GetStringSlice("notify.urls")
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
|
||||
"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
|
||||
"github.com/stretchr/testify/require"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestShouldNotify_MustSkipPassingDevices(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusPassed,
|
||||
}
|
||||
smartAttrs := measurements.Smart{}
|
||||
notifyLevel := pkg.NotifyLevelFail
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesAll
|
||||
|
||||
//assert
|
||||
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
|
||||
func TestShouldNotify_NotifyLevelFail_FailingSmartDevice(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusFailedSmart,
|
||||
}
|
||||
smartAttrs := measurements.Smart{}
|
||||
notifyLevel := pkg.NotifyLevelFail
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesAll
|
||||
|
||||
//assert
|
||||
require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
|
||||
func TestShouldNotify_NotifyLevelFailSmart_FailingSmartDevice(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusFailedSmart,
|
||||
}
|
||||
smartAttrs := measurements.Smart{}
|
||||
notifyLevel := pkg.NotifyLevelFailSmart
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesAll
|
||||
|
||||
//assert
|
||||
require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
|
||||
func TestShouldNotify_NotifyLevelFailScrutiny_FailingSmartDevice(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusFailedSmart,
|
||||
}
|
||||
smartAttrs := measurements.Smart{}
|
||||
notifyLevel := pkg.NotifyLevelFailScrutiny
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesAll
|
||||
|
||||
//assert
|
||||
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
|
||||
func TestShouldNotify_NotifyFilterAttributesCritical_WithCriticalAttrs(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusFailedSmart,
|
||||
}
|
||||
smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
|
||||
"5": &measurements.SmartAtaAttribute{
|
||||
Status: pkg.AttributeStatusFailedSmart,
|
||||
},
|
||||
}}
|
||||
notifyLevel := pkg.NotifyLevelFail
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
|
||||
|
||||
//assert
|
||||
require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
|
||||
func TestShouldNotify_NotifyFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusFailedSmart,
|
||||
}
|
||||
smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
|
||||
"5": &measurements.SmartAtaAttribute{
|
||||
Status: pkg.AttributeStatusPassed,
|
||||
},
|
||||
"10": &measurements.SmartAtaAttribute{
|
||||
Status: pkg.AttributeStatusFailedScrutiny,
|
||||
},
|
||||
}}
|
||||
notifyLevel := pkg.NotifyLevelFail
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
|
||||
|
||||
//assert
|
||||
require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
|
||||
func TestShouldNotify_NotifyFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusFailedSmart,
|
||||
}
|
||||
smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
|
||||
"1": &measurements.SmartAtaAttribute{
|
||||
Status: pkg.AttributeStatusFailedSmart,
|
||||
},
|
||||
}}
|
||||
notifyLevel := pkg.NotifyLevelFail
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
|
||||
|
||||
//assert
|
||||
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
|
||||
func TestShouldNotify_NotifyFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusFailedSmart,
|
||||
}
|
||||
smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
|
||||
"5": &measurements.SmartAtaAttribute{
|
||||
Status: pkg.AttributeStatusPassed,
|
||||
},
|
||||
}}
|
||||
notifyLevel := pkg.NotifyLevelFail
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
|
||||
|
||||
//assert
|
||||
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
|
||||
func TestShouldNotify_NotifyFilterAttributesCritical_NotifyLevelFailSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) {
|
||||
t.Parallel()
|
||||
//setup
|
||||
device := models.Device{
|
||||
DeviceStatus: pkg.DeviceStatusFailedSmart,
|
||||
}
|
||||
smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{
|
||||
"5": &measurements.SmartAtaAttribute{
|
||||
Status: pkg.AttributeStatusPassed,
|
||||
},
|
||||
"10": &measurements.SmartAtaAttribute{
|
||||
Status: pkg.AttributeStatusFailedScrutiny,
|
||||
},
|
||||
}}
|
||||
notifyLevel := pkg.NotifyLevelFailSmart
|
||||
notifyFilterAttributes := pkg.NotifyFilterAttributesCritical
|
||||
|
||||
//assert
|
||||
require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes))
|
||||
}
|
||||
@@ -15,17 +15,16 @@ func SendTestNotification(c *gin.Context) {
|
||||
appConfig := c.MustGet("CONFIG").(config.Interface)
|
||||
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
|
||||
|
||||
testNotify := notify.Notify{
|
||||
Logger: logger,
|
||||
Config: appConfig,
|
||||
Payload: notify.Payload{
|
||||
FailureType: "EmailTest",
|
||||
DeviceSerial: "FAKEWDDJ324KSO",
|
||||
testNotify := notify.New(
|
||||
logger,
|
||||
appConfig,
|
||||
models.Device{
|
||||
SerialNumber: "FAKEWDDJ324KSO",
|
||||
DeviceType: pkg.DeviceProtocolAta,
|
||||
DeviceName: "/dev/sda",
|
||||
Test: true,
|
||||
},
|
||||
}
|
||||
true,
|
||||
)
|
||||
err := testNotify.Send()
|
||||
if err != nil {
|
||||
logger.Errorln("An error occurred while sending test notification", err)
|
||||
|
||||
@@ -63,20 +63,16 @@ func UploadDeviceMetrics(c *gin.Context) {
|
||||
}
|
||||
|
||||
//check for error
|
||||
if updatedDevice.DeviceStatus != pkg.DeviceStatusPassed {
|
||||
if notify.ShouldNotify(updatedDevice, smartData, appConfig.GetString("notify.level"), appConfig.GetString("notify.filter_attributes")) {
|
||||
//send notifications
|
||||
testNotify := notify.Notify{
|
||||
Config: appConfig,
|
||||
Payload: notify.Payload{
|
||||
FailureType: notify.NotifyFailureTypeSmartFailure,
|
||||
DeviceName: updatedDevice.DeviceName,
|
||||
DeviceType: updatedDevice.DeviceProtocol,
|
||||
DeviceSerial: updatedDevice.SerialNumber,
|
||||
Test: false,
|
||||
},
|
||||
Logger: logger,
|
||||
}
|
||||
_ = testNotify.Send() //we ignore error message when sending notifications.
|
||||
|
||||
liveNotify := notify.New(
|
||||
logger,
|
||||
appConfig,
|
||||
updatedDevice,
|
||||
false,
|
||||
)
|
||||
_ = liveNotify.Send() //we ignore error message when sending notifications.
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
|
||||
Reference in New Issue
Block a user