sending notifications on failure.

example.scrutiny.yaml

@@ -32,19 +32,12 @@ log:
   file: '' #absolute or relative paths allowed, eg. web.log
   level: INFO
 
-# The following commented out sections are a preview of additional configuration options that will be available soon.
-
-#disks:
-#  include:
-#    # - /dev/sda
-#  exclude:
-#    # - /dev/sdb
 
 #notify:
 #  urls:
 #    - "discord://token@channel"
 #    - "telegram://token@telegram?channels=channel-1[,channel-2,...]"
-#    - "pushover://shoutrrr:apiToken@userKey/?devices=device1[,device2, ...]"
+#    - "pushover://shoutrrr:apiToken@userKey/?priority=1&devices=device1[,device2, ...]"
 #    - "slack://[botname@]token-a/token-b/token-c"
 #    - "smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]"
 #    - "teams://token-a/token-b/token-c"
@@ -58,6 +51,15 @@ log:
 #    - "script:///file/path/on/disk"
 #    - "https://www.example.com/path"
 
+
+# The following commented out sections are a preview of additional configuration options that will be available soon.
+
+#disks:
+#  include:
+#    # - /dev/sda
+#  exclude:
+#    # - /dev/sdb
+
 #limits:
 #  ata:
 #    critical:

webapp/backend/pkg/notify/notify.go

@@ -7,26 +7,57 @@ import (
 	"github.com/analogj/go-util/utils"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/containrrr/shoutrrr"
-	log "github.com/sirupsen/logrus"
+	shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types"
+	"github.com/sirupsen/logrus"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"sync"
 	"time"
 )
 
+const NotifyFailureTypeEmailTest = "EmailTest"
+const NotifyFailureTypeSmartPrefail = "SmartPreFailure"
+const NotifyFailureTypeSmartFailure = "SmartFailure"
+const NotifyFailureTypeSmartErrorLog = "SmartErrorLog"
+const NotifyFailureTypeSmartSelfTest = "SmartSelfTestLog"
+
+// TODO: include host and/or user label for device.
 type Payload struct {
-	Mailer       string `json:"mailer"`
+	Date         string `json:"date"`          //populated by Send function.
-	Subject      string `json:"subject"`
+	FailureType  string `json:"failure_type"`  //EmailTest, SmartFail, ScrutinyFail
-	Date         string `json:"date"`
+	DeviceType   string `json:"device_type"`   //ATA/SCSI/NVMe
-	FailureType  string `json:"failure_type"`
+	DeviceName   string `json:"device_string"` //dev/sda
-	Device       string `json:"device"`
+	DeviceSerial string `json:"device"`        //WDDJ324KSO
-	DeviceType   string `json:"device_type"`
+	Test         bool   `json:"-"`             // false
-	DeviceString string `json:"device_string"`
+}
-	Message      string `json:"message"`
+
+func (p *Payload) GenerateMessage() string {
+	//generate a detailed failure message
+	return fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName)
+}
+
+func (p *Payload) GenerateSubject() string {
+	//generate a detailed failure message
+	message := fmt.Sprintf(
+		`Scrutiny SMART error notification for device: %s
+Failure Type: %s
+Device Name: %s
+Device Serial: %s
+Device Type: %s
+
+Date: %s`, p.DeviceName, p.FailureType, p.DeviceName, p.DeviceSerial, p.DeviceType, p.Date)
+
+	if p.Test {
+		message = "TEST NOTIFICATION:\n" + message
+	}
+
+	return message
 }
 
 type Notify struct {
+	Logger  logrus.FieldLogger
 	Config  config.Interface
 	Payload Payload
 }
@@ -38,6 +69,7 @@ func (n *Notify) Send() error {
 
 	//retrieve list of notification endpoints from config file
 	configUrls := n.Config.GetStringSlice("notify.urls")
+	n.Logger.Debugf("Configured notification services: %v", configUrls)
 
 	//remove http:// https:// and script:// prefixed urls
 	notifyWebhooks := []string{}
@@ -54,6 +86,10 @@ func (n *Notify) Send() error {
 		}
 	}
 
+	n.Logger.Debugf("Configured scripts: %v", notifyScripts)
+	n.Logger.Debugf("Configured webhooks: %v", notifyWebhooks)
+	n.Logger.Debugf("Configured shoutrrr: %v", notifyShoutrrr)
+
 	//run all scripts, webhooks and shoutrr commands in parallel
 	var wg sync.WaitGroup
 
@@ -67,12 +103,14 @@ func (n *Notify) Send() error {
 		wg.Add(1)
 		go n.SendScriptNotification(&wg, notifyScript)
 	}
-	if len(notifyScripts) > 0 {
+	for _, shoutrrrUrl := range notifyShoutrrr {
 		wg.Add(1)
-		go n.SendShoutrrrNotification(&wg, notifyShoutrrr)
+		go n.SendShoutrrrNotification(&wg, shoutrrrUrl)
 	}
 
 	//and wait for completion, error or timeout.
+	n.Logger.Debugf("Main: waiting for notifications to complete.")
+	//wg.Wait()
 	if waitTimeout(&wg, time.Minute) { //wait for 1 minute
 		fmt.Println("Timed out while sending notifications")
 	} else {
@@ -83,16 +121,16 @@ func (n *Notify) Send() error {
 
 func (n *Notify) SendWebhookNotification(wg *sync.WaitGroup, webhookUrl string) {
 	defer wg.Done()
-	log.Infof("Sending Webhook to %s", webhookUrl)
+	n.Logger.Infof("Sending Webhook to %s", webhookUrl)
 	requestBody, err := json.Marshal(n.Payload)
 	if err != nil {
-		log.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err)
+		n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err)
 		return
 	}
 
 	resp, err := http.Post(webhookUrl, "application/json", bytes.NewBuffer(requestBody))
 	if err != nil {
-		log.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err)
+		n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err)
 		return
 	}
 	defer resp.Body.Close()
@@ -104,43 +142,95 @@ func (n *Notify) SendScriptNotification(wg *sync.WaitGroup, scriptUrl string) {
 
 	//check if the script exists.
 	scriptPath := strings.TrimPrefix(scriptUrl, "script://")
-	log.Infof("Executing Script %s", scriptPath)
+	n.Logger.Infof("Executing Script %s", scriptPath)
 
 	if !utils.FileExists(scriptPath) {
-		log.Errorf("Script does not exist: %s", scriptPath)
+		n.Logger.Errorf("Script does not exist: %s", scriptPath)
 		return
 	}
 
 	copyEnv := os.Environ()
-	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MAILER=%s", n.Payload.Mailer))
+	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.GenerateSubject()))
-	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.Subject))
 	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DATE=%s", n.Payload.Date))
 	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_FAILURE_TYPE=%s", n.Payload.FailureType))
-	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE=%s", n.Payload.Device))
+	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_NAME=%s", n.Payload.DeviceName))
 	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_TYPE=%s", n.Payload.DeviceType))
-	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_STRING=%s", n.Payload.DeviceString))
+	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_SERIAL=%s", n.Payload.DeviceSerial))
-	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.Message))
+	copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.GenerateMessage()))
 	err := utils.CmdExec(scriptPath, []string{}, "", copyEnv, "")
 	if err != nil {
-		log.Errorf("An error occurred while executing script %s: %v", scriptPath, err)
+		n.Logger.Errorf("An error occurred while executing script %s: %v", scriptPath, err)
 	}
 	return
 }
 
-func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrls []string) {
+func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrl string) {
-	log.Infof("Sending notifications to %v", shoutrrrUrls)
+
+	fmt.Printf("Sending Notifications to %v", shoutrrrUrl)
+	n.Logger.Infof("Sending notifications to %v", shoutrrrUrl)
 
 	defer wg.Done()
-	sender, err := shoutrrr.CreateSender(shoutrrrUrls...)
+	sender, err := shoutrrr.CreateSender(shoutrrrUrl)
 	if err != nil {
-		log.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrls, err)
+		n.Logger.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrl, err)
 		return
 	}
 
-	errs := sender.Send(n.Payload.Subject, nil) //structs.Map(n.Payload).())
+	//sender.SetLogger(n.Logger.)
-	if len(errs) > 0 {
+	serviceName, params, err := n.GenShoutrrrNotificationParams(shoutrrrUrl)
-		log.Errorf("One or more errors occurred  occurred while sending notifications %v:\n %v", shoutrrrUrls, errs)
+	n.Logger.Debug("notification data for %s: (%s)\n%v", serviceName, shoutrrrUrl, params)
+
+	if err != nil {
+		n.Logger.Errorf("An error occurred  occurred while generating notification payload for %s:\n %v", serviceName, shoutrrrUrl, err)
 	}
+
+	errs := sender.Send(n.Payload.GenerateMessage(), params)
+	if len(errs) > 0 {
+		n.Logger.Errorf("One or more errors occurred  occurred while sending notifications for %s:\n %v", shoutrrrUrl, errs)
+		for _, err := range errs {
+			n.Logger.Error(err)
+		}
+	}
+}
+
+func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *shoutrrrTypes.Params, error) {
+	serviceURL, err := url.Parse(shoutrrrUrl)
+	if err != nil {
+		return "", nil, err
+	}
+
+	serviceName := serviceURL.Scheme
+	params := &shoutrrrTypes.Params{}
+
+	logoUrl := "https://raw.githubusercontent.com/AnalogJ/scrutiny/master/webapp/frontend/src/ms-icon-144x144.png"
+	subject := n.Payload.GenerateSubject()
+	switch serviceName {
+	// no params supported for these services
+	case "discord", "hangouts", "ifttt", "mattermost", "teams":
+		break
+	case "gotify":
+		(*params)["title"] = subject
+	case "join":
+		(*params)["title"] = subject
+		(*params)["icon"] = logoUrl
+	case "pushbullet":
+		(*params)["title"] = subject
+	case "pushover":
+		(*params)["subject"] = subject
+	case "slack":
+		(*params)["title"] = subject
+		(*params)["thumb_url"] = logoUrl
+	case "smtp":
+		(*params)["subject"] = subject
+	case "standard":
+		(*params)["subject"] = subject
+	case "telegram":
+		(*params)["subject"] = subject
+	case "zulip":
+		(*params)["topic"] = subject
+	}
+
+	return serviceName, params, nil
 }
 
 //utility functions

webapp/backend/pkg/web/handler/send_test_notification.go

@@ -1,14 +1,12 @@
 package handler
 
 import (
-	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"net/http"
-	"os"
 )
 
 // Send test notification
@@ -17,15 +15,14 @@ func SendTestNotification(c *gin.Context) {
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 
 	testNotify := notify.Notify{
+		Logger: logger,
 		Config: appConfig,
 		Payload: notify.Payload{
-			Mailer:       os.Args[0],
-			Subject:      fmt.Sprintf("Scrutiny SMART error (EmailTest) detected on disk: XXXXX"),
 			FailureType:  "EmailTest",
-			Device:       "/dev/sda",
+			DeviceSerial: "FAKEWDDJ324KSO",
-			DeviceType:   "ata",
+			DeviceType:   dbModels.DeviceProtocolAta,
-			DeviceString: "/dev/sda",
+			DeviceName:   "/dev/sda",
-			Message:      "TEST EMAIL from smartd for device: /dev/sda",
+			Test:         true,
 		},
 	}
 	err := testNotify.Send()

webapp/backend/pkg/web/handler/upload_device_metrics.go

@@ -1,8 +1,10 @@
 package handler
 
 import (
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
 	"github.com/jinzhu/gorm"
 	"github.com/sirupsen/logrus"
@@ -12,6 +14,7 @@ import (
 func UploadDeviceMetrics(c *gin.Context) {
 	db := c.MustGet("DB").(*gorm.DB)
 	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
+	appConfig := c.MustGet("CONFIG").(config.Interface)
 
 	var collectorSmartData collector.SmartInfo
 	err := c.BindJSON(&collectorSmartData)
@@ -45,5 +48,21 @@ func UploadDeviceMetrics(c *gin.Context) {
 		return
 	}
 
+	//check for error
+	if deviceSmartData.SmartStatus == dbModels.SmartStatusFailed {
+		//send notifications
+		testNotify := notify.Notify{
+			Config: appConfig,
+			Payload: notify.Payload{
+				FailureType:  notify.NotifyFailureTypeSmartFailure,
+				DeviceName:   device.DeviceName,
+				DeviceType:   device.DeviceProtocol,
+				DeviceSerial: device.SerialNumber,
+				Test:         false,
+			},
+		}
+		_ = testNotify.Send() //we ignore error message when sending notifications.
+	}
+
 	c.JSON(http.StatusOK, gin.H{"success": true})
 }