(0.2.4) Automated packaging of release by Packagr

Signed-off-by: Jason Kulatunga <jason@thesparktree.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -20,7 +20,22 @@ If applicable, add screenshots to help explain your problem.
 If related to missing devices or SMART data, please run the `collector` in DEBUG mode, and attach the log file.
 
 ```
-docker exec scrutiny scrutiny-collector-metrics run --debug --log-file /tmp/test.log
-# then use docker cp to copy the log file out of the container. 
-docker cp scrutiny:/tmp/test.log test.log
+docker run -it --rm -p 8080:8080 \
+-v /run/udev:/run/udev:ro \
+--cap-add SYS_RAWIO \
+--device=/dev/sda \
+--device=/dev/sdb \
+-e DEBUG=true \
+-e COLLECTOR_LOG_FILE=/tmp/collector.log \
+-e SCRUTINY_LOG_FILE=/tmp/web.log \
+--name scrutiny \
+analogj/scrutiny
+
+# in another terminal trigger the collector
+docker exec scrutiny scrutiny-collector-metrics run
+
+# then use docker cp to copy the log files out of the container.
+docker cp scrutiny:/tmp/collector.log collector.log
+docker cp scrutiny:/tmp/web.log web.log
+
 ```

.github/workflows/release-frontend.yaml

@@ -0,0 +1,35 @@
+# compiles angular frontend and attaches it to the latest release.
+name: Release Frontend
+
+on:
+  release:
+    # Only use the types keyword to narrow down the activity types that will trigger your workflow.
+    types: [published]
+jobs:
+  release-frontend:
+    name: Release Frontend
+    runs-on: ubuntu-latest
+    container: node:lts-slim
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{github.event.release.tag_name}}
+      - name: Build Frontend
+        run: |
+          cd webapp/frontend
+          npm install -g @angular/cli@9.1.4
+          npm install
+          mkdir -p dist
+          ng build --output-path=dist --deploy-url="/web/" --base-href="/web/" --prod
+          tar -czf scrutiny-web-frontend.tar.gz dist
+      - name: Upload Frontend Asset
+        id: upload-release-asset3
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: './webapp/frontend/scrutiny-web-frontend.tar.gz'
+          asset_name: scrutiny-web-frontend.tar.gz
+          asset_content_type: application/gzip

.gitignore

@@ -62,5 +62,5 @@ vendor
 /scrutiny-web-linux-amd64
 scrutiny-*.db
 scrutiny_test.db
-
+scrutiny.yaml
 coverage.txt

CONTRIBUTING.md

@@ -7,7 +7,12 @@ There are multiple ways to develop on the scrutiny codebase locally. The two mos
 ## Docker Development
 ```
 docker build -f docker/Dockerfile . -t analogj/scrutiny
-docker run -it --rm -p 9090:8080 -v /run:/run  -v /dev/disk:/dev/disk --privileged analogj/scrutiny
+docker run -it --rm -p 8080:8080 \
+-v /run/udev:/run/udev:ro \
+--cap-add SYS_RAWIO \
+--device=/dev/sda \
+--device=/dev/sdb \
+analogj/scrutiny
 /scrutiny/bin/scrutiny-collector-metrics run
 ```
 
@@ -29,9 +34,38 @@ cd webapp/frontend && ng build --watch --output-path=../../dist --deploy-url="/w
 > Note: if you do not add `--prod` flag, app will display mocked data for api calls.
 
 ### Backend
+
+If you're using the `ng build` command above to generate your frontend, you'll need to create a custom config file and
+override the `web.src.frontend.path` value.
+
 ```
-go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./example.scrutiny.yaml
+# config file for local development. store as scrutiny.yaml
+version: 1
+
+web:
+  listen:
+    port: 8080
+    host: 0.0.0.0
+  database:
+    # can also set absolute path here
+    location: ./scrutiny.db
+  src:
+    frontend:
+      path: ./dist
+
+
+log:
+  file: 'web.log' #absolute or relative paths allowed, eg. web.log
+  level: DEBUG
+
 ```
+
+Once you've created a config file, you can pass it to the scrutiny binary during startup.
+
+```
+go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
+```
+
 Now visit http://localhost:8080
 
 
@@ -40,3 +74,24 @@ Now visit http://localhost:8080
 brew install smartmontools
 go run collector/cmd/collector-metrics/collector-metrics.go run --debug
 ```
+
+
+## Debugging
+
+If you need more verbose logs for debugging, you can use the following environmental variables:
+
+- `DEBUG=true` - enables debug level logging on both the `collector` and `webapp`
+- `COLLECTOR_DEBUG=true` - enables debug level logging on the `collector`
+- `SCRUTINY_DEBUG=true` - enables debug level logging on the `webapp`
+
+In addition, you can instruct scrutiny to write its logs to a file using the following environmental variables:
+
+- `COLLECTOR_LOG_FILE=/tmp/collector.log` - write the `collector` logs to a file
+- `SCRUTINY_LOG_FILE=/tmp/web.log` - write the `webapp` logs to a file
+
+Finally, you can copy the files from the scrutiny container to your host using the following command(s)
+
+```
+docker cp scrutiny:/tmp/collector.log collector.log
+docker cp scrutiny:/tmp/web.log web.log
+```

README.md

@@ -59,13 +59,17 @@ If you're using Docker, getting started is as simple as running the following co
 ```bash
 docker run -it --rm -p 8080:8080 \
 -v /run/udev:/run/udev:ro \
--v /dev/disk:/dev/disk \
+--cap-add SYS_RAWIO \
+--device=/dev/sda \
+--device=/dev/sdb \
 --name scrutiny \
---privileged analogj/scrutiny
+analogj/scrutiny
 ```
 
-- `/run/udev` and `/dev/disk` are necessary to provide the Scrutiny collector with access to your drive metadata.
-- `--privileged` is required to ensure that your hard disk devices are accessible within the container (this will be changed in a future release)
+- `/run/udev` is necessary to provide the Scrutiny collector with access to your device metadata
+- `--cap-add SYS_RAWIO` is necessary to allow `smartctl` permission to query your device SMART data
+    - NOTE: If you have NVMe drives, you must use `--cap-add SYS_ADMIN` instead. See issue [#26](https://github.com/AnalogJ/scrutiny/issues/26#issuecomment-696817130)
+- `--device` entries are required to ensure that your hard disk devices are accessible within the container
 - `analogj/scrutiny` is a omnibus image, containing both the webapp server (frontend & api) as well as the S.M.A.R.T metric collector. (see below)
 
 ### Hub/Spoke Deployment
@@ -82,10 +86,12 @@ analogj/scrutiny:web
 
 docker run -it --rm \
 -v /run/udev:/run/udev:ro \
--v /dev/disk:/dev/disk \
+--cap-add SYS_RAWIO \
+--device=/dev/sda \
+--device=/dev/sdb \
 -e SCRUTINY_API_ENDPOINT=http://SCRUTINY_WEB_IPADDRESS:8080 \
 --name scrutiny-collector \
---privileged analogj/scrutiny:collector
+analogj/scrutiny:collector
 ```
 
 

collector/cmd/collector-metrics/collector-metrics.go

@@ -117,15 +117,16 @@ OPTIONS:
 					},
 
 					&cli.StringFlag{
-						Name:  "log-file",
-						Usage: "Path to file for logging. Leave empty to use STDOUT",
-						Value: "",
+						Name:    "log-file",
+						Usage:   "Path to file for logging. Leave empty to use STDOUT",
+						Value:   "",
+						EnvVars: []string{"COLLECTOR_LOG_FILE"},
 					},
 
 					&cli.BoolFlag{
 						Name:    "debug",
 						Usage:   "Enable debug logging",
-						EnvVars: []string{"DEBUG"},
+						EnvVars: []string{"COLLECTOR_DEBUG", "DEBUG"},
 					},
 				},
 			},

collector/cmd/collector-selftest/collector-selftest.go

@@ -117,15 +117,16 @@ OPTIONS:
 					},
 
 					&cli.StringFlag{
-						Name:  "log-file",
-						Usage: "Path to file for logging. Leave empty to use STDOUT",
-						Value: "",
+						Name:    "log-file",
+						Usage:   "Path to file for logging. Leave empty to use STDOUT",
+						Value:   "",
+						EnvVars: []string{"COLLECTOR_LOG_FILE"},
 					},
 
 					&cli.BoolFlag{
 						Name:    "debug",
 						Usage:   "Enable debug logging",
-						EnvVars: []string{"DEBUG"},
+						EnvVars: []string{"COLLECTOR_DEBUG", "DEBUG"},
 					},
 				},
 			},

collector/pkg/collector/metrics.go

@@ -13,7 +13,6 @@ import (
 	"os"
 	"os/exec"
 	"strings"
-	"sync"
 )
 
 type MetricsCollector struct {
@@ -71,16 +70,19 @@ func (mc *MetricsCollector) Run() error {
 		return errors.ApiServerCommunicationError("An error occurred while retrieving filtered devices")
 	} else {
 		mc.logger.Debugln(deviceRespWrapper)
-		var wg sync.WaitGroup
-
+		//var wg sync.WaitGroup
 		for _, device := range deviceRespWrapper.Data {
 			// execute collection in parallel go-routines
-			wg.Add(1)
-			go mc.Collect(&wg, device.WWN, device.DeviceName, device.DeviceType)
+			//wg.Add(1)
+			//go mc.Collect(&wg, device.WWN, device.DeviceName, device.DeviceType)
+			mc.Collect(device.WWN, device.DeviceName, device.DeviceType)
+
+			// TODO: we may need to sleep for between each call to smartctl -a
+			//time.Sleep(30 * time.Millisecond)
 		}
 
-		mc.logger.Infoln("Main: Waiting for workers to finish")
-		wg.Wait()
+		//mc.logger.Infoln("Main: Waiting for workers to finish")
+		//wg.Wait()
 		mc.logger.Infoln("Main: Completed")
 	}
 
@@ -98,11 +100,12 @@ func (mc *MetricsCollector) Validate() error {
 	return nil
 }
 
-func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
-	defer wg.Done()
+//func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
+func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceType string) {
+	//defer wg.Done()
 	mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
 
-	args := []string{"-a", "-j"}
+	args := []string{"-x", "-j"}
 	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
 	if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" {
 		args = append(args, "-d", deviceType)

collector/pkg/detect/detect.go

@@ -100,7 +100,7 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
 			Oui: availableDeviceInfo.Wwn.Oui,
 			Id:  availableDeviceInfo.Wwn.ID,
 		}
-		device.WWN = wwn.ToString()
+		device.WWN = strings.ToLower(wwn.ToString())
 		d.Logger.Debugf("NAA: %d OUI: %d Id: %d => WWN: %s", wwn.Naa, wwn.Oui, wwn.Id, device.WWN)
 	} else {
 		d.Logger.Info("Using WWN Fallback")

collector/pkg/detect/devices_darwin.go

@@ -105,4 +105,7 @@ func (d *Detect) wwnFallback(detectedDevice *models.Device) {
 		d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber)
 		detectedDevice.WWN = detectedDevice.SerialNumber
 	}
+
+	//wwn must always be lowercase.
+	detectedDevice.WWN = strings.ToLower(detectedDevice.WWN)
 }

collector/pkg/detect/devices_linux.go

@@ -3,6 +3,7 @@ package detect
 import (
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/jaypipes/ghw"
+	"strings"
 )
 
 func DevicePrefix() string {
@@ -42,4 +43,7 @@ func (d *Detect) wwnFallback(detectedDevice *models.Device) {
 		d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber)
 		detectedDevice.WWN = detectedDevice.SerialNumber
 	}
+
+	//wwn must always be lowercase.
+	detectedDevice.WWN = strings.ToLower(detectedDevice.WWN)
 }

collector/pkg/detect/devices_linux_test.go

@@ -0,0 +1,16 @@
+package detect_test
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/detect"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestDevicePrefix(t *testing.T) {
+	//setup
+
+	//test
+
+	//assert
+	require.Equal(t, "/dev/", detect.DevicePrefix())
+}

collector/pkg/detect/devices_windows.go

@@ -1,5 +1,11 @@
 package detect
 
+import (
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/jaypipes/ghw"
+	"strings"
+)
+
 func DevicePrefix() string {
 	return ""
 }
@@ -26,4 +32,7 @@ func (d *Detect) wwnFallback(detectedDevice *models.Device) {
 	if len(detectedDevice.WWN) == 0 {
 		detectedDevice.WWN = detectedDevice.SerialNumber
 	}
+
+	//wwn must always be lowercase.
+	detectedDevice.WWN = strings.ToLower(detectedDevice.WWN)
 }

collector/pkg/detect/wwn.go

@@ -2,6 +2,7 @@ package detect
 
 import (
 	"fmt"
+	"strings"
 )
 
 type Wwn struct {
@@ -54,5 +55,5 @@ func (wwn *Wwn) ToString() string {
 
 	//TODO: may need to support additional versions in the future.
 
-	return fmt.Sprintf("%#x", wwnBuffer)
+	return strings.ToLower(fmt.Sprintf("%#x", wwnBuffer))
 }

example.scrutiny.yaml

@@ -22,57 +22,64 @@ web:
     host: 0.0.0.0
   database:
     # can also set absolute path here
-    location: ./scrutiny.db
+    location: /scrutiny/config/scrutiny.db
   src:
     frontend:
-      path: ./dist
-
-disks:
-  include:
-    # - /dev/sda
-  exclude:
-    # - /dev/sdb
-
-notify:
-  urls:
-    - "discord://token@channel"
-    - "telegram://token@telegram?channels=channel-1[,channel-2,...]"
-    - "pushover://shoutrrr:apiToken@userKey/?devices=device1[,device2, ...]"
-    - "slack://[botname@]token-a/token-b/token-c"
-    - "smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]"
-    - "teams://token-a/token-b/token-c"
-    - "gotify://gotify-host/token"
-    - "pushbullet://api-token[/device/#channel/email]"
-    - "ifttt://key/?events=event1[,event2,...]&value1=value1&value2=value2&value3=value3"
-    - "mattermost://[username@]mattermost-host/token[/channel]"
-    - "hangouts://chat.googleapis.com/v1/spaces/FOO/messages?key=bar&token=baz"
-    - "zulip://bot-mail:bot-key@zulip-domain/?stream=name-or-id&topic=name"
-    - "join://shoutrrr:api-key@join/?devices=device1[,device2, ...][&icon=icon][&title=title]"
-    - "script:///file/path/on/disk"
-    - "https://www.example.com/path"
-
-limits:
-  ata:
-    critical:
-      error: 10
-    standard:
-      error: 20
-      warn: 10
-  scsi:
-    critical: true
-    standard: true
-  nvme:
-    critical: true
-    standard: true
+      path: /scrutiny/web
 
 
-collect:
-  metric:
-    enable: true
-    command: '-a -o on -S on'
-  long:
-    enable: false
-    command: ''
-  short:
-    enable: false
-    command: ''
+log:
+  file: '' #absolute or relative paths allowed, eg. web.log
+  level: INFO
+
+# The following commented out sections are a preview of additional configuration options that will be available soon.
+
+#disks:
+#  include:
+#    # - /dev/sda
+#  exclude:
+#    # - /dev/sdb
+
+#notify:
+#  urls:
+#    - "discord://token@channel"
+#    - "telegram://token@telegram?channels=channel-1[,channel-2,...]"
+#    - "pushover://shoutrrr:apiToken@userKey/?devices=device1[,device2, ...]"
+#    - "slack://[botname@]token-a/token-b/token-c"
+#    - "smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]"
+#    - "teams://token-a/token-b/token-c"
+#    - "gotify://gotify-host/token"
+#    - "pushbullet://api-token[/device/#channel/email]"
+#    - "ifttt://key/?events=event1[,event2,...]&value1=value1&value2=value2&value3=value3"
+#    - "mattermost://[username@]mattermost-host/token[/channel]"
+#    - "hangouts://chat.googleapis.com/v1/spaces/FOO/messages?key=bar&token=baz"
+#    - "zulip://bot-mail:bot-key@zulip-domain/?stream=name-or-id&topic=name"
+#    - "join://shoutrrr:api-key@join/?devices=device1[,device2, ...][&icon=icon][&title=title]"
+#    - "script:///file/path/on/disk"
+#    - "https://www.example.com/path"
+
+#limits:
+#  ata:
+#    critical:
+#      error: 10
+#    standard:
+#      error: 20
+#      warn: 10
+#  scsi:
+#    critical: true
+#    standard: true
+#  nvme:
+#    critical: true
+#    standard: true
+
+
+#collect:
+#  metric:
+#    enable: true
+#    command: '-a -o on -S on'
+#  long:
+#    enable: false
+#    command: ''
+#  short:
+#    enable: false
+#    command: ''

webapp/backend/cmd/scrutiny/scrutiny.go

@@ -95,10 +95,18 @@ OPTIONS:
 						if err != nil {                             // Handle errors reading the config file
 							//ignore "could not find config file"
 							fmt.Printf("Could not find config file at specified path: %s", c.String("config"))
-							os.Exit(1)
+							return err
 						}
 					}
 
+					if c.Bool("debug") {
+						config.Set("log.level", "DEBUG")
+					}
+
+					if c.IsSet("log-file") {
+						config.Set("log.file", c.String("log-file"))
+					}
+
 					webServer := web.AppEngine{Config: config}
 
 					return webServer.Start()
@@ -109,6 +117,18 @@ OPTIONS:
 						Name:  "config",
 						Usage: "Specify the path to the config file",
 					},
+					&cli.StringFlag{
+						Name:    "log-file",
+						Usage:   "Path to file for logging. Leave empty to use STDOUT",
+						Value:   "",
+						EnvVars: []string{"SCRUTINY_LOG_FILE"},
+					},
+
+					&cli.BoolFlag{
+						Name:    "debug",
+						Usage:   "Enable debug logging",
+						EnvVars: []string{"SCRUTINY_DEBUG", "DEBUG"},
+					},
 				},
 			},
 		},

webapp/backend/pkg/config/config.go

@@ -30,22 +30,24 @@ func (c *configuration) Init() error {
 	c.SetDefault("web.listen.port", "8080")
 	c.SetDefault("web.listen.host", "0.0.0.0")
 	c.SetDefault("web.src.frontend.path", "/scrutiny/web")
-
 	c.SetDefault("web.database.location", "/scrutiny/config/scrutiny.db")
 
-	c.SetDefault("disks.include", []string{})
-	c.SetDefault("disks.exclude", []string{})
+	c.SetDefault("log.level", "INFO")
+	c.SetDefault("log.file", "")
 
-	c.SetDefault("notify.metric.script", "/scrutiny/config/notify-metrics.sh")
-	c.SetDefault("notify.long.script", "/scrutiny/config/notify-long-test.sh")
-	c.SetDefault("notify.short.script", "/scrutiny/config/notify-short-test.sh")
+	//c.SetDefault("disks.include", []string{})
+	//c.SetDefault("disks.exclude", []string{})
 
-	c.SetDefault("collect.metric.enable", true)
-	c.SetDefault("collect.metric.command", "-a -o on -S on")
-	c.SetDefault("collect.long.enable", true)
-	c.SetDefault("collect.long.command", "-a -o on -S on")
-	c.SetDefault("collect.short.enable", true)
-	c.SetDefault("collect.short.command", "-a -o on -S on")
+	//c.SetDefault("notify.metric.script", "/scrutiny/config/notify-metrics.sh")
+	//c.SetDefault("notify.long.script", "/scrutiny/config/notify-long-test.sh")
+	//c.SetDefault("notify.short.script", "/scrutiny/config/notify-short-test.sh")
+
+	//c.SetDefault("collect.metric.enable", true)
+	//c.SetDefault("collect.metric.command", "-a -o on -S on")
+	//c.SetDefault("collect.long.enable", true)
+	//c.SetDefault("collect.long.command", "-a -o on -S on")
+	//c.SetDefault("collect.short.enable", true)
+	//c.SetDefault("collect.short.command", "-a -o on -S on")
 
 	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
 	c.SetConfigType("yaml")

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.2.0"
+const VERSION = "0.2.4"

webapp/backend/pkg/web/handler/get_device_details.go

@@ -5,25 +5,40 @@ import (
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
 	"github.com/jinzhu/gorm"
+	"github.com/sirupsen/logrus"
 	"net/http"
 )
 
 func GetDeviceDetails(c *gin.Context) {
 	db := c.MustGet("DB").(*gorm.DB)
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 	device := dbModels.Device{}
 
-	db.Debug().
-		Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
-			return db.Order("smarts.created_at DESC").Limit(40)
-		}).
+	if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
+		return db.Order("smarts.created_at DESC").Limit(40)
+	}).
 		Preload("SmartResults.AtaAttributes").
 		Preload("SmartResults.NvmeAttributes").
 		Preload("SmartResults.ScsiAttributes").
 		Where("wwn = ?", c.Param("wwn")).
-		First(&device)
+		First(&device).Error; err != nil {
 
-	device.SquashHistory()
-	device.ApplyMetadataRules()
+		logger.Errorln("An error occurred while retrieving device details", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	if err := device.SquashHistory(); err != nil {
+		logger.Errorln("An error occurred while squashing device history", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+
+	if err := device.ApplyMetadataRules(); err != nil {
+		logger.Errorln("An error occurred while applying scrutiny thresholds & rules", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
 
 	var deviceMetadata interface{}
 	if device.IsAta() {

webapp/backend/pkg/web/handler/get_devices_summary.go

@@ -4,20 +4,25 @@ import (
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
 	"github.com/jinzhu/gorm"
+	"github.com/sirupsen/logrus"
 	"net/http"
 )
 
 func GetDevicesSummary(c *gin.Context) {
 	db := c.MustGet("DB").(*gorm.DB)
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 	devices := []dbModels.Device{}
 
 	//We need the last x (for now all) Smart objects for each Device, so that we can graph Temperature
 	//We also need the last
-	db.Debug().
-		Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
-			return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
-		}).
-		Find(&devices)
+	if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
+		return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
+	}).
+		Find(&devices).Error; err != nil {
+		logger.Errorln("Could not get device summary from DB", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
 
 	c.JSON(http.StatusOK, gin.H{
 		"success": true,

webapp/backend/pkg/web/handler/register_devices.go

@@ -4,36 +4,43 @@ import (
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
 	"github.com/jinzhu/gorm"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 	"net/http"
 )
 
 // filter devices that are detected by various collectors.
 func RegisterDevices(c *gin.Context) {
 	db := c.MustGet("DB").(*gorm.DB)
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 
 	var collectorDeviceWrapper dbModels.DeviceWrapper
 	err := c.BindJSON(&collectorDeviceWrapper)
 	if err != nil {
-		log.Error("Cannot parse detected devices")
-		c.JSON(http.StatusOK, gin.H{"success": false})
+		logger.Errorln("Cannot parse detected devices", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
 	}
 
 	//TODO: filter devices here (remove excludes, force includes)
-
+	errs := []error{}
 	for _, dev := range collectorDeviceWrapper.Data {
 		//insert devices into DB if not already there.
-		db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev)
+		if err := db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev).Error; err != nil {
+			errs = append(errs, err)
+		}
 	}
 
-	if err != nil {
-		c.JSON(http.StatusOK, gin.H{
+	if len(errs) > 0 {
+		logger.Errorln("An error occurred while registering devices", errs)
+		c.JSON(http.StatusInternalServerError, gin.H{
 			"success": false,
 		})
+		return
 	} else {
 		c.JSON(http.StatusOK, dbModels.DeviceWrapper{
 			Success: true,
 			Data:    collectorDeviceWrapper.Data,
 		})
+		return
 	}
 }

webapp/backend/pkg/web/handler/send_test_notification.go

@@ -6,6 +6,7 @@ import (
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
 	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
 	"net/http"
 	"os"
 )
@@ -13,6 +14,7 @@ import (
 // Send test notification
 func SendTestNotification(c *gin.Context) {
 	appConfig := c.MustGet("CONFIG").(config.Interface)
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 
 	testNotify := notify.Notify{
 		Config: appConfig,
@@ -28,7 +30,8 @@ func SendTestNotification(c *gin.Context) {
 	}
 	err := testNotify.Send()
 	if err != nil {
-		c.JSON(http.StatusOK, gin.H{
+		logger.Errorln("An error occurred while sending test notification", err)
+		c.JSON(http.StatusInternalServerError, gin.H{
 			"success": false,
 		})
 	} else {

webapp/backend/pkg/web/handler/upload_device_metrics.go

@@ -5,36 +5,45 @@ import (
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
 	"github.com/jinzhu/gorm"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 	"net/http"
 )
 
 func UploadDeviceMetrics(c *gin.Context) {
 	db := c.MustGet("DB").(*gorm.DB)
+	logger := c.MustGet("LOGGER").(logrus.FieldLogger)
 
 	var collectorSmartData collector.SmartInfo
 	err := c.BindJSON(&collectorSmartData)
 	if err != nil {
-		//TODO: cannot parse smart data
-		log.Error("Cannot parse SMART data")
-		c.JSON(http.StatusOK, gin.H{"success": false})
-
+		logger.Errorln("Cannot parse SMART data", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
 	}
 
 	//update the device information if necessary
 	var device dbModels.Device
 	db.Where("wwn = ?", c.Param("wwn")).First(&device)
 	device.UpdateFromCollectorSmartInfo(collectorSmartData)
-	db.Model(&device).Updates(device)
+	if err := db.Model(&device).Updates(device).Error; err != nil {
+		logger.Errorln("An error occurred while updating device data from smartctl metrics", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
 
 	// insert smart info
 	deviceSmartData := dbModels.Smart{}
 	err = deviceSmartData.FromCollectorSmartInfo(c.Param("wwn"), collectorSmartData)
 	if err != nil {
-		c.JSON(http.StatusOK, gin.H{"success": false})
+		logger.Errorln("Could not process SMART metrics", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
+		return
+	}
+	if err := db.Create(&deviceSmartData).Error; err != nil {
+		logger.Errorln("An error occurred while saving smartctl metrics", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"success": false})
 		return
 	}
-	db.Create(&deviceSmartData)
 
 	c.JSON(http.StatusOK, gin.H{"success": true})
 }

webapp/backend/pkg/web/middleware/logger.go

@@ -0,0 +1,122 @@
+package middleware
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"io"
+	"io/ioutil"
+	"math"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+// Middleware based on https://github.com/toorop/gin-logrus/blob/master/logger.go
+// Body recording based on
+// - https://github.com/gin-gonic/gin/issues/1363
+// - https://stackoverflow.com/questions/38501325/how-to-log-response-body-in-gin
+
+// 2016-09-27 09:38:21.541541811 +0200 CEST
+// 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700]
+// "GET /apache_pb.gif HTTP/1.0" 200 2326
+// "http://www.example.com/start.html"
+// "Mozilla/4.08 [en] (Win98; I ;Nav)"
+
+var timeFormat = "02/Jan/2006:15:04:05 -0700"
+
+// Logger is the logrus logger handler
+func LoggerMiddleware(logger logrus.FieldLogger) gin.HandlerFunc {
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		hostname = "unknow"
+	}
+
+	return func(c *gin.Context) {
+
+		//clone the request body reader.
+		var reqBody string
+		if c.Request.Body != nil {
+			buf, _ := ioutil.ReadAll(c.Request.Body)
+			reqBodyReader1 := ioutil.NopCloser(bytes.NewBuffer(buf))
+			reqBodyReader2 := ioutil.NopCloser(bytes.NewBuffer(buf)) //We have to create a new Buffer, because reqBodyReader1 will be read.
+			c.Request.Body = reqBodyReader2
+			reqBody = readBody(reqBodyReader1)
+		}
+
+		// other handler can change c.Path so:
+		path := c.Request.URL.Path
+		blw := &responseBodyLogWriter{body: &bytes.Buffer{}, ResponseWriter: c.Writer}
+		c.Writer = blw
+		c.Set("LOGGER", logger)
+		start := time.Now()
+		c.Next()
+		stop := time.Since(start)
+		latency := int(math.Ceil(float64(stop.Nanoseconds()) / 1000000.0))
+		statusCode := c.Writer.Status()
+		clientIP := c.ClientIP()
+		clientUserAgent := c.Request.UserAgent()
+		referer := c.Request.Referer()
+		respLength := c.Writer.Size()
+		if respLength < 0 {
+			respLength = 0
+		}
+
+		entry := logger.WithFields(logrus.Fields{
+			"hostname":   hostname,
+			"statusCode": statusCode,
+			"latency":    latency, // time to process
+			"clientIP":   clientIP,
+			"method":     c.Request.Method,
+			"path":       path,
+			"referer":    referer,
+			"respLength": respLength,
+			"userAgent":  clientUserAgent,
+		})
+
+		if len(c.Errors) > 0 {
+			entry.Error(c.Errors.ByType(gin.ErrorTypePrivate).String())
+		} else {
+			msg := fmt.Sprintf("%s - %s [%s] \"%s %s\" %d %d \"%s\" \"%s\" (%dms)", clientIP, hostname, time.Now().Format(timeFormat), c.Request.Method, path, statusCode, respLength, referer, clientUserAgent, latency)
+			if statusCode >= http.StatusInternalServerError {
+				entry.Error(msg)
+			} else if statusCode >= http.StatusBadRequest {
+				entry.Warn(msg)
+			} else {
+				entry.Info(msg)
+			}
+		}
+		if strings.HasPrefix(path, "/api/") {
+			//only debug log request/response from api endpoint.
+			if len(reqBody) > 0 {
+				entry.WithField("bodyType", "request").Debugln(reqBody) // Print request body
+			}
+			entry.WithField("bodyType", "response").Debugln(blw.body.String())
+		}
+	}
+}
+
+// Response Logging
+
+type responseBodyLogWriter struct {
+	gin.ResponseWriter
+	body *bytes.Buffer
+}
+
+func (w responseBodyLogWriter) Write(b []byte) (int, error) {
+	w.body.Write(b)
+	return w.ResponseWriter.Write(b)
+}
+
+// Request Logging
+
+func readBody(reader io.Reader) string {
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(reader)
+
+	s := buf.String()
+	return s
+}

webapp/backend/pkg/web/middleware/sqlite3.go

@@ -2,21 +2,23 @@ package middleware
 
 import (
 	"fmt"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/gin-gonic/gin"
 	"github.com/jinzhu/gorm"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"github.com/sirupsen/logrus"
 )
 
-func DatabaseMiddleware(dbPath string) gin.HandlerFunc {
+func DatabaseMiddleware(appConfig config.Interface, logger logrus.FieldLogger) gin.HandlerFunc {
 	//var database *gorm.DB
-	fmt.Printf("Trying to connect to database stored: %s", dbPath)
-	database, err := gorm.Open("sqlite3", dbPath)
-
+	fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
+	database, err := gorm.Open("sqlite3", appConfig.GetString("web.database.location"))
 	if err != nil {
 		panic("Failed to connect to database!")
 	}
 
+	database.SetLogger(&GormLogger{Logger: logger})
 	database.AutoMigrate(&db.Device{})
 	database.AutoMigrate(&db.SelfTest{})
 	database.AutoMigrate(&db.Smart{})
@@ -30,3 +32,24 @@ func DatabaseMiddleware(dbPath string) gin.HandlerFunc {
 		c.Next()
 	}
 }
+
+// GormLogger is a custom logger for Gorm, making it use logrus.
+type GormLogger struct{ Logger logrus.FieldLogger }
+
+// Print handles log events from Gorm for the custom logger.
+func (gl *GormLogger) Print(v ...interface{}) {
+	switch v[0] {
+	case "sql":
+		gl.Logger.WithFields(
+			logrus.Fields{
+				"module":  "gorm",
+				"type":    "sql",
+				"rows":    v[5],
+				"src_ref": v[1],
+				"values":  v[4],
+			},
+		).Debug(v[3])
+	case "log":
+		gl.Logger.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2])
+	}
+}

webapp/backend/pkg/web/server.go

@@ -6,18 +6,23 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web/handler"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web/middleware"
 	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"io"
 	"net/http"
+	"os"
 )
 
 type AppEngine struct {
 	Config config.Interface
 }
 
-func (ae *AppEngine) Setup() *gin.Engine {
-	r := gin.Default()
+func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
+	r := gin.New()
 
-	r.Use(middleware.DatabaseMiddleware(ae.Config.GetString("web.database.location")))
+	r.Use(middleware.LoggerMiddleware(logger))
+	r.Use(middleware.DatabaseMiddleware(ae.Config, logger))
 	r.Use(middleware.ConfigMiddleware(ae.Config))
+	r.Use(gin.Recovery())
 
 	api := r.Group("/api")
 	{
@@ -51,7 +56,28 @@ func (ae *AppEngine) Setup() *gin.Engine {
 }
 
 func (ae *AppEngine) Start() error {
-	r := ae.Setup()
+
+	logger := logrus.New()
+	//set default log level
+	logLevel, err := logrus.ParseLevel(ae.Config.GetString("log.level"))
+	if err != nil {
+		return err
+	}
+	logger.SetLevel(logLevel)
+	//set the log file if present
+	if len(ae.Config.GetString("log.file")) != 0 {
+		logFile, err := os.OpenFile(ae.Config.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
+		defer logFile.Close()
+		if err != nil {
+			logrus.Errorf("Failed to open log file %s for output: %s", ae.Config.GetString("log.file"), err)
+			return err
+		}
+
+		//configure the logrus default
+		logger.SetOutput(io.MultiWriter(os.Stderr, logFile))
+	}
+
+	r := ae.Setup(logger)
 
 	return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port")))
 }

webapp/backend/pkg/web/server_test.go

@@ -6,6 +6,7 @@ import (
 	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web"
 	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"io/ioutil"
 	"net/http"
@@ -23,14 +24,14 @@ func TestHealthRoute(t *testing.T) {
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
-	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db"))
-	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
 
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
 
-	router := ae.Setup()
+	router := ae.Setup(logrus.New())
 
 	//test
 	w := httptest.NewRecorder()
@@ -49,12 +50,12 @@ func TestRegisterDevicesRoute(t *testing.T) {
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
-	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db"))
-	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath)
+	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")).AnyTimes()
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath).AnyTimes()
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup()
+	router := ae.Setup(logrus.New())
 	file, err := os.Open("testdata/register-devices-req.json")
 	require.NoError(t, err)
 
@@ -79,7 +80,7 @@ func TestUploadDeviceMetricsRoute(t *testing.T) {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup()
+	router := ae.Setup(logrus.New())
 	devicesfile, err := os.Open("testdata/register-devices-single-req.json")
 	require.NoError(t, err)
 
@@ -113,7 +114,7 @@ func TestPopulateMultiple(t *testing.T) {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup()
+	router := ae.Setup(logrus.New())
 	devicesfile, err := os.Open("testdata/register-devices-req.json")
 	require.NoError(t, err)
 
@@ -175,7 +176,7 @@ func TestSendTestNotificationRoute(t *testing.T) {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup()
+	router := ae.Setup(logrus.New())
 
 	//test
 	wr := httptest.NewRecorder()
@@ -198,7 +199,7 @@ func TestGetDevicesSummaryRoute_Nvme(t *testing.T) {
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
-	router := ae.Setup()
+	router := ae.Setup(logrus.New())
 	devicesfile, err := os.Open("testdata/register-devices-req-2.json")
 	require.NoError(t, err)