Merge pull request #38 from AnalogJ/smartctl_scan

Smartctl scan

collector/cmd/collector-metrics/collector-metrics.go

@@ -5,6 +5,7 @@ import (
 	"github.com/analogj/scrutiny/collector/pkg/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/version"
 	"github.com/sirupsen/logrus"
+	"io"
 	"log"
 	"os"
 	"time"
@@ -85,6 +86,16 @@ OPTIONS:
 						logrus.SetLevel(logrus.InfoLevel)
 					}
 
+					if c.IsSet("log-file") {
+						logFile, err := os.OpenFile(c.String("log-file"), os.O_CREATE|os.O_WRONLY, 0644)
+						if err != nil {
+							logrus.Errorf("Failed to open log file %s for output: %s", c.String("log-file"), err)
+							return err
+						}
+						defer logFile.Close()
+						logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
+					}
+
 					metricCollector, err := collector.CreateMetricsCollector(
 						collectorLogger,
 						c.String("api-endpoint"),
@@ -105,6 +116,12 @@ OPTIONS:
 						EnvVars: []string{"SCRUTINY_API_ENDPOINT"},
 					},
 
+					&cli.StringFlag{
+						Name:  "log-file",
+						Usage: "Path to file for logging. Leave empty to use STDOUT",
+						Value: "",
+					},
+
 					&cli.BoolFlag{
 						Name:    "debug",
 						Usage:   "Enable debug logging",

collector/cmd/collector-selftest/collector-selftest.go

@@ -5,6 +5,7 @@ import (
 	"github.com/analogj/scrutiny/collector/pkg/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/version"
 	"github.com/sirupsen/logrus"
+	"io"
 	"log"
 	"os"
 	"time"
@@ -85,6 +86,16 @@ OPTIONS:
 						logrus.SetLevel(logrus.InfoLevel)
 					}
 
+					if c.IsSet("log-file") {
+						logFile, err := os.OpenFile(c.String("log-file"), os.O_CREATE|os.O_WRONLY, 0644)
+						if err != nil {
+							logrus.Errorf("Failed to open log file %s for output: %s", c.String("log-file"), err)
+							return err
+						}
+						defer logFile.Close()
+						logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
+					}
+
 					stCollector, err := collector.CreateSelfTestCollector(
 						collectorLogger,
 						c.String("api-endpoint"),
@@ -105,6 +116,12 @@ OPTIONS:
 						EnvVars: []string{"SCRUTINY_API_ENDPOINT"},
 					},
 
+					&cli.StringFlag{
+						Name:  "log-file",
+						Usage: "Path to file for logging. Leave empty to use STDOUT",
+						Value: "",
+					},
+
 					&cli.BoolFlag{
 						Name:    "debug",
 						Usage:   "Enable debug logging",

collector/pkg/collector/base.go

@@ -3,15 +3,8 @@ package collector
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
-	"github.com/analogj/scrutiny/collector/pkg/models"
-	"github.com/jaypipes/ghw"
 	"github.com/sirupsen/logrus"
-	"io"
 	"net/http"
-	"os"
-	"os/exec"
-	"path"
 	"time"
 )
 
@@ -21,77 +14,6 @@ type BaseCollector struct {
 	logger *logrus.Entry
 }
 
-func (c *BaseCollector) DetectStorageDevices() ([]models.Device, error) {
-
-	//availableDisksJson, err := c.ExecCmd("smartctl", []string{"-j", "--scan"}, "", os.Environ())
-	//if err != nil {
-	//	c.logger.Errorf("Error getting block storage info: %v", err)
-	//	return nil, err
-	//}
-	//
-	//var smartctlScan models.Scan
-	//err = json.Unmarshal([]byte(availableDisksJson), &smartctlScan)
-	//if err != nil {
-	//	return nil, err
-	//}
-
-	block, err := ghw.Block()
-	if err != nil {
-		c.logger.Errorf("Error getting block storage info: %v", err)
-		return nil, err
-	}
-
-	approvedDisks := []models.Device{}
-	for _, disk := range block.Disks {
-
-		// ignore optical drives and floppy disks
-		if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
-			c.logger.Debugf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
-			continue
-		}
-
-		// ignore removable disks
-		if disk.IsRemovable {
-			c.logger.Debugf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
-			continue
-		}
-
-		// ignore virtual disks & mobile phone storage devices
-		if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
-			c.logger.Debugf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
-			continue
-		}
-
-		// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
-		if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
-			c.logger.Debugf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
-			continue
-		}
-
-		diskModel := models.Device{
-			WWN:           disk.WWN,
-			Manufacturer:  disk.Vendor,
-			ModelName:     disk.Model,
-			InterfaceType: disk.StorageController.String(),
-			//InterfaceSpeed: string
-			SerialNumber: disk.SerialNumber,
-			Capacity:     int64(disk.SizeBytes),
-			//Firmware       string
-			//RotationSpeed  int
-
-			DeviceName: disk.Name,
-		}
-		if len(diskModel.WWN) == 0 {
-			//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
-			//diskname as identifier if WWN is not present
-			diskModel.WWN = disk.Name
-		}
-
-		approvedDisks = append(approvedDisks, diskModel)
-	}
-	return approvedDisks, nil
-}
-
 func (c *BaseCollector) getJson(url string, target interface{}) error {
 
 	r, err := httpClient.Get(url)
@@ -118,29 +40,6 @@ func (c *BaseCollector) postJson(url string, body interface{}, target interface{
 	return json.NewDecoder(r.Body).Decode(target)
 }
 
-func (c *BaseCollector) ExecCmd(cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
-
-	cmd := exec.Command(cmdName, cmdArgs...)
-	var stdBuffer bytes.Buffer
-	mw := io.MultiWriter(os.Stdout, &stdBuffer)
-
-	cmd.Stdout = mw
-	cmd.Stderr = mw
-
-	if environ != nil {
-		cmd.Env = environ
-	}
-	if workingDir != "" && path.IsAbs(workingDir) {
-		cmd.Dir = workingDir
-	} else if workingDir != "" {
-		return "", errors.New("Working Directory must be an absolute path")
-	}
-
-	err := cmd.Run()
-	return stdBuffer.String(), err
-
-}
-
 func (c *BaseCollector) LogSmartctlExitCode(exitCode int) {
 	if exitCode&0x01 != 0 {
 		c.logger.Errorln("smartctl could not parse commandline")

collector/pkg/collector/metrics.go

@@ -2,11 +2,15 @@ package collector
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/sirupsen/logrus"
 	"net/url"
+	"os"
 	"os/exec"
 	"strings"
 	"sync"
@@ -43,13 +47,18 @@ func (mc *MetricsCollector) Run() error {
 	apiEndpoint.Path = "/api/devices/register"
 
 	deviceRespWrapper := new(models.DeviceWrapper)
-	detectedStorageDevices, err := mc.DetectStorageDevices()
+
+	deviceDetector := detect.Detect{
+		Logger: mc.logger,
+	}
+	detectedStorageDevices, err := deviceDetector.Start()
 	if err != nil {
 		return err
 	}
 
 	mc.logger.Infoln("Sending detected devices to API, for filtering & validation")
-	mc.logger.Debugf("Detected devices: %v", detectedStorageDevices)
+	jsonObj, _ := json.Marshal(detectedStorageDevices)
+	mc.logger.Debugf("Detected devices: %v", string(jsonObj))
 	err = mc.postJson(apiEndpoint.String(), models.DeviceWrapper{
 		Data: detectedStorageDevices,
 	}, &deviceRespWrapper)
@@ -67,7 +76,7 @@ func (mc *MetricsCollector) Run() error {
 		for _, device := range deviceRespWrapper.Data {
 			// execute collection in parallel go-routines
 			wg.Add(1)
-			go mc.Collect(&wg, device.WWN, device.DeviceName)
+			go mc.Collect(&wg, device.WWN, device.DeviceName, device.DeviceType)
 		}
 
 		mc.logger.Infoln("Main: Waiting for workers to finish")
@@ -89,11 +98,18 @@ func (mc *MetricsCollector) Validate() error {
 	return nil
 }
 
-func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string) {
+func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
 	defer wg.Done()
 	mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
 
-	result, err := mc.ExecCmd("smartctl", []string{"-a", "-j", fmt.Sprintf("/dev/%s", deviceName)}, "", nil)
+	args := []string{"-a", "-j"}
+	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
+	if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" {
+		args = append(args, "-d", deviceType)
+	}
+	args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName))
+
+	result, err := common.ExecCmd(mc.logger, "smartctl", args, "", os.Environ())
 	resultBytes := []byte(result)
 	if err != nil {
 		if exitError, ok := err.(*exec.ExitError); ok {
@@ -121,6 +137,7 @@ func (mc *MetricsCollector) Publish(deviceWWN string, payload []byte) error {
 
 	resp, err := httpClient.Post(apiEndpoint.String(), "application/json", bytes.NewBuffer(payload))
 	if err != nil {
+		mc.logger.Errorf("An error occurred while publishing SMART data for device (%s): %v", deviceWWN, err)
 		return err
 	}
 	defer resp.Body.Close()

collector/pkg/common/exec.go

@@ -0,0 +1,35 @@
+package common
+
+import (
+	"bytes"
+	"errors"
+	"github.com/sirupsen/logrus"
+	"io"
+	"os/exec"
+	"path"
+	"strings"
+)
+
+func ExecCmd(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
+	logger.Infof("Executing command: %s %s", cmdName, strings.Join(cmdArgs, " "))
+
+	cmd := exec.Command(cmdName, cmdArgs...)
+	var stdBuffer bytes.Buffer
+	mw := io.MultiWriter(logger.Logger.Out, &stdBuffer)
+
+	cmd.Stdout = mw
+	cmd.Stderr = mw
+
+	if environ != nil {
+		cmd.Env = environ
+	}
+	if workingDir != "" && path.IsAbs(workingDir) {
+		cmd.Dir = workingDir
+	} else if workingDir != "" {
+		return "", errors.New("Working Directory must be an absolute path")
+	}
+
+	err := cmd.Run()
+	return stdBuffer.String(), err
+
+}

collector/pkg/common/exec_test.go

@@ -1,7 +1,8 @@
-package collector_test
+package common_test
 
 import (
-	"github.com/analogj/scrutiny/collector/pkg/collector"
+	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"os/exec"
 	"testing"
@@ -11,10 +12,9 @@ func TestExecCmd(t *testing.T) {
 	t.Parallel()
 
 	//setup
-	bc := collector.BaseCollector{}
 
 	//test
-	result, err := bc.ExecCmd("echo", []string{"hello world"}, "", nil)
+	result, err := common.ExecCmd(logrus.WithField("exec", "test"), "echo", []string{"hello world"}, "", nil)
 
 	//assert
 	require.NoError(t, err)
@@ -25,10 +25,9 @@ func TestExecCmd_Date(t *testing.T) {
 	t.Parallel()
 
 	//setup
-	bc := collector.BaseCollector{}
 
 	//test
-	_, err := bc.ExecCmd("date", []string{}, "", nil)
+	_, err := common.ExecCmd(logrus.WithField("exec", "test"), "date", []string{}, "", nil)
 
 	//assert
 	require.NoError(t, err)
@@ -56,10 +55,9 @@ func TestExecCmd_InvalidCommand(t *testing.T) {
 	t.Parallel()
 
 	//setup
-	bc := collector.BaseCollector{}
 
 	//test
-	_, err := bc.ExecCmd("invalid_binary", []string{}, "", nil)
+	_, err := common.ExecCmd(logrus.WithField("exec", "test"), "invalid_binary", []string{}, "", nil)
 
 	//assert
 	_, castOk := err.(*exec.ExitError)

collector/pkg/detect/detect.go

@@ -0,0 +1,116 @@
+package detect
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/denisbrodbeck/machineid"
+	"github.com/sirupsen/logrus"
+	"os"
+	"strings"
+)
+
+type Detect struct {
+	Logger *logrus.Entry
+}
+
+//private/common functions
+
+// This function calls smartctl --scan which can be used to detect storage devices.
+// It has a couple of issues however:
+// - --scan does not return any results on mac
+//
+// To handle these issues, we have OS specific wrapper functions that update/modify these detected devices.
+// models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
+func (d *Detect) smartctlScan() ([]models.Device, error) {
+	//we use smartctl to detect all the drives available.
+	detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
+	if err != nil {
+		d.Logger.Errorf("Error scanning for devices: %v", err)
+		return nil, err
+	}
+
+	var detectedDeviceConns models.Scan
+	err = json.Unmarshal([]byte(detectedDeviceConnJson), &detectedDeviceConns)
+	if err != nil {
+		d.Logger.Errorf("Error decoding detected devices: %v", err)
+		return nil, err
+	}
+
+	detectedDevices := []models.Device{}
+
+	for _, detectedDevice := range detectedDeviceConns.Devices {
+		detectedDevices = append(detectedDevices, models.Device{
+			DeviceType: detectedDevice.Type,
+			DeviceName: strings.TrimPrefix(detectedDevice.Name, DevicePrefix()),
+		})
+	}
+
+	return detectedDevices, nil
+}
+
+//updates a device model with information from smartctl --scan
+// It has a couple of issues however:
+// - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
+// - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
+func (d *Detect) smartCtlInfo(device *models.Device) error {
+
+	args := []string{"--info", "-j"}
+	//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
+	if len(device.DeviceType) > 0 && device.DeviceType != "scsi" && device.DeviceType != "ata" {
+		args = append(args, "-d", device.DeviceType)
+	}
+	args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName))
+
+	availableDeviceInfoJson, err := common.ExecCmd(d.Logger, "smartctl", args, "", os.Environ())
+	if err != nil {
+		d.Logger.Errorf("Could not retrieve device information for %s: %v", device.DeviceName, err)
+		return err
+	}
+
+	var availableDeviceInfo collector.SmartInfo
+	err = json.Unmarshal([]byte(availableDeviceInfoJson), &availableDeviceInfo)
+	if err != nil {
+		d.Logger.Errorf("Could not decode device information for %s: %v", device.DeviceName, err)
+		return err
+	}
+
+	//DeviceType and DeviceName are already populated.
+	//WWN
+	//InterfaceType:
+	device.ModelName = availableDeviceInfo.ModelName
+	device.InterfaceSpeed = availableDeviceInfo.InterfaceSpeed.Current.String
+	device.SerialNumber = availableDeviceInfo.SerialNumber
+	device.Firmware = availableDeviceInfo.FirmwareVersion
+	device.RotationSpeed = availableDeviceInfo.RotationRate
+	device.Capacity = availableDeviceInfo.UserCapacity.Bytes
+	device.FormFactor = availableDeviceInfo.FormFactor.Name
+	device.DeviceProtocol = availableDeviceInfo.Device.Protocol
+	if len(availableDeviceInfo.Vendor) > 0 {
+		device.Manufacturer = availableDeviceInfo.Vendor
+	}
+
+	//populate WWN is possible if present
+	if availableDeviceInfo.Wwn.Naa != 0 { //valid values are 1-6 (5 is what we handle correctly)
+		d.Logger.Info("Generating WWN")
+		wwn := Wwn{
+			Naa: availableDeviceInfo.Wwn.Naa,
+			Oui: availableDeviceInfo.Wwn.Oui,
+			Id:  availableDeviceInfo.Wwn.ID,
+		}
+		device.WWN = wwn.ToString()
+		d.Logger.Debugf("NAA: %d OUI: %d Id: %d => WWN: %s", wwn.Naa, wwn.Oui, wwn.Id, device.WWN)
+	} else {
+		d.Logger.Info("Using WWN Fallback")
+		d.wwnFallback(device)
+	}
+
+	return nil
+}
+
+//uses https://github.com/denisbrodbeck/machineid to get a OS specific unique machine ID.
+func (d *Detect) getMachineId() (string, error) {
+	return machineid.ProtectedID("scrutiny")
+}

collector/pkg/detect/devices_darwin.go

@@ -0,0 +1,108 @@
+package detect
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/jaypipes/ghw"
+	"strings"
+)
+
+func DevicePrefix() string {
+	return "/dev/"
+}
+
+func (d *Detect) Start() ([]models.Device, error) {
+	// call the base/common functionality to get a list of devicess
+	detectedDevices, err := d.smartctlScan()
+	if err != nil {
+		return nil, err
+	}
+
+	//smartctl --scan doesn't seem to detect mac nvme drives, lets see if we can detect them manually.
+	missingDevices, err := d.findMissingDevices(detectedDevices) //we dont care about the error here, just continue retrieving device info.
+	if err == nil {
+		detectedDevices = append(detectedDevices, missingDevices...)
+	}
+
+	//inflate device info for detected devices.
+	for ndx, _ := range detectedDevices {
+		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+	}
+
+	return detectedDevices, nil
+}
+
+func (d *Detect) findMissingDevices(detectedDevices []models.Device) ([]models.Device, error) {
+
+	missingDevices := []models.Device{}
+
+	block, err := ghw.Block()
+	if err != nil {
+		d.Logger.Errorf("Error getting block storage info: %v", err)
+		return nil, err
+	}
+
+	for _, disk := range block.Disks {
+
+		// ignore optical drives and floppy disks
+		if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
+			d.Logger.Debugf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
+			continue
+		}
+
+		// ignore removable disks
+		if disk.IsRemovable {
+			d.Logger.Debugf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
+			continue
+		}
+
+		// ignore virtual disks & mobile phone storage devices
+		if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
+			d.Logger.Debugf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
+			continue
+		}
+
+		// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
+		if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
+			d.Logger.Debugf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
+			continue
+		}
+
+		//check if device is already detected.
+		alreadyDetected := false
+		diskName := strings.TrimPrefix(disk.Name, DevicePrefix())
+		for _, detectedDevice := range detectedDevices {
+
+			if detectedDevice.DeviceName == diskName {
+				alreadyDetected = true
+				break
+			}
+		}
+		if !alreadyDetected {
+			missingDevices = append(missingDevices, models.Device{
+				DeviceName: diskName,
+				DeviceType: "",
+			})
+		}
+	}
+	return missingDevices, nil
+}
+
+//WWN values NVMe and SCSI
+func (d *Detect) wwnFallback(detectedDevice *models.Device) {
+	block, err := ghw.Block()
+	if err == nil {
+		for _, disk := range block.Disks {
+			if disk.Name == detectedDevice.DeviceName {
+				d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN)
+				detectedDevice.WWN = disk.WWN
+				break
+			}
+		}
+	}
+
+	//no WWN found, or could not open Block devices. Either way, fallback to serial number
+	if len(detectedDevice.WWN) == 0 {
+		d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber)
+		detectedDevice.WWN = detectedDevice.SerialNumber
+	}
+}

collector/pkg/detect/devices_linux.go

@@ -0,0 +1,45 @@
+package detect
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/jaypipes/ghw"
+)
+
+func DevicePrefix() string {
+	return "/dev/"
+}
+
+func (d *Detect) Start() ([]models.Device, error) {
+	// call the base/common functionality to get a list of devices
+	detectedDevices, err := d.smartctlScan()
+	if err != nil {
+		return nil, err
+	}
+
+	//inflate device info for detected devices.
+	for ndx, _ := range detectedDevices {
+		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+	}
+
+	return detectedDevices, nil
+}
+
+//WWN values NVMe and SCSI
+func (d *Detect) wwnFallback(detectedDevice *models.Device) {
+	block, err := ghw.Block()
+	if err == nil {
+		for _, disk := range block.Disks {
+			if disk.Name == detectedDevice.DeviceName {
+				d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN)
+				detectedDevice.WWN = disk.WWN
+				break
+			}
+		}
+	}
+
+	//no WWN found, or could not open Block devices. Either way, fallback to serial number
+	if len(detectedDevice.WWN) == 0 {
+		d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber)
+		detectedDevice.WWN = detectedDevice.SerialNumber
+	}
+}

collector/pkg/detect/devices_windows.go

@@ -0,0 +1,29 @@
+package detect
+
+func DevicePrefix() string {
+	return ""
+}
+
+func (d *Detect) Start() ([]models.Device, error) {
+	// call the base/common functionality to get a list of devices
+	detectedDevices, err := d.smartctlScan()
+	if err != nil {
+		return nil, err
+	}
+
+	//inflate device info for detected devices.
+	for ndx, _ := range detectedDevices {
+		d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
+	}
+
+	return detectedDevices, nil
+}
+
+//WWN values NVMe and SCSI
+func (d *Detect) wwnFallback(detectedDevice *models.Device) {
+
+	//fallback to serial number
+	if len(detectedDevice.WWN) == 0 {
+		detectedDevice.WWN = detectedDevice.SerialNumber
+	}
+}

collector/pkg/detect/wwn.go

@@ -0,0 +1,58 @@
+package detect
+
+import (
+	"fmt"
+)
+
+type Wwn struct {
+	Naa        uint64 `json:"naa"`
+	Oui        uint64 `json:"oui"`
+	Id         uint64 `json:"id"`
+	VendorCode string `json:"vendor_code"`
+}
+
+// this is an incredibly basic converter, that only works for "Registered" IEEE format - NAA5
+// https://standards.ieee.org/content/dam/ieee-standards/standards/web/documents/tutorials/fibre.pdf
+// references:
+// - https://metacpan.org/pod/Device::WWN
+// - https://en.wikipedia.org/wiki/World_Wide_Name
+// - https://storagemeat.blogspot.com/2012/08/decoding-wwids-or-how-to-tell-whats-what.html
+// - https://bryanchain.com/2016/01/20/breaking-down-an-naa-id-world-wide-name/
+
+/*
++----------+---+---+---+---+---+---+---+---+
+| Byte/Bit | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
++----------+---+---+---+---+---+---+---+---+
+| 0        | NAA (5h)      | (MSB)         |
++----------+---------------+               +
+| 1        |                               |
++----------+            IEEE OUI           |
+| 2        |                               |
++----------+               +---------------+
+| 3        | (LSB)         | (MSB)         |
++----------+---------------+               +
+| 4        |                               |
+|          |                               |
++----------+                               |
+| 5        |            Vendor ID          |
++----------+                               |
+| 6        |                               |
++----------+                               |
+| 7        |                         (LSB) |
++----------+-------------------------------+
+
+
+*/
+
+func (wwn *Wwn) ToString() string {
+
+	var wwnBuffer uint64
+
+	wwnBuffer = wwn.Id           //start with vendor ID
+	wwnBuffer += (wwn.Oui << 36) //add left-shifted OUI
+	wwnBuffer += (wwn.Naa << 60) //NAA is a number from 1-6, so decimal == hex.
+
+	//TODO: may need to support additional versions in the future.
+
+	return fmt.Sprintf("%#x", wwnBuffer)
+}

collector/pkg/detect/wwn_test.go

@@ -0,0 +1,35 @@
+package detect_test
+
+import (
+	"fmt"
+	"github.com/analogj/scrutiny/collector/pkg/detect"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestWwn_FromStringTable(t *testing.T) {
+
+	//setup
+	var tests = []struct {
+		wwnStr string
+		wwn    detect.Wwn
+	}{
+
+		{"0x5002538e40a22954", detect.Wwn{Naa: 5, Oui: 9528, Id: 61213911380}}, //sda
+		{"0x5000cca264eb01d7", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283057623}}, //sdb
+		{"0x5000cca264ec3183", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283135363}}, //sdc
+		{"0x5000cca252c859cc", detect.Wwn{Naa: 5, Oui: 3274, Id: 9978796492}},  //sdd
+		{"0x50014ee20b2a72a9", detect.Wwn{Naa: 5, Oui: 5358, Id: 8777265833}},  //sde
+		{"0x5000cca264ebc248", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283106888}}, //sdf
+		{"0x5000c500673e6b5f", detect.Wwn{Naa: 5, Oui: 3152, Id: 1732143967}},  //sdg
+	}
+	//test
+	for _, tt := range tests {
+		testname := fmt.Sprintf("%s", tt.wwnStr)
+		t.Run(testname, func(t *testing.T) {
+			str := tt.wwn.ToString()
+			require.Equal(t, tt.wwnStr, str)
+		})
+	}
+
+}

go.mod

@@ -3,9 +3,10 @@ module github.com/analogj/scrutiny
 go 1.13
 
 require (
-	github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae
+	github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae // indirect
 	github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
 	github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a
+	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/fatih/color v1.9.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/golang/mock v1.4.3

go.sum

@@ -50,6 +50,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
+github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=

webapp/backend/pkg/models/collector/smart.go

@@ -23,9 +23,9 @@ type SmartInfo struct {
 	ModelName    string `json:"model_name"`
 	SerialNumber string `json:"serial_number"`
 	Wwn          struct {
-		Naa int   `json:"naa"`
+		Naa uint64 `json:"naa"`
-		Oui int   `json:"oui"`
+		Oui uint64 `json:"oui"`
-		ID  int64 `json:"id"`
+		ID  uint64 `json:"id"`
 	} `json:"wwn"`
 	FirmwareVersion string `json:"firmware_version"`
 	UserCapacity    struct {

webapp/backend/pkg/models/db/device.go

@@ -152,16 +152,6 @@ func (dv *Device) ApplyMetadataRules() error {
 }
 
 func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
-	dv.InterfaceSpeed = info.InterfaceSpeed.Current.String
 	dv.Firmware = info.FirmwareVersion
-	dv.RotationSpeed = info.RotationRate
-	dv.Capacity = info.UserCapacity.Bytes
-	dv.FormFactor = info.FormFactor.Name
-	dv.DeviceProtocol = info.Device.Protocol
-	dv.DeviceType = info.Device.Type
-	if len(info.Vendor) > 0 {
-		dv.Manufacturer = info.Vendor
-	}
-
 	return nil
 }

webapp/backend/pkg/models/testdata/smart-nvme2.json

@@ -0,0 +1,105 @@
+{
+  "json_format_version": [
+    1,
+    0
+  ],
+  "smartctl": {
+    "version": [
+      7,
+      0
+    ],
+    "svn_revision": "4883",
+    "platform_info": "x86_64-linux-4.19.107-Unraid",
+    "build_info": "(local build)",
+    "argv": [
+      "smartctl",
+      "-a",
+      "-j",
+      "-d",
+      "nvme",
+      "/dev/nvme0"
+    ],
+    "exit_status": 0
+  },
+  "device": {
+    "name": "/dev/nvme0",
+    "info_name": "/dev/nvme0",
+    "type": "nvme",
+    "protocol": "NVMe"
+  },
+  "model_name": "Force MP510",
+  "serial_number": "yes",
+  "firmware_version": "ECFM12.3",
+  "nvme_pci_vendor": {
+    "id": 6535,
+    "subsystem_id": 6535
+  },
+  "nvme_ieee_oui_identifier": 6584743,
+  "nvme_total_capacity": 480103981056,
+  "nvme_unallocated_capacity": 0,
+  "nvme_controller_id": 1,
+  "nvme_number_of_namespaces": 1,
+  "nvme_namespaces": [
+    {
+      "id": 1,
+      "size": {
+        "blocks": 937703088,
+        "bytes": 480103981056
+      },
+      "capacity": {
+        "blocks": 937703088,
+        "bytes": 480103981056
+      },
+      "utilization": {
+        "blocks": 937703088,
+        "bytes": 480103981056
+      },
+      "formatted_lba_size": 512,
+      "eui64": {
+        "oui": 6584743,
+        "ext_id": 171819811633
+      }
+    }
+  ],
+  "user_capacity": {
+    "blocks": 937703088,
+    "bytes": 480103981056
+  },
+  "logical_block_size": 512,
+  "local_time": {
+    "time_t": 1600619090,
+    "asctime": "Sun Sep 20 16:24:50 2020 Europe"
+  },
+  "smart_status": {
+    "passed": true,
+    "nvme": {
+      "value": 0
+    }
+  },
+  "nvme_smart_health_information_log": {
+    "critical_warning": 0,
+    "temperature": 38,
+    "available_spare": 100,
+    "available_spare_threshold": 5,
+    "percentage_used": 1,
+    "data_units_read": 6932144,
+    "data_units_written": 16093122,
+    "host_reads": 29878811,
+    "host_writes": 17533252,
+    "controller_busy_time": 305,
+    "power_cycles": 4,
+    "power_on_hours": 6487,
+    "unsafe_shutdowns": 4,
+    "media_errors": 0,
+    "num_err_log_entries": 8382,
+    "warning_temp_time": 0,
+    "critical_comp_time": 0
+  },
+  "temperature": {
+    "current": 38
+  },
+  "power_cycle_count": 4,
+  "power_on_time": {
+    "hours": 6487
+  }
+}

webapp/backend/pkg/web/server_test.go

@@ -1,24 +1,30 @@
 package web_test
 
 import (
+	"encoding/json"
 	mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
+	dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path"
 	"strings"
 	"testing"
 )
 
 func TestHealthRoute(t *testing.T) {
 	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
-	fakeConfig.EXPECT().GetString("web.database.location").Return("testdata/scrutiny_test.db")
+	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db"))
-	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return("testdata")
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath)
 
 	ae := web.AppEngine{
 		Config: fakeConfig,
@@ -38,11 +44,13 @@ func TestHealthRoute(t *testing.T) {
 
 func TestRegisterDevicesRoute(t *testing.T) {
 	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
-	fakeConfig.EXPECT().GetString("web.database.location").Return("testdata/scrutiny_test.db")
+	fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db"))
-	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return("testdata")
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath)
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
@@ -61,11 +69,13 @@ func TestRegisterDevicesRoute(t *testing.T) {
 
 func TestUploadDeviceMetricsRoute(t *testing.T) {
 	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
-	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata")
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
@@ -92,11 +102,14 @@ func TestUploadDeviceMetricsRoute(t *testing.T) {
 
 func TestPopulateMultiple(t *testing.T) {
 	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
-	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
+	//fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
-	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata")
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	ae := web.AppEngine{
 		Config: fakeConfig,
 	}
@@ -151,11 +164,13 @@ func TestPopulateMultiple(t *testing.T) {
 
 func TestSendTestNotificationRoute(t *testing.T) {
 	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 	fakeConfig := mock_config.NewMockInterface(mockCtrl)
-	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
-	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata")
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
 	fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://scrutiny.requestcatcher.com/test"})
 	ae := web.AppEngine{
 		Config: fakeConfig,
@@ -170,3 +185,45 @@ func TestSendTestNotificationRoute(t *testing.T) {
 	//assert
 	require.Equal(t, 200, wr.Code)
 }
+
+func TestGetDevicesSummaryRoute_Nvme(t *testing.T) {
+	//setup
+	parentPath, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(parentPath)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	fakeConfig := mock_config.NewMockInterface(mockCtrl)
+	fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
+	fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
+	ae := web.AppEngine{
+		Config: fakeConfig,
+	}
+	router := ae.Setup()
+	devicesfile, err := os.Open("testdata/register-devices-req-2.json")
+	require.NoError(t, err)
+
+	metricsfile, err := os.Open("../models/testdata/smart-nvme2.json")
+	require.NoError(t, err)
+
+	//test
+	wr := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/api/devices/register", devicesfile)
+	router.ServeHTTP(wr, req)
+	require.Equal(t, 200, wr.Code)
+
+	mr := httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/api/device/a4c8e8ed-11a0-4c97-9bba-306440f1b944/smart", metricsfile)
+	router.ServeHTTP(mr, req)
+	require.Equal(t, 200, mr.Code)
+
+	sr := httptest.NewRecorder()
+	req, _ = http.NewRequest("GET", "/api/summary", nil)
+	router.ServeHTTP(sr, req)
+	require.Equal(t, 200, sr.Code)
+	var device dbModels.DeviceWrapper
+	json.Unmarshal(sr.Body.Bytes(), &device)
+
+	//assert
+	require.Equal(t, "a4c8e8ed-11a0-4c97-9bba-306440f1b944", device.Data[0].WWN)
+	require.Equal(t, "passed", device.Data[0].SmartResults[0].SmartStatus)
+}

webapp/backend/pkg/web/testdata/register-devices-req-2.json

@@ -0,0 +1,20 @@
+{
+  "data": [
+    {
+      "wwn": "a4c8e8ed-11a0-4c97-9bba-306440f1b944",
+      "device_name": "nvme0",
+      "manufacturer": "",
+      "model_name": "Force MP510",
+      "interface_type": "",
+      "interface_speed": "",
+      "serial_number": "a4c8e8ed-11a0-4c97-9bba-306440f1b944",
+      "firmware": "ECFM12.3",
+      "rotational_speed": 0,
+      "capacity": 480103981056,
+      "form_factor": "",
+      "smart_support": false,
+      "device_protocol": "NVMe",
+      "device_type": "nvme"
+    }
+  ]
+}

webapp/frontend/src/app/modules/dashboard/dashboard.component.html

@@ -66,7 +66,7 @@
                         <div class="flex items-center">
                             <div class="flex flex-col">
                                 <a [routerLink]="'/device/'+ disk.wwn"
-                                   class="font-bold text-md text-secondary uppercase tracking-wider">/dev/{{disk.device_name}} - {{disk.model_name}}</a>
+                                   class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceTitle(disk)}}</a>
                                 <div [ngClass]="{'text-green': disk.smart_results[0]?.smart_status == 'passed',
                                     'text-red': disk.smart_results[0]?.smart_status == 'failed' }" class="font-medium text-sm" *ngIf="disk.smart_results[0]">
                                     Last Updated on {{disk.smart_results[0]?.date | date:'MMMM dd, yyyy' }}

webapp/frontend/src/app/modules/dashboard/dashboard.component.ts

@@ -164,6 +164,15 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
         });
     }
 
+    deviceTitle(disk){
+        var title = [`/dev/${disk.device_name}`]
+        if (disk.device_type && disk.device_type != 'scsi' && disk.device_type != 'ata'){
+            title.push(disk.device_type)
+        }
+        title.push(disk.model_name)
+        return title.join(' - ')
+    }
+
     /**
      * Track by function for ngFor loops
      *

webapp/frontend/src/app/modules/detail/detail.component.html

@@ -56,6 +56,10 @@
                     <div class="text-2xl font-semibold leading-tight">/dev/{{data.data.device_name}}</div>
                 </div>
                 <div class="flex flex-col my-2 grid grid-cols-2">
+                    <div *ngIf="data.data.device_type && data.data.device_type != 'ata' && data.data.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
+                        <div>{{data.data.device_type}}</div>
+                        <div class="text-secondary text-md">Device Type</div>
+                    </div>
                     <div *ngIf="data.data.manufacturer" class="my-2 col-span-2 lt-md:col-span-1">
                         <div>{{data.data.manufacturer}}</div>
                         <div class="text-secondary text-md">Model Family</div>