gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #38 from AnalogJ/smartctl_scan

Browse Source 
Smartctl scan
This commit is contained in:
Jason Kulatunga
2020-09-20 20:42:26 -06:00
committed by GitHub
parent f53833d617 fd82e9a4da
commit d6fad90fa4
23 changed files with 701 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
collector/cmd/collector-metrics/collector-metrics.go
+17
View File
@@ -5,6 +5,7 @@ import (
"github.com/analogj/scrutiny/collector/pkg/collector"
"github.com/analogj/scrutiny/webapp/backend/pkg/version"
"github.com/sirupsen/logrus"
"io"
"log"
"os"
"time"
@@ -85,6 +86,16 @@ OPTIONS:
logrus.SetLevel(logrus.InfoLevel)
}
if c.IsSet("log-file") {
logFile, err := os.OpenFile(c.String("log-file"), os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
logrus.Errorf("Failed to open log file %s for output: %s", c.String("log-file"), err)
return err
}
defer logFile.Close()
logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
}
metricCollector, err := collector.CreateMetricsCollector(
collectorLogger,
c.String("api-endpoint"),
@@ -105,6 +116,12 @@ OPTIONS:
EnvVars: []string{"SCRUTINY_API_ENDPOINT"},
},
&cli.StringFlag{
Name: "log-file",
Usage: "Path to file for logging. Leave empty to use STDOUT",
Value: "",
},
&cli.BoolFlag{
Name: "debug",
Usage: "Enable debug logging",
collector/cmd/collector-selftest/collector-selftest.go
+17
View File
@@ -5,6 +5,7 @@ import (
"github.com/analogj/scrutiny/collector/pkg/collector"
"github.com/analogj/scrutiny/webapp/backend/pkg/version"
"github.com/sirupsen/logrus"
"io"
"log"
"os"
"time"
@@ -85,6 +86,16 @@ OPTIONS:
logrus.SetLevel(logrus.InfoLevel)
}
if c.IsSet("log-file") {
logFile, err := os.OpenFile(c.String("log-file"), os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
logrus.Errorf("Failed to open log file %s for output: %s", c.String("log-file"), err)
return err
}
defer logFile.Close()
logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
}
stCollector, err := collector.CreateSelfTestCollector(
collectorLogger,
c.String("api-endpoint"),
@@ -105,6 +116,12 @@ OPTIONS:
EnvVars: []string{"SCRUTINY_API_ENDPOINT"},
},
&cli.StringFlag{
Name: "log-file",
Usage: "Path to file for logging. Leave empty to use STDOUT",
Value: "",
},
&cli.BoolFlag{
Name: "debug",
Usage: "Enable debug logging",
collector/pkg/collector/base.go
-101
View File
@@ -3,15 +3,8 @@ package collector
import (
"bytes"
"encoding/json"
"errors"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/jaypipes/ghw"
"github.com/sirupsen/logrus"
"io"
"net/http"
"os"
"os/exec"
"path"
"time"
)
@@ -21,77 +14,6 @@ type BaseCollector struct {
logger *logrus.Entry
}
func (c *BaseCollector) DetectStorageDevices() ([]models.Device, error) {
//availableDisksJson, err := c.ExecCmd("smartctl", []string{"-j", "--scan"}, "", os.Environ())
//if err != nil {
// c.logger.Errorf("Error getting block storage info: %v", err)
// return nil, err
//}
//
//var smartctlScan models.Scan
//err = json.Unmarshal([]byte(availableDisksJson), &smartctlScan)
//if err != nil {
// return nil, err
//}
block, err := ghw.Block()
if err != nil {
c.logger.Errorf("Error getting block storage info: %v", err)
return nil, err
}
approvedDisks := []models.Device{}
for _, disk := range block.Disks {
// ignore optical drives and floppy disks
if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
c.logger.Debugf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
continue
}
// ignore removable disks
if disk.IsRemovable {
c.logger.Debugf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
continue
}
// ignore virtual disks & mobile phone storage devices
if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
c.logger.Debugf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
c.logger.Debugf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
continue
}
diskModel := models.Device{
WWN: disk.WWN,
Manufacturer: disk.Vendor,
ModelName: disk.Model,
InterfaceType: disk.StorageController.String(),
//InterfaceSpeed: string
SerialNumber: disk.SerialNumber,
Capacity: int64(disk.SizeBytes),
//Firmware string
//RotationSpeed int
DeviceName: disk.Name,
}
if len(diskModel.WWN) == 0 {
//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
//diskname as identifier if WWN is not present
diskModel.WWN = disk.Name
}
approvedDisks = append(approvedDisks, diskModel)
}
return approvedDisks, nil
}
func (c *BaseCollector) getJson(url string, target interface{}) error {
r, err := httpClient.Get(url)
@@ -118,29 +40,6 @@ func (c *BaseCollector) postJson(url string, body interface{}, target interface{
return json.NewDecoder(r.Body).Decode(target)
}
func (c *BaseCollector) ExecCmd(cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
cmd := exec.Command(cmdName, cmdArgs...)
var stdBuffer bytes.Buffer
mw := io.MultiWriter(os.Stdout, &stdBuffer)
cmd.Stdout = mw
cmd.Stderr = mw
if environ != nil {
cmd.Env = environ
}
if workingDir != "" && path.IsAbs(workingDir) {
cmd.Dir = workingDir
} else if workingDir != "" {
return "", errors.New("Working Directory must be an absolute path")
}
err := cmd.Run()
return stdBuffer.String(), err
}
func (c *BaseCollector) LogSmartctlExitCode(exitCode int) {
if exitCode&0x01 != 0 {
c.logger.Errorln("smartctl could not parse commandline")
collector/pkg/collector/metrics.go
+22 -5
View File
@@ -2,11 +2,15 @@ package collector
import (
"bytes"
"encoding/json"
"fmt"
"github.com/analogj/scrutiny/collector/pkg/common"
"github.com/analogj/scrutiny/collector/pkg/detect"
"github.com/analogj/scrutiny/collector/pkg/errors"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/sirupsen/logrus"
"net/url"
"os"
"os/exec"
"strings"
"sync"
@@ -43,13 +47,18 @@ func (mc *MetricsCollector) Run() error {
apiEndpoint.Path = "/api/devices/register"
deviceRespWrapper := new(models.DeviceWrapper)
detectedStorageDevices, err := mc.DetectStorageDevices()
deviceDetector := detect.Detect{
Logger: mc.logger,
}
detectedStorageDevices, err := deviceDetector.Start()
if err != nil {
return err
}
mc.logger.Infoln("Sending detected devices to API, for filtering & validation")
mc.logger.Debugf("Detected devices: %v", detectedStorageDevices)
jsonObj, _ := json.Marshal(detectedStorageDevices)
mc.logger.Debugf("Detected devices: %v", string(jsonObj))
err = mc.postJson(apiEndpoint.String(), models.DeviceWrapper{
Data: detectedStorageDevices,
}, &deviceRespWrapper)
@@ -67,7 +76,7 @@ func (mc *MetricsCollector) Run() error {
for _, device := range deviceRespWrapper.Data {
// execute collection in parallel go-routines
wg.Add(1)
go mc.Collect(&wg, device.WWN, device.DeviceName)
go mc.Collect(&wg, device.WWN, device.DeviceName, device.DeviceType)
}
mc.logger.Infoln("Main: Waiting for workers to finish")
@@ -89,11 +98,18 @@ func (mc *MetricsCollector) Validate() error {
return nil
}
func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string) {
func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
defer wg.Done()
mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
result, err := mc.ExecCmd("smartctl", []string{"-a", "-j", fmt.Sprintf("/dev/%s", deviceName)}, "", nil)
args := []string{"-a", "-j"}
//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" {
args = append(args, "-d", deviceType)
}
args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName))
result, err := common.ExecCmd(mc.logger, "smartctl", args, "", os.Environ())
resultBytes := []byte(result)
if err != nil {
if exitError, ok := err.(*exec.ExitError); ok {
@@ -121,6 +137,7 @@ func (mc *MetricsCollector) Publish(deviceWWN string, payload []byte) error {
resp, err := httpClient.Post(apiEndpoint.String(), "application/json", bytes.NewBuffer(payload))
if err != nil {
mc.logger.Errorf("An error occurred while publishing SMART data for device (%s): %v", deviceWWN, err)
return err
}
defer resp.Body.Close()
collector/pkg/common/exec.go
+35
View File
@@ -0,0 +1,35 @@
package common
import (
"bytes"
"errors"
"github.com/sirupsen/logrus"
"io"
"os/exec"
"path"
"strings"
)
func ExecCmd(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
logger.Infof("Executing command: %s %s", cmdName, strings.Join(cmdArgs, " "))
cmd := exec.Command(cmdName, cmdArgs...)
var stdBuffer bytes.Buffer
mw := io.MultiWriter(logger.Logger.Out, &stdBuffer)
cmd.Stdout = mw
cmd.Stderr = mw
if environ != nil {
cmd.Env = environ
}
if workingDir != "" && path.IsAbs(workingDir) {
cmd.Dir = workingDir
} else if workingDir != "" {
return "", errors.New("Working Directory must be an absolute path")
}
err := cmd.Run()
return stdBuffer.String(), err
}
collector/pkg/collector/base_test.go → collector/pkg/common/exec_test.go
+6 -8
View File
@@ -1,7 +1,8 @@
package collector_test
package common_test
import (
"github.com/analogj/scrutiny/collector/pkg/collector"
"github.com/analogj/scrutiny/collector/pkg/common"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"os/exec"
"testing"
@@ -11,10 +12,9 @@ func TestExecCmd(t *testing.T) {
t.Parallel()
//setup
bc := collector.BaseCollector{}
//test
result, err := bc.ExecCmd("echo", []string{"hello world"}, "", nil)
result, err := common.ExecCmd(logrus.WithField("exec", "test"), "echo", []string{"hello world"}, "", nil)
//assert
require.NoError(t, err)
@@ -25,10 +25,9 @@ func TestExecCmd_Date(t *testing.T) {
t.Parallel()
//setup
bc := collector.BaseCollector{}
//test
_, err := bc.ExecCmd("date", []string{}, "", nil)
_, err := common.ExecCmd(logrus.WithField("exec", "test"), "date", []string{}, "", nil)
//assert
require.NoError(t, err)
@@ -56,10 +55,9 @@ func TestExecCmd_InvalidCommand(t *testing.T) {
t.Parallel()
//setup
bc := collector.BaseCollector{}
//test
_, err := bc.ExecCmd("invalid_binary", []string{}, "", nil)
_, err := common.ExecCmd(logrus.WithField("exec", "test"), "invalid_binary", []string{}, "", nil)
//assert
_, castOk := err.(*exec.ExitError)
collector/pkg/detect/detect.go
+116
View File
@@ -0,0 +1,116 @@
package detect
import (
"encoding/json"
"fmt"
"github.com/analogj/scrutiny/collector/pkg/common"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/denisbrodbeck/machineid"
"github.com/sirupsen/logrus"
"os"
"strings"
)
type Detect struct {
Logger *logrus.Entry
}
//private/common functions
// This function calls smartctl --scan which can be used to detect storage devices.
// It has a couple of issues however:
// - --scan does not return any results on mac
//
// To handle these issues, we have OS specific wrapper functions that update/modify these detected devices.
// models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
func (d *Detect) smartctlScan() ([]models.Device, error) {
//we use smartctl to detect all the drives available.
detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
if err != nil {
d.Logger.Errorf("Error scanning for devices: %v", err)
return nil, err
}
var detectedDeviceConns models.Scan
err = json.Unmarshal([]byte(detectedDeviceConnJson), &detectedDeviceConns)
if err != nil {
d.Logger.Errorf("Error decoding detected devices: %v", err)
return nil, err
}
detectedDevices := []models.Device{}
for _, detectedDevice := range detectedDeviceConns.Devices {
detectedDevices = append(detectedDevices, models.Device{
DeviceType: detectedDevice.Type,
DeviceName: strings.TrimPrefix(detectedDevice.Name, DevicePrefix()),
})
}
return detectedDevices, nil
}
//updates a device model with information from smartctl --scan
// It has a couple of issues however:
// - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
// - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
func (d *Detect) smartCtlInfo(device *models.Device) error {
args := []string{"--info", "-j"}
//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
if len(device.DeviceType) > 0 && device.DeviceType != "scsi" && device.DeviceType != "ata" {
args = append(args, "-d", device.DeviceType)
}
args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName))
availableDeviceInfoJson, err := common.ExecCmd(d.Logger, "smartctl", args, "", os.Environ())
if err != nil {
d.Logger.Errorf("Could not retrieve device information for %s: %v", device.DeviceName, err)
return err
}
var availableDeviceInfo collector.SmartInfo
err = json.Unmarshal([]byte(availableDeviceInfoJson), &availableDeviceInfo)
if err != nil {
d.Logger.Errorf("Could not decode device information for %s: %v", device.DeviceName, err)
return err
}
//DeviceType and DeviceName are already populated.
//WWN
//InterfaceType:
device.ModelName = availableDeviceInfo.ModelName
device.InterfaceSpeed = availableDeviceInfo.InterfaceSpeed.Current.String
device.SerialNumber = availableDeviceInfo.SerialNumber
device.Firmware = availableDeviceInfo.FirmwareVersion
device.RotationSpeed = availableDeviceInfo.RotationRate
device.Capacity = availableDeviceInfo.UserCapacity.Bytes
device.FormFactor = availableDeviceInfo.FormFactor.Name
device.DeviceProtocol = availableDeviceInfo.Device.Protocol
if len(availableDeviceInfo.Vendor) > 0 {
device.Manufacturer = availableDeviceInfo.Vendor
}
//populate WWN is possible if present
if availableDeviceInfo.Wwn.Naa != 0 { //valid values are 1-6 (5 is what we handle correctly)
d.Logger.Info("Generating WWN")
wwn := Wwn{
Naa: availableDeviceInfo.Wwn.Naa,
Oui: availableDeviceInfo.Wwn.Oui,
Id: availableDeviceInfo.Wwn.ID,
}
device.WWN = wwn.ToString()
d.Logger.Debugf("NAA: %d OUI: %d Id: %d => WWN: %s", wwn.Naa, wwn.Oui, wwn.Id, device.WWN)
} else {
d.Logger.Info("Using WWN Fallback")
d.wwnFallback(device)
}
return nil
}
//uses https://github.com/denisbrodbeck/machineid to get a OS specific unique machine ID.
func (d *Detect) getMachineId() (string, error) {
return machineid.ProtectedID("scrutiny")
}
collector/pkg/detect/devices_darwin.go
+108
View File
@@ -0,0 +1,108 @@
package detect
import (
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/jaypipes/ghw"
"strings"
)
func DevicePrefix() string {
return "/dev/"
}
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devicess
detectedDevices, err := d.smartctlScan()
if err != nil {
return nil, err
}
//smartctl --scan doesn't seem to detect mac nvme drives, lets see if we can detect them manually.
missingDevices, err := d.findMissingDevices(detectedDevices) //we dont care about the error here, just continue retrieving device info.
if err == nil {
detectedDevices = append(detectedDevices, missingDevices...)
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
}
func (d *Detect) findMissingDevices(detectedDevices []models.Device) ([]models.Device, error) {
missingDevices := []models.Device{}
block, err := ghw.Block()
if err != nil {
d.Logger.Errorf("Error getting block storage info: %v", err)
return nil, err
}
for _, disk := range block.Disks {
// ignore optical drives and floppy disks
if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
d.Logger.Debugf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
continue
}
// ignore removable disks
if disk.IsRemovable {
d.Logger.Debugf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
continue
}
// ignore virtual disks & mobile phone storage devices
if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
d.Logger.Debugf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
d.Logger.Debugf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
continue
}
//check if device is already detected.
alreadyDetected := false
diskName := strings.TrimPrefix(disk.Name, DevicePrefix())
for _, detectedDevice := range detectedDevices {
if detectedDevice.DeviceName == diskName {
alreadyDetected = true
break
}
}
if !alreadyDetected {
missingDevices = append(missingDevices, models.Device{
DeviceName: diskName,
DeviceType: "",
})
}
}
return missingDevices, nil
}
//WWN values NVMe and SCSI
func (d *Detect) wwnFallback(detectedDevice *models.Device) {
block, err := ghw.Block()
if err == nil {
for _, disk := range block.Disks {
if disk.Name == detectedDevice.DeviceName {
d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN)
detectedDevice.WWN = disk.WWN
break
}
}
}
//no WWN found, or could not open Block devices. Either way, fallback to serial number
if len(detectedDevice.WWN) == 0 {
d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber)
detectedDevice.WWN = detectedDevice.SerialNumber
}
}
collector/pkg/detect/devices_linux.go
+45
View File
@@ -0,0 +1,45 @@
package detect
import (
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/jaypipes/ghw"
)
func DevicePrefix() string {
return "/dev/"
}
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devices
detectedDevices, err := d.smartctlScan()
if err != nil {
return nil, err
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
}
//WWN values NVMe and SCSI
func (d *Detect) wwnFallback(detectedDevice *models.Device) {
block, err := ghw.Block()
if err == nil {
for _, disk := range block.Disks {
if disk.Name == detectedDevice.DeviceName {
d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN)
detectedDevice.WWN = disk.WWN
break
}
}
}
//no WWN found, or could not open Block devices. Either way, fallback to serial number
if len(detectedDevice.WWN) == 0 {
d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber)
detectedDevice.WWN = detectedDevice.SerialNumber
}
}
collector/pkg/detect/devices_windows.go
+29
View File
@@ -0,0 +1,29 @@
package detect
func DevicePrefix() string {
return ""
}
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devices
detectedDevices, err := d.smartctlScan()
if err != nil {
return nil, err
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
}
//WWN values NVMe and SCSI
func (d *Detect) wwnFallback(detectedDevice *models.Device) {
//fallback to serial number
if len(detectedDevice.WWN) == 0 {
detectedDevice.WWN = detectedDevice.SerialNumber
}
}
collector/pkg/detect/wwn.go
+58
View File
@@ -0,0 +1,58 @@
package detect
import (
"fmt"
)
type Wwn struct {
Naa uint64 `json:"naa"`
Oui uint64 `json:"oui"`
Id uint64 `json:"id"`
VendorCode string `json:"vendor_code"`
}
// this is an incredibly basic converter, that only works for "Registered" IEEE format - NAA5
// https://standards.ieee.org/content/dam/ieee-standards/standards/web/documents/tutorials/fibre.pdf
// references:
// - https://metacpan.org/pod/Device::WWN
// - https://en.wikipedia.org/wiki/World_Wide_Name
// - https://storagemeat.blogspot.com/2012/08/decoding-wwids-or-how-to-tell-whats-what.html
// - https://bryanchain.com/2016/01/20/breaking-down-an-naa-id-world-wide-name/
/*
+----------+---+---+---+---+---+---+---+---+
| Byte/Bit | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+----------+---+---+---+---+---+---+---+---+
| 0 | NAA (5h) | (MSB) |
+----------+---------------+ +
| 1 | |
+----------+ IEEE OUI |
| 2 | |
+----------+ +---------------+
| 3 | (LSB) | (MSB) |
+----------+---------------+ +
| 4 | |
| | |
+----------+ |
| 5 | Vendor ID |
+----------+ |
| 6 | |
+----------+ |
| 7 | (LSB) |
+----------+-------------------------------+
*/
func (wwn *Wwn) ToString() string {
var wwnBuffer uint64
wwnBuffer = wwn.Id //start with vendor ID
wwnBuffer += (wwn.Oui << 36) //add left-shifted OUI
wwnBuffer += (wwn.Naa << 60) //NAA is a number from 1-6, so decimal == hex.
//TODO: may need to support additional versions in the future.
return fmt.Sprintf("%#x", wwnBuffer)
}
collector/pkg/detect/wwn_test.go
+35
View File
@@ -0,0 +1,35 @@
package detect_test
import (
"fmt"
"github.com/analogj/scrutiny/collector/pkg/detect"
"github.com/stretchr/testify/require"
"testing"
)
func TestWwn_FromStringTable(t *testing.T) {
//setup
var tests = []struct {
wwnStr string
wwn detect.Wwn
}{
{"0x5002538e40a22954", detect.Wwn{Naa: 5, Oui: 9528, Id: 61213911380}}, //sda
{"0x5000cca264eb01d7", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283057623}}, //sdb
{"0x5000cca264ec3183", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283135363}}, //sdc
{"0x5000cca252c859cc", detect.Wwn{Naa: 5, Oui: 3274, Id: 9978796492}}, //sdd
{"0x50014ee20b2a72a9", detect.Wwn{Naa: 5, Oui: 5358, Id: 8777265833}}, //sde
{"0x5000cca264ebc248", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283106888}}, //sdf
{"0x5000c500673e6b5f", detect.Wwn{Naa: 5, Oui: 3152, Id: 1732143967}}, //sdg
}
//test
for _, tt := range tests {
testname := fmt.Sprintf("%s", tt.wwnStr)
t.Run(testname, func(t *testing.T) {
str := tt.wwn.ToString()
require.Equal(t, tt.wwnStr, str)
})
}
}
go.mod
+2 -1
View File
@@ -3,9 +3,10 @@ module github.com/analogj/scrutiny
go 1.13
require (
github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae
github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae // indirect
github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14
github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a
github.com/denisbrodbeck/machineid v1.0.1
github.com/fatih/color v1.9.0
github.com/gin-gonic/gin v1.6.3
github.com/golang/mock v1.4.3
go.sum
+2
View File
@@ -50,6 +50,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
webapp/backend/pkg/models/collector/smart.go
+3 -3
View File
@@ -23,9 +23,9 @@ type SmartInfo struct {
ModelName string `json:"model_name"`
SerialNumber string `json:"serial_number"`
Wwn struct {
Naa int `json:"naa"`
Oui int `json:"oui"`
ID int64 `json:"id"`
Naa uint64 `json:"naa"`
Oui uint64 `json:"oui"`
ID uint64 `json:"id"`
} `json:"wwn"`
FirmwareVersion string `json:"firmware_version"`
UserCapacity struct {
webapp/backend/pkg/models/db/device.go
-10
View File
@@ -152,16 +152,6 @@ func (dv *Device) ApplyMetadataRules() error {
}
func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
dv.InterfaceSpeed = info.InterfaceSpeed.Current.String
dv.Firmware = info.FirmwareVersion
dv.RotationSpeed = info.RotationRate
dv.Capacity = info.UserCapacity.Bytes
dv.FormFactor = info.FormFactor.Name
dv.DeviceProtocol = info.Device.Protocol
dv.DeviceType = info.Device.Type
if len(info.Vendor) > 0 {
dv.Manufacturer = info.Vendor
}
return nil
}
webapp/backend/pkg/models/testdata/smart-nvme2.json
Vendored
+105
View File
@@ -0,0 +1,105 @@
{
"json_format_version": [
1,
0
],
"smartctl": {
"version": [
7,
0
],
"svn_revision": "4883",
"platform_info": "x86_64-linux-4.19.107-Unraid",
"build_info": "(local build)",
"argv": [
"smartctl",
"-a",
"-j",
"-d",
"nvme",
"/dev/nvme0"
],
"exit_status": 0
},
"device": {
"name": "/dev/nvme0",
"info_name": "/dev/nvme0",
"type": "nvme",
"protocol": "NVMe"
},
"model_name": "Force MP510",
"serial_number": "yes",
"firmware_version": "ECFM12.3",
"nvme_pci_vendor": {
"id": 6535,
"subsystem_id": 6535
},
"nvme_ieee_oui_identifier": 6584743,
"nvme_total_capacity": 480103981056,
"nvme_unallocated_capacity": 0,
"nvme_controller_id": 1,
"nvme_number_of_namespaces": 1,
"nvme_namespaces": [
{
"id": 1,
"size": {
"blocks": 937703088,
"bytes": 480103981056
},
"capacity": {
"blocks": 937703088,
"bytes": 480103981056
},
"utilization": {
"blocks": 937703088,
"bytes": 480103981056
},
"formatted_lba_size": 512,
"eui64": {
"oui": 6584743,
"ext_id": 171819811633
}
}
],
"user_capacity": {
"blocks": 937703088,
"bytes": 480103981056
},
"logical_block_size": 512,
"local_time": {
"time_t": 1600619090,
"asctime": "Sun Sep 20 16:24:50 2020 Europe"
},
"smart_status": {
"passed": true,
"nvme": {
"value": 0
}
},
"nvme_smart_health_information_log": {
"critical_warning": 0,
"temperature": 38,
"available_spare": 100,
"available_spare_threshold": 5,
"percentage_used": 1,
"data_units_read": 6932144,
"data_units_written": 16093122,
"host_reads": 29878811,
"host_writes": 17533252,
"controller_busy_time": 305,
"power_cycles": 4,
"power_on_hours": 6487,
"unsafe_shutdowns": 4,
"media_errors": 0,
"num_err_log_entries": 8382,
"warning_temp_time": 0,
"critical_comp_time": 0
},
"temperature": {
"current": 38
},
"power_cycle_count": 4,
"power_on_time": {
"hours": 6487
}
}
webapp/backend/pkg/web/server_test.go
+67 -10
View File
@@ -1,24 +1,30 @@
package web_test
import (
"encoding/json"
mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/web"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"path"
"strings"
"testing"
)
func TestHealthRoute(t *testing.T) {
//setup
parentPath, _ := ioutil.TempDir("", "")
defer os.RemoveAll(parentPath)
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("web.database.location").Return("testdata/scrutiny_test.db")
fakeConfig.EXPECT().GetString("web.src.frontend.path").Return("testdata")
fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db"))
fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath)
ae := web.AppEngine{
Config: fakeConfig,
@@ -38,11 +44,13 @@ func TestHealthRoute(t *testing.T) {
func TestRegisterDevicesRoute(t *testing.T) {
//setup
parentPath, _ := ioutil.TempDir("", "")
defer os.RemoveAll(parentPath)
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("web.database.location").Return("testdata/scrutiny_test.db")
fakeConfig.EXPECT().GetString("web.src.frontend.path").Return("testdata")
fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db"))
fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath)
ae := web.AppEngine{
Config: fakeConfig,
}
@@ -61,11 +69,13 @@ func TestRegisterDevicesRoute(t *testing.T) {
func TestUploadDeviceMetricsRoute(t *testing.T) {
//setup
parentPath, _ := ioutil.TempDir("", "")
defer os.RemoveAll(parentPath)
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata")
fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
ae := web.AppEngine{
Config: fakeConfig,
}
@@ -92,11 +102,14 @@ func TestUploadDeviceMetricsRoute(t *testing.T) {
func TestPopulateMultiple(t *testing.T) {
//setup
parentPath, _ := ioutil.TempDir("", "")
defer os.RemoveAll(parentPath)
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata")
//fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
ae := web.AppEngine{
Config: fakeConfig,
}
@@ -151,11 +164,13 @@ func TestPopulateMultiple(t *testing.T) {
func TestSendTestNotificationRoute(t *testing.T) {
//setup
parentPath, _ := ioutil.TempDir("", "")
defer os.RemoveAll(parentPath)
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db")
fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata")
fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://scrutiny.requestcatcher.com/test"})
ae := web.AppEngine{
Config: fakeConfig,
@@ -170,3 +185,45 @@ func TestSendTestNotificationRoute(t *testing.T) {
//assert
require.Equal(t, 200, wr.Code)
}
func TestGetDevicesSummaryRoute_Nvme(t *testing.T) {
//setup
parentPath, _ := ioutil.TempDir("", "")
defer os.RemoveAll(parentPath)
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db"))
fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath)
ae := web.AppEngine{
Config: fakeConfig,
}
router := ae.Setup()
devicesfile, err := os.Open("testdata/register-devices-req-2.json")
require.NoError(t, err)
metricsfile, err := os.Open("../models/testdata/smart-nvme2.json")
require.NoError(t, err)
//test
wr := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/api/devices/register", devicesfile)
router.ServeHTTP(wr, req)
require.Equal(t, 200, wr.Code)
mr := httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/api/device/a4c8e8ed-11a0-4c97-9bba-306440f1b944/smart", metricsfile)
router.ServeHTTP(mr, req)
require.Equal(t, 200, mr.Code)
sr := httptest.NewRecorder()
req, _ = http.NewRequest("GET", "/api/summary", nil)
router.ServeHTTP(sr, req)
require.Equal(t, 200, sr.Code)
var device dbModels.DeviceWrapper
json.Unmarshal(sr.Body.Bytes(), &device)
//assert
require.Equal(t, "a4c8e8ed-11a0-4c97-9bba-306440f1b944", device.Data[0].WWN)
require.Equal(t, "passed", device.Data[0].SmartResults[0].SmartStatus)
}
webapp/backend/pkg/web/testdata/register-devices-req-2.json
Vendored
+20
View File
@@ -0,0 +1,20 @@
{
"data": [
{
"wwn": "a4c8e8ed-11a0-4c97-9bba-306440f1b944",
"device_name": "nvme0",
"manufacturer": "",
"model_name": "Force MP510",
"interface_type": "",
"interface_speed": "",
"serial_number": "a4c8e8ed-11a0-4c97-9bba-306440f1b944",
"firmware": "ECFM12.3",
"rotational_speed": 0,
"capacity": 480103981056,
"form_factor": "",
"smart_support": false,
"device_protocol": "NVMe",
"device_type": "nvme"
}
]
}
webapp/backend/pkg/web/testdata/scrutiny_test.db
Vendored
BIN
View File
Binary file not shown.
webapp/frontend/src/app/modules/dashboard/dashboard.component.html
+1 -1
View File
@@ -66,7 +66,7 @@
<div class="flex items-center">
<div class="flex flex-col">
<a [routerLink]="'/device/'+ disk.wwn"
class="font-bold text-md text-secondary uppercase tracking-wider">/dev/{{disk.device_name}} - {{disk.model_name}}</a>
class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceTitle(disk)}}</a>
<div [ngClass]="{'text-green': disk.smart_results[0]?.smart_status == 'passed',
'text-red': disk.smart_results[0]?.smart_status == 'failed' }" class="font-medium text-sm" *ngIf="disk.smart_results[0]">
Last Updated on {{disk.smart_results[0]?.date | date:'MMMM dd, yyyy' }}
webapp/frontend/src/app/modules/dashboard/dashboard.component.ts
+9
View File
@@ -164,6 +164,15 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy
});
}
deviceTitle(disk){
var title = [`/dev/${disk.device_name}`]
if (disk.device_type && disk.device_type != 'scsi' && disk.device_type != 'ata'){
title.push(disk.device_type)
}
title.push(disk.model_name)
return title.join(' - ')
}
/**
* Track by function for ngFor loops
*
webapp/frontend/src/app/modules/detail/detail.component.html
+4
View File
@@ -56,6 +56,10 @@
<div class="text-2xl font-semibold leading-tight">/dev/{{data.data.device_name}}</div>
</div>
<div class="flex flex-col my-2 grid grid-cols-2">
<div *ngIf="data.data.device_type && data.data.device_type != 'ata' && data.data.device_type != 'scsi'" class="my-2 col-span-2 lt-md:col-span-1">
<div>{{data.data.device_type}}</div>
<div class="text-secondary text-md">Device Type</div>
</div>
<div *ngIf="data.data.manufacturer" class="my-2 col-span-2 lt-md:col-span-1">
<div>{{data.data.manufacturer}}</div>
<div class="text-secondary text-md">Model Family</div>