moved hard drive device detection to collector (So we can run multiple in parallel).

collector/pkg/collector/base.go

@@ -4,14 +4,87 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/jaypipes/ghw"
 	"io"
+	"net/http"
 	"os"
 	"os/exec"
 	"path"
+	"time"
 )
 
+var httpClient = &http.Client{Timeout: 10 * time.Second}
+
 type BaseCollector struct{}
 
+func (c *BaseCollector) detectStorageDevices() ([]models.Device, error) {
+
+	block, err := ghw.Block()
+	if err != nil {
+		fmt.Printf("Error getting block storage info: %v", err)
+		return nil, err
+	}
+
+	approvedDisks := []models.Device{}
+	for _, disk := range block.Disks {
+
+		// ignore optical drives and floppy disks
+		if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
+			fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
+			continue
+		}
+
+		// ignore removable disks
+		if disk.IsRemovable {
+			fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
+			continue
+		}
+
+		// ignore virtual disks & mobile phone storage devices
+		if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
+			fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
+			continue
+		}
+
+		// ignore NVMe devices (not currently supported) TBA
+		if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME {
+			fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String())
+			continue
+		}
+
+		// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
+		if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
+			fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
+			continue
+		}
+
+		diskModel := models.Device{
+			WWN:           disk.WWN,
+			Manufacturer:  disk.Vendor,
+			ModelName:     disk.Model,
+			InterfaceType: disk.StorageController.String(),
+			//InterfaceSpeed: string
+			SerialNumber: disk.SerialNumber,
+			Capacity:     int64(disk.SizeBytes),
+			//Firmware       string
+			//RotationSpeed  int
+
+			DeviceName: disk.Name,
+		}
+		if len(diskModel.WWN) == 0 {
+			//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
+			//diskname as identifier if WWN is not present
+			diskModel.WWN = disk.Name
+		}
+
+		approvedDisks = append(approvedDisks, diskModel)
+	}
+
+	return approvedDisks, nil
+}
+
 func (c *BaseCollector) getJson(url string, target interface{}) error {
 
 	r, err := httpClient.Get(url)
@@ -23,6 +96,21 @@ func (c *BaseCollector) getJson(url string, target interface{}) error {
 	return json.NewDecoder(r.Body).Decode(target)
 }
 
+func (c *BaseCollector) postJson(url string, body interface{}, target interface{}) error {
+	requestBody, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+
+	r, err := httpClient.Post(url, "application/json", bytes.NewBuffer(requestBody))
+	if err != nil {
+		return err
+	}
+	defer r.Body.Close()
+
+	return json.NewDecoder(r.Body).Decode(target)
+}
+
 func (c *BaseCollector) execCmd(cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
 
 	cmd := exec.Command(cmdName, cmdArgs...)

collector/pkg/collector/metrics.go

@@ -6,16 +6,12 @@ import (
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
 	"github.com/sirupsen/logrus"
-	"net/http"
 	"net/url"
 	"os/exec"
 	"strings"
 	"sync"
-	"time"
 )
 
-var httpClient = &http.Client{Timeout: 10 * time.Second}
-
 type MetricsCollector struct {
 	BaseCollector
 
@@ -44,12 +40,15 @@ func (mc *MetricsCollector) Run() error {
 	}
 
 	apiEndpoint, _ := url.Parse(mc.apiEndpoint.String())
-	apiEndpoint.Path = "/api/devices"
+	apiEndpoint.Path = "/api/devices/register"
 
-	deviceRespWrapper := new(models.DeviceRespWrapper)
+	deviceRespWrapper := new(models.DeviceWrapper)
+	detectedStorageDevices, err := mc.detectStorageDevices()
 
-	fmt.Println("Getting devices")
+	fmt.Println("Sending detected devices to API, for filtering & validation")
-	err = mc.getJson(apiEndpoint.String(), &deviceRespWrapper)
+	err = mc.postJson(apiEndpoint.String(), models.DeviceWrapper{
+		Data: detectedStorageDevices,
+	}, &deviceRespWrapper)
 	if err != nil {
 		return err
 	}

collector/pkg/models/device.go

@@ -1,21 +1,23 @@
 package models
 
 type Device struct {
-	WWN string `json:"wwn" gorm:"primary_key"`
+	WWN string `json:"wwn"`
 
 	DeviceName     string `json:"device_name"`
 	Manufacturer   string `json:"manufacturer"`
 	ModelName      string `json:"model_name"`
 	InterfaceType  string `json:"interface_type"`
 	InterfaceSpeed string `json:"interface_speed"`
-	SerialNumber   string `json:"serial_name"`
+	SerialNumber   string `json:"serial_number"`
-	Capacity       int64  `json:"capacity"`
 	Firmware       string `json:"firmware"`
 	RotationSpeed  int    `json:"rotational_speed"`
+	Capacity       int64  `json:"capacity"`
+	FormFactor     string `json:"form_factor"`
+	SmartSupport   bool   `json:"smart_support"`
 }
 
-type DeviceRespWrapper struct {
+type DeviceWrapper struct {
-	Success bool     `json:"success"`
+	Success bool     `json:"success,omitempty"`
-	Errors  []error  `json:"errors"`
+	Errors  []error  `json:"errors,omitempty"`
 	Data    []Device `json:"data"`
 }

webapp/backend/pkg/models/db/device.go

@@ -7,7 +7,7 @@ import (
 	"time"
 )
 
-type DeviceRespWrapper struct {
+type DeviceWrapper struct {
 	Success bool     `json:"success"`
 	Errors  []error  `json:"errors"`
 	Data    []Device `json:"data"`

webapp/backend/pkg/web/disk.go

@@ -1,77 +0,0 @@
-package web
-
-import (
-	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
-	"github.com/jaypipes/ghw"
-)
-
-func RetrieveStorageDevices() ([]db.Device, error) {
-
-	block, err := ghw.Block()
-	if err != nil {
-		fmt.Printf("Error getting block storage info: %v", err)
-		return nil, err
-	}
-
-	approvedDisks := []db.Device{}
-	for _, disk := range block.Disks {
-		//TODO: always allow if in approved list
-		fmt.Printf(" %v\n", disk)
-
-		// ignore optical drives and floppy disks
-		if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
-			fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
-			continue
-		}
-
-		// ignore removable disks
-		if disk.IsRemovable {
-			fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
-			continue
-		}
-
-		// ignore virtual disks & mobile phone storage devices
-		if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
-			fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
-			continue
-		}
-
-		// ignore NVMe devices (not currently supported) TBA
-		if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME {
-			fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String())
-			continue
-		}
-
-		// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
-		if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
-			fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
-			continue
-		}
-
-		//TODO: remove if in excluded list
-
-		diskModel := db.Device{
-			WWN:           disk.WWN,
-			Manufacturer:  disk.Vendor,
-			ModelName:     disk.Model,
-			InterfaceType: disk.StorageController.String(),
-			//InterfaceSpeed: string
-			SerialNumber: disk.SerialNumber,
-			Capacity:     int64(disk.SizeBytes),
-			//Firmware       string
-			//RotationSpeed  int
-
-			DeviceName: disk.Name,
-		}
-		if len(diskModel.WWN) == 0 {
-			//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
-			//diskname as identifier if WWN is not present
-			diskModel.WWN = disk.Name
-		}
-
-		approvedDisks = append(approvedDisks, diskModel)
-	}
-
-	return approvedDisks, nil
-}

webapp/backend/pkg/web/server.go

@@ -31,7 +31,7 @@ func (ae *AppEngine) Start() error {
 		})
 
 		//TODO: notifications
-		api.GET("/devices", GetDevicesHandler)
+		api.POST("/devices/register", RegisterDevices)
 		api.GET("/summary", GetDevicesSummary)
 		api.POST("/device/:wwn/smart", UploadDeviceSmartData)
 		api.POST("/device/:wwn/selftest", UploadDeviceSelfTestData)
@@ -55,12 +55,20 @@ func (ae *AppEngine) Start() error {
 	return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port"))) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080")
 }
 
-// Get all active disks for processing by collectors
+// filter devices that are detected by various collectors.
-func GetDevicesHandler(c *gin.Context) {
+func RegisterDevices(c *gin.Context) {
-	storageDevices, err := RetrieveStorageDevices()
-
 	db := c.MustGet("DB").(*gorm.DB)
-	for _, dev := range storageDevices {
+
+	var collectorDeviceWrapper dbModels.DeviceWrapper
+	err := c.BindJSON(&collectorDeviceWrapper)
+	if err != nil {
+		log.Error("Cannot parse detected devices")
+		c.JSON(http.StatusOK, gin.H{"success": false})
+	}
+
+	//TODO: filter devices here (remove excludes, force includes)
+
+	for _, dev := range collectorDeviceWrapper.Data {
 		//insert devices into DB if not already there.
 		db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev)
 	}
@@ -70,9 +78,9 @@ func GetDevicesHandler(c *gin.Context) {
 			"success": false,
 		})
 	} else {
-		c.JSON(http.StatusOK, gin.H{
+		c.JSON(http.StatusOK, dbModels.DeviceWrapper{
-			"success": true,
+			Success: true,
-			"data":    storageDevices,
+			Data:    collectorDeviceWrapper.Data,
 		})
 	}
 }