gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved hard drive device detection to collector (So we can run multiple in parallel).

Browse Source
This commit is contained in:
Jason Kulatunga
2020-08-19 22:34:27 -07:00
parent c6a0033786
commit 40f5cbef6c
6 changed files with 121 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
collector/pkg/collector/base.go
+88
View File
@@ -4,14 +4,87 @@ import (
"bytes"
"encoding/json"
"errors"
"fmt"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/jaypipes/ghw"
"io"
"net/http"
"os"
"os/exec"
"path"
"time"
)
var httpClient = &http.Client{Timeout: 10 * time.Second}
type BaseCollector struct{}
func (c *BaseCollector) detectStorageDevices() ([]models.Device, error) {
block, err := ghw.Block()
if err != nil {
fmt.Printf("Error getting block storage info: %v", err)
return nil, err
}
approvedDisks := []models.Device{}
for _, disk := range block.Disks {
// ignore optical drives and floppy disks
if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
continue
}
// ignore removable disks
if disk.IsRemovable {
fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
continue
}
// ignore virtual disks & mobile phone storage devices
if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// ignore NVMe devices (not currently supported) TBA
if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME {
fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
continue
}
diskModel := models.Device{
WWN: disk.WWN,
Manufacturer: disk.Vendor,
ModelName: disk.Model,
InterfaceType: disk.StorageController.String(),
//InterfaceSpeed: string
SerialNumber: disk.SerialNumber,
Capacity: int64(disk.SizeBytes),
//Firmware string
//RotationSpeed int
DeviceName: disk.Name,
}
if len(diskModel.WWN) == 0 {
//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
//diskname as identifier if WWN is not present
diskModel.WWN = disk.Name
}
approvedDisks = append(approvedDisks, diskModel)
}
return approvedDisks, nil
}
func (c *BaseCollector) getJson(url string, target interface{}) error {
r, err := httpClient.Get(url)
@@ -23,6 +96,21 @@ func (c *BaseCollector) getJson(url string, target interface{}) error {
return json.NewDecoder(r.Body).Decode(target)
}
func (c *BaseCollector) postJson(url string, body interface{}, target interface{}) error {
requestBody, err := json.Marshal(body)
if err != nil {
return err
}
r, err := httpClient.Post(url, "application/json", bytes.NewBuffer(requestBody))
if err != nil {
return err
}
defer r.Body.Close()
return json.NewDecoder(r.Body).Decode(target)
}
func (c *BaseCollector) execCmd(cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
cmd := exec.Command(cmdName, cmdArgs...)
collector/pkg/collector/metrics.go
+7 -8
View File
@@ -6,16 +6,12 @@ import (
"github.com/analogj/scrutiny/collector/pkg/errors"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/sirupsen/logrus"
"net/http"
"net/url"
"os/exec"
"strings"
"sync"
"time"
)
var httpClient = &http.Client{Timeout: 10 * time.Second}
type MetricsCollector struct {
BaseCollector
@@ -44,12 +40,15 @@ func (mc *MetricsCollector) Run() error {
}
apiEndpoint, _ := url.Parse(mc.apiEndpoint.String())
apiEndpoint.Path = "/api/devices"
apiEndpoint.Path = "/api/devices/register"
deviceRespWrapper := new(models.DeviceRespWrapper)
deviceRespWrapper := new(models.DeviceWrapper)
detectedStorageDevices, err := mc.detectStorageDevices()
fmt.Println("Getting devices")
err = mc.getJson(apiEndpoint.String(), &deviceRespWrapper)
fmt.Println("Sending detected devices to API, for filtering & validation")
err = mc.postJson(apiEndpoint.String(), models.DeviceWrapper{
Data: detectedStorageDevices,
}, &deviceRespWrapper)
if err != nil {
return err
}
collector/pkg/models/device.go
+8 -6
View File
@@ -1,21 +1,23 @@
package models
type Device struct {
WWN string `json:"wwn" gorm:"primary_key"`
WWN string `json:"wwn"`
DeviceName string `json:"device_name"`
Manufacturer string `json:"manufacturer"`
ModelName string `json:"model_name"`
InterfaceType string `json:"interface_type"`
InterfaceSpeed string `json:"interface_speed"`
SerialNumber string `json:"serial_name"`
Capacity int64 `json:"capacity"`
SerialNumber string `json:"serial_number"`
Firmware string `json:"firmware"`
RotationSpeed int `json:"rotational_speed"`
Capacity int64 `json:"capacity"`
FormFactor string `json:"form_factor"`
SmartSupport bool `json:"smart_support"`
}
type DeviceRespWrapper struct {
Success bool `json:"success"`
Errors []error `json:"errors"`
type DeviceWrapper struct {
Success bool `json:"success,omitempty"`
Errors []error `json:"errors,omitempty"`
Data []Device `json:"data"`
}
webapp/backend/pkg/models/db/device.go
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"time"
)
type DeviceRespWrapper struct {
type DeviceWrapper struct {
Success bool `json:"success"`
Errors []error `json:"errors"`
Data []Device `json:"data"`
webapp/backend/pkg/web/disk.go
-77
View File
@@ -1,77 +0,0 @@
package web
import (
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/jaypipes/ghw"
)
func RetrieveStorageDevices() ([]db.Device, error) {
block, err := ghw.Block()
if err != nil {
fmt.Printf("Error getting block storage info: %v", err)
return nil, err
}
approvedDisks := []db.Device{}
for _, disk := range block.Disks {
//TODO: always allow if in approved list
fmt.Printf(" %v\n", disk)
// ignore optical drives and floppy disks
if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
continue
}
// ignore removable disks
if disk.IsRemovable {
fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
continue
}
// ignore virtual disks & mobile phone storage devices
if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// ignore NVMe devices (not currently supported) TBA
if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME {
fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
continue
}
//TODO: remove if in excluded list
diskModel := db.Device{
WWN: disk.WWN,
Manufacturer: disk.Vendor,
ModelName: disk.Model,
InterfaceType: disk.StorageController.String(),
//InterfaceSpeed: string
SerialNumber: disk.SerialNumber,
Capacity: int64(disk.SizeBytes),
//Firmware string
//RotationSpeed int
DeviceName: disk.Name,
}
if len(diskModel.WWN) == 0 {
//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
//diskname as identifier if WWN is not present
diskModel.WWN = disk.Name
}
approvedDisks = append(approvedDisks, diskModel)
}
return approvedDisks, nil
}
webapp/backend/pkg/web/server.go
+17 -9
View File
@@ -31,7 +31,7 @@ func (ae *AppEngine) Start() error {
})
//TODO: notifications
api.GET("/devices", GetDevicesHandler)
api.POST("/devices/register", RegisterDevices)
api.GET("/summary", GetDevicesSummary)
api.POST("/device/:wwn/smart", UploadDeviceSmartData)
api.POST("/device/:wwn/selftest", UploadDeviceSelfTestData)
@@ -55,12 +55,20 @@ func (ae *AppEngine) Start() error {
return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port"))) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080")
}
// Get all active disks for processing by collectors
func GetDevicesHandler(c *gin.Context) {
storageDevices, err := RetrieveStorageDevices()
// filter devices that are detected by various collectors.
func RegisterDevices(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
for _, dev := range storageDevices {
var collectorDeviceWrapper dbModels.DeviceWrapper
err := c.BindJSON(&collectorDeviceWrapper)
if err != nil {
log.Error("Cannot parse detected devices")
c.JSON(http.StatusOK, gin.H{"success": false})
}
//TODO: filter devices here (remove excludes, force includes)
for _, dev := range collectorDeviceWrapper.Data {
//insert devices into DB if not already there.
db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev)
}
@@ -70,9 +78,9 @@ func GetDevicesHandler(c *gin.Context) {
"success": false,
})
} else {
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": storageDevices,
c.JSON(http.StatusOK, dbModels.DeviceWrapper{
Success: true,
Data: collectorDeviceWrapper.Data,
})
}
}