gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moved version file into webapp source tree. Replacing fmt with logger calls. moved Jobber config file into /scrutiny/jobber directory. Added scsi json file. Moved API rooute handlers into their own files in a module. Added not yet implemnented tooltips.

Browse Source
This commit is contained in:
Jason Kulatunga
2020-08-21 18:06:00 -07:00
parent c14f313fa1
commit d636709aa1
23 changed files with 267 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/backend/pkg/models/testdata/scsi-smart.json
Vendored
+64
View File
@@ -0,0 +1,64 @@
{
"device": {
"name": "/dev/sdg",
"info_name": "/dev/sdg",
"type": "scsi",
"protocol": "SCSI"
},
"vendor": "SEAGATE",
"product": "ST4000NM0043",
"model_name": "SEAGATE ST4000NM0043",
"revision": "MS03",
"scsi_version": "SPC-4",
"user_capacity": {
"blocks": 7814037168,
"bytes": 4000787030016
},
"logical_block_size": 512,
"rotation_rate": 7200,
"form_factor": {
"scsi_value": 2,
"name": "3.5 inches"
},
"serial_number": "Z1Z5DWJK0000XXXXXXXX",
"device_type": {
"scsi_value": 0,
"name": "disk"
},
"local_time": {
"time_t": 1598048822,
"asctime": "Fri Aug 21 22:27:02 2020 UTC"
},
"smart_status": {
"passed": true
},
"temperature": {
"current": 34,
"drive_trip": 68
},
"scsi_grown_defect_list": 56,
"power_on_time": {
"hours": 43549,
"minutes": 33
},
"scsi_error_counter_log": {
"read": {
"errors_corrected_by_eccfast": 300357663,
"errors_corrected_by_eccdelayed": 0,
"errors_corrected_by_rereads_rewrites": 0,
"total_errors_corrected": 300357663,
"correction_algorithm_invocations": 0,
"gigabytes_processed": "176987.332",
"total_uncorrected_errors": 0
},
"write": {
"errors_corrected_by_eccfast": 0,
"errors_corrected_by_eccdelayed": 0,
"errors_corrected_by_rereads_rewrites": 0,
"total_errors_corrected": 0,
"correction_algorithm_invocations": 0,
"gigabytes_processed": "86472.611",
"total_uncorrected_errors": 0
}
}
}
webapp/backend/pkg/version/version.go
+1 -1
View File
@@ -2,4 +2,4 @@ package version
// VERSION is the app-global version string, which will be replaced with a
// new value during packaging
const VERSION = "0.1.0"
const VERSION = "0.1.2"
webapp/backend/pkg/web/cron.go
-8
View File
@@ -1,8 +0,0 @@
package web
// the following cronjobs need to be defined here:
// - get storage information for all approved disks
// - run short test against approved disks
// - run long test against approved disks
// - get S.M.A.R.T. metrics from approved disks
// - clean up / resolution for time-series data in sqlite database.
webapp/backend/pkg/web/handler/get_device_details.go
+27
View File
@@ -0,0 +1,27 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"net/http"
)
func GetDeviceDetails(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
device := dbModels.Device{}
db.Debug().
Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
return db.Order("smarts.created_at DESC").Limit(40)
}).
Preload("SmartResults.SmartAttributes").
Where("wwn = ?", c.Param("wwn")).
First(&device)
device.SquashHistory()
device.ApplyMetadataRules()
c.JSON(http.StatusOK, gin.H{"success": true, "data": device, "lookup": metadata.AtaSmartAttributes})
}
webapp/backend/pkg/web/handler/get_devices_summary.go
+26
View File
@@ -0,0 +1,26 @@
package handler
import (
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"net/http"
)
func GetDevicesSummary(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
devices := []dbModels.Device{}
//We need the last x (for now all) Smart objects for each Device, so that we can graph Temperature
//We also need the last
db.Debug().
Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
}).
Find(&devices)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": devices,
})
}
webapp/backend/pkg/web/handler/register_devices.go
+39
View File
@@ -0,0 +1,39 @@
package handler
import (
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
log "github.com/sirupsen/logrus"
"net/http"
)
// filter devices that are detected by various collectors.
func RegisterDevices(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
var collectorDeviceWrapper dbModels.DeviceWrapper
err := c.BindJSON(&collectorDeviceWrapper)
if err != nil {
log.Error("Cannot parse detected devices")
c.JSON(http.StatusOK, gin.H{"success": false})
}
//TODO: filter devices here (remove excludes, force includes)
for _, dev := range collectorDeviceWrapper.Data {
//insert devices into DB if not already there.
db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev)
}
if err != nil {
c.JSON(http.StatusOK, gin.H{
"success": false,
})
} else {
c.JSON(http.StatusOK, dbModels.DeviceWrapper{
Success: true,
Data: collectorDeviceWrapper.Data,
})
}
}
webapp/backend/pkg/web/handler/upload_device_metrics.go
+40
View File
@@ -0,0 +1,40 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
log "github.com/sirupsen/logrus"
"net/http"
)
func UploadDeviceMetrics(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
var collectorSmartData collector.SmartInfo
err := c.BindJSON(&collectorSmartData)
if err != nil {
//TODO: cannot parse smart data
log.Error("Cannot parse SMART data")
c.JSON(http.StatusOK, gin.H{"success": false})
}
//update the device information if necessary
var device dbModels.Device
db.Where("wwn = ?", c.Param("wwn")).First(&device)
device.UpdateFromCollectorSmartInfo(collectorSmartData)
db.Model(&device).Updates(device)
// insert smart info
deviceSmartData := dbModels.Smart{}
err = deviceSmartData.FromCollectorSmartInfo(c.Param("wwn"), collectorSmartData)
if err != nil {
c.JSON(http.StatusOK, gin.H{"success": false})
return
}
db.Create(&deviceSmartData)
c.JSON(http.StatusOK, gin.H{"success": true})
}
webapp/backend/pkg/web/handler/upload_device_self_tests.go
+7
View File
@@ -0,0 +1,7 @@
package handler
import "github.com/gin-gonic/gin"
func UploadDeviceSelfTests(c *gin.Context) {
}
webapp/backend/pkg/web/server.go
+7 -131
View File
@@ -4,12 +4,8 @@ import (
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/web/handler"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
log "github.com/sirupsen/logrus"
"net/http"
)
@@ -30,13 +26,12 @@ func (ae *AppEngine) Start() error {
})
})
//TODO: notifications
api.POST("/devices/register", RegisterDevices)
api.GET("/summary", GetDevicesSummary)
api.POST("/device/:wwn/smart", UploadDeviceSmartData)
api.POST("/device/:wwn/selftest", UploadDeviceSelfTestData)
api.POST("/devices/register", handler.RegisterDevices)
api.GET("/summary", handler.GetDevicesSummary)
api.POST("/device/:wwn/smart", handler.UploadDeviceMetrics)
api.POST("/device/:wwn/selftest", handler.UploadDeviceSelfTests)
api.GET("/device/:wwn/details", GetDeviceDetails)
api.GET("/device/:wwn/details", handler.GetDeviceDetails)
}
//Static request routing
@@ -52,124 +47,5 @@ func (ae *AppEngine) Start() error {
c.File(fmt.Sprintf("%s/index.html", ae.Config.GetString("web.src.frontend.path")))
})
return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port"))) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080")
}
// filter devices that are detected by various collectors.
func RegisterDevices(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
var collectorDeviceWrapper dbModels.DeviceWrapper
err := c.BindJSON(&collectorDeviceWrapper)
if err != nil {
log.Error("Cannot parse detected devices")
c.JSON(http.StatusOK, gin.H{"success": false})
}
//TODO: filter devices here (remove excludes, force includes)
for _, dev := range collectorDeviceWrapper.Data {
//insert devices into DB if not already there.
db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev)
}
if err != nil {
c.JSON(http.StatusOK, gin.H{
"success": false,
})
} else {
c.JSON(http.StatusOK, dbModels.DeviceWrapper{
Success: true,
Data: collectorDeviceWrapper.Data,
})
}
}
func UploadDeviceSmartData(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
var collectorSmartData collector.SmartInfo
err := c.BindJSON(&collectorSmartData)
if err != nil {
//TODO: cannot parse smart data
log.Error("Cannot parse SMART data")
c.JSON(http.StatusOK, gin.H{"success": false})
}
//update the device information if necessary
var device dbModels.Device
db.Where("wwn = ?", c.Param("wwn")).First(&device)
device.UpdateFromCollectorSmartInfo(collectorSmartData)
db.Model(&device).Updates(device)
// insert smart info
deviceSmartData := dbModels.Smart{}
err = deviceSmartData.FromCollectorSmartInfo(c.Param("wwn"), collectorSmartData)
if err != nil {
c.JSON(http.StatusOK, gin.H{"success": false})
return
}
db.Create(&deviceSmartData)
c.JSON(http.StatusOK, gin.H{"success": true})
}
func UploadDeviceSelfTestData(c *gin.Context) {
}
func GetDeviceDetails(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
device := dbModels.Device{}
db.Debug().
Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
return db.Order("smarts.created_at DESC").Limit(40)
}).
Preload("SmartResults.SmartAttributes").
Where("wwn = ?", c.Param("wwn")).
First(&device)
device.SquashHistory()
device.ApplyMetadataRules()
c.JSON(http.StatusOK, gin.H{"success": true, "data": device, "lookup": metadata.AtaSmartAttributes})
}
func GetDevicesSummary(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
devices := []dbModels.Device{}
//OLD: cant seem to figure out how to get the latest SmartResults for each Device, so instead
// we're going to assume that results were retrieved at the same time, so we'll just get the last x number of results
//var devicesCount int
//db.Table("devices").Count(&devicesCount)
//We need the last x (for now all) Smart objects for each Device, so that we can graph Temperature
//We also need the last
db.Debug().
Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
}).
//Preload("SmartResults").
// Preload("SmartResults.SmartAttributes").
Find(&devices)
//for _, dev := range devices {
// log.Printf("===== device: %s\n", dev.WWN)
// log.Print(len(dev.SmartResults))
//}
//a, _ := json.Marshal(devices) //get json byte array
//n := len(a) //Find the length of the byte array
//s := string(a[:n]) //convert to string
//log.Print(s) //write to response
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": devices,
})
//c.Data(http.StatusOK, "application/json", a)
return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port")))
}
webapp/frontend/src/app/modules/admin/dashboard/dashboard.component.html
+8 -3
View File
@@ -11,12 +11,14 @@
<!-- Action buttons -->
<div class="flex items-center">
<button class="xs:hidden"
matTooltip="not yet implemented"
mat-stroked-button>
<mat-icon class="icon-size-20"
[svgIcon]="'save'"></mat-icon>
<span class="ml-2">Export</span>
</button>
<button class="ml-2 xs:hidden"
matTooltip="not yet implemented"
mat-stroked-button>
<mat-icon class="icon-size-20 rotate-90 mirror"
[svgIcon]="'tune'"></mat-icon>
@@ -30,12 +32,14 @@
<mat-icon [svgIcon]="'more_vert'"></mat-icon>
</button>
<mat-menu #actionsMenu="matMenu">
<button mat-menu-item>
<button mat-menu-item
matTooltip="not yet implemented">
<mat-icon class="icon-size-20"
[svgIcon]="'save'"></mat-icon>
<span class="ml-2">Export</span>
</button>
<button mat-menu-item>
<button mat-menu-item
matTooltip="not yet implemented">
<mat-icon class="icon-size-20 rotate-90 mirror"
[svgIcon]="'tune'"></mat-icon>
<span class="ml-2">Settings</span>
@@ -47,7 +51,7 @@
<div class="flex flex-wrap w-full">
<div *ngFor="let disk of data.data " class="flex flex-auto w-1/2 min-w-80 p-4">
<div *ngFor="let disk of data.data " class="flex w-1/2 min-w-80 p-4">
<div [ngClass]="{'border-green': disk.smart_results[0]?.smart_status == 'passed', 'border-red': disk.smart_results[0]?.smart_status == 'failed'}"
class="relative flex flex-col flex-auto p-6 pr-3 pb-3 bg-card rounded border-l-4 shadow-md overflow-hidden">
<div class="absolute bottom-0 right-0 w-24 h-24 -m-6">
@@ -115,6 +119,7 @@
</div>
<div>
<button class="h-8 min-h-8 px-2"
matTooltip="not yet implemented"
mat-button
[matMenuTriggerFor]="accountBalanceMenu">
<span class="font-medium text-sm text-hint">12 months</span>
webapp/frontend/src/app/modules/admin/dashboard/dashboard.module.ts
+2
View File
@@ -11,6 +11,7 @@ import { MatProgressBarModule } from '@angular/material/progress-bar';
import { MatSortModule } from '@angular/material/sort';
import { MatTableModule } from '@angular/material/table';
import { NgApexchartsModule } from 'ng-apexcharts';
import { MatTooltipModule } from '@angular/material/tooltip'
@NgModule({
declarations: [
@@ -20,6 +21,7 @@ import { NgApexchartsModule } from 'ng-apexcharts';
RouterModule.forChild(dashboardRoutes),
MatButtonModule,
MatDividerModule,
MatTooltipModule,
MatIconModule,
MatMenuModule,
MatProgressBarModule,
webapp/frontend/src/app/modules/admin/detail/detail.component.html
+7 -3
View File
@@ -10,12 +10,14 @@
<!-- Action buttons -->
<div class="flex items-center">
<button class="xs:hidden"
matTooltip="not yet implemented"
mat-stroked-button>
<mat-icon class="icon-size-20"
[svgIcon]="'save'"></mat-icon>
<span class="ml-2">Export</span>
</button>
<button class="ml-2 xs:hidden"
matTooltip="not yet implemented"
mat-stroked-button>
<mat-icon class="icon-size-20 rotate-90 mirror"
[svgIcon]="'tune'"></mat-icon>
@@ -29,12 +31,14 @@
<mat-icon [svgIcon]="'more_vert'"></mat-icon>
</button>
<mat-menu #actionsMenu="matMenu">
<button mat-menu-item>
<button mat-menu-item
matTooltip="not yet implemented">
<mat-icon class="icon-size-20"
[svgIcon]="'save'"></mat-icon>
<span class="ml-2">Export</span>
</button>
<button mat-menu-item>
<button mat-menu-item
matTooltip="not yet implemented">
<mat-icon class="icon-size-20 rotate-90 mirror"
[svgIcon]="'tune'"></mat-icon>
<span class="ml-2">Settings</span>
@@ -249,7 +253,7 @@
mat-header-cell
mat-sort-header
*matHeaderCellDef>
<span class="whitespace-no-wrap">
<span class="whitespace-no-wrap" matTooltip="Failure rate is based on data provided by BackBlaze. The current attribute value is matched against the observed failure categories and an annual failure rate is determined.">
Failure Rate <mat-icon [svgIcon]="'info'"></mat-icon>
</span>
</th>