gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added AttributeStatus bit flag

Browse Source 
ensure DeviceStatus is a valid bit flag.
[docs] added running tests section to contribution guide.
make sure UI correctly treats scrutiny failures as failed.
This commit is contained in:
Jason Kulatunga
2022-05-31 13:31:34 -07:00
parent b5dad487e5
commit 488fcfc820
11 changed files with 115 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CONTRIBUTING.md
+16
View File
@@ -169,3 +169,19 @@ docker run -it --rm -p 8080:8080 \
ghcr.io/analogj/scrutiny:master-omnibus
/opt/scrutiny/bin/scrutiny-collector-metrics run
```
# Running Tests
```bash
docker run -p 8086:8086 -d --rm \
-e DOCKER_INFLUXDB_INIT_MODE=setup \
-e DOCKER_INFLUXDB_INIT_USERNAME=admin \
-e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
-e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
-e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
-e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \
influxdb:2.2
go test ./...
```
webapp/backend/pkg/constants.go
+27 -18
View File
@@ -4,25 +4,34 @@ const DeviceProtocolAta = "ATA"
const DeviceProtocolScsi = "SCSI"
const DeviceProtocolNvme = "NVMe"
const SmartAttributeStatusPassed = 0
const SmartAttributeStatusFailed = 1
const SmartAttributeStatusWarning = 2
const SmartWhenFailedFailingNow = "FAILING_NOW"
const SmartWhenFailedInThePast = "IN_THE_PAST"
//const SmartStatusPassed = "passed"
//const SmartStatusFailed = "failed"
type DeviceStatus int
type AttributeStatus uint8
const (
DeviceStatusPassed DeviceStatus = 0
DeviceStatusFailedSmart DeviceStatus = iota
DeviceStatusFailedScrutiny DeviceStatus = iota
// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
AttributeStatusPassed AttributeStatus = 0
AttributeStatusFailedSmart AttributeStatus = 1
AttributeStatusWarningScrutiny AttributeStatus = 2
AttributeStatusFailedScrutiny AttributeStatus = 4
)
func Set(b, flag DeviceStatus) DeviceStatus { return b | flag }
func Clear(b, flag DeviceStatus) DeviceStatus { return b &^ flag }
func Toggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
func Has(b, flag DeviceStatus) bool { return b&flag != 0 }
const AttributeWhenFailedFailingNow = "FAILING_NOW"
const AttributeWhenFailedInThePast = "IN_THE_PAST"
func AttributeStatusSet(b, flag AttributeStatus) AttributeStatus { return b | flag }
func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus { return b &^ flag }
func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag }
func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 }
type DeviceStatus uint8
const (
// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
DeviceStatusPassed DeviceStatus = 0
DeviceStatusFailedSmart DeviceStatus = 1
DeviceStatusFailedScrutiny DeviceStatus = 2
)
func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus { return b | flag }
func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus { return b &^ flag }
func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
func DeviceStatusHas(b, flag DeviceStatus) bool { return b&flag != 0 }
webapp/backend/pkg/database/scrutiny_repository_device.go
+1 -1
View File
@@ -58,7 +58,7 @@ func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string
return device, fmt.Errorf("Could not get device from DB: %v", err)
}
device.DeviceStatus = pkg.Set(device.DeviceStatus, status)
device.DeviceStatus = pkg.DeviceStatusSet(device.DeviceStatus, status)
return device, sr.gormClient.Model(&device).Updates(device).Error
}
webapp/backend/pkg/models/device.go
+4 -4
View File
@@ -21,9 +21,9 @@ type Device struct {
WWN string `json:"wwn" gorm:"primary_key"`
DeviceName string `json:"device_name"`
DeviceUUID string `json:"device_uuid"`
DeviceSerialID string `json:"device_serial_id"`
DeviceLabel string `json:"device_label"`
DeviceUUID string `json:"device_uuid"`
DeviceSerialID string `json:"device_serial_id"`
DeviceLabel string `json:"device_label"`
Manufacturer string `json:"manufacturer"`
ModelName string `json:"model_name"`
@@ -166,7 +166,7 @@ func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error {
dv.DeviceProtocol = info.Device.Protocol
if !info.SmartStatus.Passed {
dv.DeviceStatus = pkg.Set(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
dv.DeviceStatus = pkg.DeviceStatusSet(dv.DeviceStatus, pkg.DeviceStatusFailedSmart)
}
return nil
webapp/backend/pkg/models/measurements/smart.go
+8 -7
View File
@@ -110,7 +110,7 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er
sm.PowerCycleCount = info.PowerCycleCount
sm.PowerOnHours = info.PowerOnTime.Hours
if !info.SmartStatus.Passed {
sm.Status = pkg.DeviceStatusFailedSmart
sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedSmart)
}
sm.DeviceProtocol = info.Device.Protocol
@@ -148,8 +148,9 @@ func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTa
}
attrModel.PopulateAttributeStatus()
sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel
if attrModel.Status == pkg.SmartAttributeStatusFailed {
sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) {
sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
}
}
}
@@ -178,8 +179,8 @@ func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.Nv
//find analyzed attribute status
for _, val := range sm.Attributes {
if val.GetStatus() == pkg.SmartAttributeStatusFailed {
sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
if pkg.AttributeStatusHas(val.GetStatus(), pkg.AttributeStatusFailedScrutiny) {
sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
}
}
}
@@ -204,8 +205,8 @@ func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog
//find analyzed attribute status
for _, val := range sm.Attributes {
if val.GetStatus() == pkg.SmartAttributeStatusFailed {
sm.Status = pkg.Set(sm.Status, pkg.DeviceStatusFailedScrutiny)
if pkg.AttributeStatusHas(val.GetStatus(), pkg.AttributeStatusFailedScrutiny) {
sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
}
}
}
webapp/backend/pkg/models/measurements/smart_ata_attribute.go
+19 -19
View File
@@ -18,13 +18,13 @@ type SmartAtaAttribute struct {
WhenFailed string `json:"when_failed"`
//Generated data
TransformedValue int64 `json:"transformed_value"`
Status int64 `json:"status"`
StatusReason string `json:"status_reason,omitempty"`
FailureRate float64 `json:"failure_rate,omitempty"`
TransformedValue int64 `json:"transformed_value"`
Status pkg.AttributeStatus `json:"status"`
StatusReason string `json:"status_reason,omitempty"`
FailureRate float64 `json:"failure_rate,omitempty"`
}
func (sa *SmartAtaAttribute) GetStatus() int64 {
func (sa *SmartAtaAttribute) GetStatus() pkg.AttributeStatus {
return sa.Status
}
@@ -77,7 +77,7 @@ func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
case "transformed_value":
sa.TransformedValue = val.(int64)
case "status":
sa.Status = val.(int64)
sa.Status = val.(pkg.AttributeStatus)
case "status_reason":
sa.StatusReason = val.(string)
case "failure_rate":
@@ -89,16 +89,16 @@ func (sa *SmartAtaAttribute) Inflate(key string, val interface{}) {
//populate attribute status, using SMART Thresholds & Observed Metadata
// Chainable
func (sa *SmartAtaAttribute) PopulateAttributeStatus() *SmartAtaAttribute {
if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedFailingNow {
if strings.ToUpper(sa.WhenFailed) == pkg.AttributeWhenFailedFailingNow {
//this attribute has previously failed
sa.Status = pkg.SmartAttributeStatusFailed
sa.StatusReason = "Attribute is failing manufacturer SMART threshold"
sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedSmart)
sa.StatusReason += "Attribute is failing manufacturer SMART threshold"
//if the Smart Status is failed, we should exit early, no need to look at thresholds.
return sa
} else if strings.ToUpper(sa.WhenFailed) == pkg.SmartWhenFailedInThePast {
sa.Status = pkg.SmartAttributeStatusWarning
sa.StatusReason = "Attribute has previously failed manufacturer SMART threshold"
} else if strings.ToUpper(sa.WhenFailed) == pkg.AttributeWhenFailedInThePast {
sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
sa.StatusReason += "Attribute has previously failed manufacturer SMART threshold"
}
if smartMetadata, ok := thresholds.AtaMetadata[sa.AttributeId]; ok {
@@ -138,16 +138,16 @@ func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttri
if smartMetadata.Critical {
if obsThresh.AnnualFailureRate >= 0.10 {
sa.Status = pkg.SmartAttributeStatusFailed
sa.StatusReason = "Observed Failure Rate for Critical Attribute is greater than 10%"
sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
sa.StatusReason += "Observed Failure Rate for Critical Attribute is greater than 10%"
}
} else {
if obsThresh.AnnualFailureRate >= 0.20 {
sa.Status = pkg.SmartAttributeStatusFailed
sa.StatusReason = "Observed Failure Rate for Attribute is greater than 20%"
sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
sa.StatusReason += "Observed Failure Rate for Non-Critical Attribute is greater than 20%"
} else if obsThresh.AnnualFailureRate >= 0.10 {
sa.Status = pkg.SmartAttributeStatusWarning
sa.StatusReason = "Observed Failure Rate for Attribute is greater than 10%"
sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
sa.StatusReason += "Observed Failure Rate for Non-Critical Attribute is greater than 10%"
}
}
@@ -157,7 +157,7 @@ func (sa *SmartAtaAttribute) ValidateThreshold(smartMetadata thresholds.AtaAttri
}
// no bucket found
if smartMetadata.Critical {
sa.Status = pkg.SmartAttributeStatusWarning
sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusWarningScrutiny)
sa.StatusReason = "Could not determine Observed Failure Rate for Critical Attribute"
}
webapp/backend/pkg/models/measurements/smart_attribute.go
+3 -1
View File
@@ -1,7 +1,9 @@
package measurements
import "github.com/analogj/scrutiny/webapp/backend/pkg"
type SmartAttribute interface {
Flatten() (fields map[string]interface{})
Inflate(key string, val interface{})
GetStatus() int64
GetStatus() pkg.AttributeStatus
}
webapp/backend/pkg/models/measurements/smart_nvme_attribute.go
+8 -8
View File
@@ -12,13 +12,13 @@ type SmartNvmeAttribute struct {
Value int64 `json:"value"`
Threshold int64 `json:"thresh"`
TransformedValue int64 `json:"transformed_value"`
Status int64 `json:"status"`
StatusReason string `json:"status_reason,omitempty"`
FailureRate float64 `json:"failure_rate,omitempty"`
TransformedValue int64 `json:"transformed_value"`
Status pkg.AttributeStatus `json:"status"`
StatusReason string `json:"status_reason,omitempty"`
FailureRate float64 `json:"failure_rate,omitempty"`
}
func (sa *SmartNvmeAttribute) GetStatus() int64 {
func (sa *SmartNvmeAttribute) GetStatus() pkg.AttributeStatus {
return sa.Status
}
@@ -54,7 +54,7 @@ func (sa *SmartNvmeAttribute) Inflate(key string, val interface{}) {
case "transformed_value":
sa.TransformedValue = val.(int64)
case "status":
sa.Status = val.(int64)
sa.Status = val.(pkg.AttributeStatus)
case "status_reason":
sa.StatusReason = val.(string)
case "failure_rate":
@@ -72,8 +72,8 @@ func (sa *SmartNvmeAttribute) PopulateAttributeStatus() *SmartNvmeAttribute {
//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
sa.Status = pkg.SmartAttributeStatusFailed
sa.StatusReason = "Attribute is failing recommended SMART threshold"
sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
sa.StatusReason += "Attribute is failing recommended SMART threshold"
}
}
}
webapp/backend/pkg/models/measurements/smart_scsci_attribute.go
+7 -7
View File
@@ -12,13 +12,13 @@ type SmartScsiAttribute struct {
Value int64 `json:"value"`
Threshold int64 `json:"thresh"`
TransformedValue int64 `json:"transformed_value"`
Status int64 `json:"status"`
StatusReason string `json:"status_reason,omitempty"`
FailureRate float64 `json:"failure_rate,omitempty"`
TransformedValue int64 `json:"transformed_value"`
Status pkg.AttributeStatus `json:"status"`
StatusReason string `json:"status_reason,omitempty"`
FailureRate float64 `json:"failure_rate,omitempty"`
}
func (sa *SmartScsiAttribute) GetStatus() int64 {
func (sa *SmartScsiAttribute) GetStatus() pkg.AttributeStatus {
return sa.Status
}
@@ -54,7 +54,7 @@ func (sa *SmartScsiAttribute) Inflate(key string, val interface{}) {
case "transformed_value":
sa.TransformedValue = val.(int64)
case "status":
sa.Status = val.(int64)
sa.Status = val.(pkg.AttributeStatus)
case "status_reason":
sa.StatusReason = val.(string)
case "failure_rate":
@@ -73,7 +73,7 @@ func (sa *SmartScsiAttribute) PopulateAttributeStatus() *SmartScsiAttribute {
//check what the ideal is. Ideal tells us if we our recorded value needs to be above, or below the threshold
if (smartMetadata.Ideal == "low" && sa.Value > sa.Threshold) ||
(smartMetadata.Ideal == "high" && sa.Value < sa.Threshold) {
sa.Status = pkg.SmartAttributeStatusFailed
sa.Status = pkg.AttributeStatusSet(sa.Status, pkg.AttributeStatusFailedScrutiny)
sa.StatusReason = "Attribute is failing recommended SMART threshold"
}
}
webapp/backend/pkg/models/measurements/smart_test.go
+13 -10
View File
@@ -77,7 +77,7 @@ func TestSmart_Flatten_ATA(t *testing.T) {
"attr.1.failure_rate": float64(0),
"attr.1.raw_string": "0",
"attr.1.raw_value": int64(0),
"attr.1.status": int64(0),
"attr.1.status": pkg.AttributeStatus(0),
"attr.1.status_reason": "",
"attr.1.thresh": int64(1),
"attr.1.transformed_value": int64(0),
@@ -89,7 +89,7 @@ func TestSmart_Flatten_ATA(t *testing.T) {
"attr.2.failure_rate": float64(0),
"attr.2.raw_string": "108",
"attr.2.raw_value": int64(108),
"attr.2.status": int64(0),
"attr.2.status": pkg.AttributeStatus(0),
"attr.2.status_reason": "",
"attr.2.thresh": int64(54),
"attr.2.transformed_value": int64(0),
@@ -130,7 +130,7 @@ func TestSmart_Flatten_SCSI(t *testing.T) {
require.Equal(t, map[string]interface{}{
"attr.read_errors_corrected_by_eccfast.attribute_id": "read_errors_corrected_by_eccfast",
"attr.read_errors_corrected_by_eccfast.failure_rate": float64(0),
"attr.read_errors_corrected_by_eccfast.status": int64(0),
"attr.read_errors_corrected_by_eccfast.status": pkg.AttributeStatus(0),
"attr.read_errors_corrected_by_eccfast.status_reason": "",
"attr.read_errors_corrected_by_eccfast.thresh": int64(0),
"attr.read_errors_corrected_by_eccfast.transformed_value": int64(0),
@@ -168,7 +168,7 @@ func TestSmart_Flatten_NVMe(t *testing.T) {
require.Equal(t, map[string]interface{}{
"attr.available_spare.attribute_id": "available_spare",
"attr.available_spare.failure_rate": float64(0),
"attr.available_spare.status": int64(0),
"attr.available_spare.status": pkg.AttributeStatus(0),
"attr.available_spare.status_reason": "",
"attr.available_spare.thresh": int64(0),
"attr.available_spare.transformed_value": int64(0),
@@ -189,7 +189,7 @@ func TestNewSmartFromInfluxDB_ATA(t *testing.T) {
"attr.1.failure_rate": float64(0),
"attr.1.raw_string": "108",
"attr.1.raw_value": int64(108),
"attr.1.status": int64(0),
"attr.1.status": pkg.AttributeStatus(0),
"attr.1.status_reason": "",
"attr.1.thresh": int64(54),
"attr.1.transformed_value": int64(0),
@@ -235,7 +235,7 @@ func TestNewSmartFromInfluxDB_NVMe(t *testing.T) {
"device_protocol": pkg.DeviceProtocolNvme,
"attr.available_spare.attribute_id": "available_spare",
"attr.available_spare.failure_rate": float64(0),
"attr.available_spare.status": int64(0),
"attr.available_spare.status": pkg.AttributeStatus(0),
"attr.available_spare.status_reason": "",
"attr.available_spare.thresh": int64(0),
"attr.available_spare.transformed_value": int64(0),
@@ -274,7 +274,7 @@ func TestNewSmartFromInfluxDB_SCSI(t *testing.T) {
"device_protocol": pkg.DeviceProtocolScsi,
"attr.read_errors_corrected_by_eccfast.attribute_id": "read_errors_corrected_by_eccfast",
"attr.read_errors_corrected_by_eccfast.failure_rate": float64(0),
"attr.read_errors_corrected_by_eccfast.status": int64(0),
"attr.read_errors_corrected_by_eccfast.status": pkg.AttributeStatus(0),
"attr.read_errors_corrected_by_eccfast.status_reason": "",
"attr.read_errors_corrected_by_eccfast.thresh": int64(0),
"attr.read_errors_corrected_by_eccfast.transformed_value": int64(0),
@@ -328,9 +328,12 @@ func TestFromCollectorSmartInfo(t *testing.T) {
require.Equal(t, 18, len(smartMdl.Attributes))
//check that temperature was correctly parsed
require.Equal(t, int64(163210330144), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).RawValue)
require.Equal(t, int64(32), smartMdl.Attributes["194"].(*measurements.SmartAtaAttribute).TransformedValue)
//ensure that Scrutiny warning for a non critical attribute does not set device status to failed.
require.Equal(t, pkg.AttributeStatusWarningScrutiny, smartMdl.Attributes["3"].GetStatus())
}
func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) {
@@ -402,7 +405,7 @@ func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
require.Equal(t, pkg.DeviceStatusFailedScrutiny, smartMdl.Status)
require.Equal(t, int64(pkg.SmartAttributeStatusFailed), smartMdl.Attributes["199"].GetStatus(),
require.Equal(t, pkg.AttributeStatusFailedScrutiny, smartMdl.Attributes["199"].GetStatus(),
"scrutiny should detect that %d failed (status: %d, %s)",
smartMdl.Attributes["199"].(*measurements.SmartAtaAttribute).AttributeId,
smartMdl.Attributes["199"].GetStatus(), smartMdl.Attributes["199"].(*measurements.SmartAtaAttribute).StatusReason,
@@ -435,7 +438,7 @@ func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "WWN-test", smartMdl.DeviceWWN)
require.Equal(t, pkg.DeviceStatusFailedScrutiny, smartMdl.Status)
require.Equal(t, int64(pkg.SmartAttributeStatusFailed), smartMdl.Attributes["media_errors"].GetStatus(),
require.Equal(t, pkg.AttributeStatusFailedScrutiny, smartMdl.Attributes["media_errors"].GetStatus(),
"scrutiny should detect that %s failed (status: %d, %s)",
smartMdl.Attributes["media_errors"].(*measurements.SmartNvmeAttribute).AttributeId,
smartMdl.Attributes["media_errors"].GetStatus(),
webapp/frontend/src/app/modules/detail/detail.component.ts
+9 -3
View File
@@ -122,11 +122,17 @@ export class DetailComponent implements OnInit, AfterViewInit, OnDestroy {
// @ Private methods
// -----------------------------------------------------------------------------------------------------
getAttributeStatusName(attribute_status){
if(attribute_status == 0){
//from Constants.go
// AttributeStatusPassed AttributeStatus = 0
// AttributeStatusFailedSmart AttributeStatus = 1
// AttributeStatusWarningScrutiny AttributeStatus = 2
// AttributeStatusFailedScrutiny AttributeStatus = 4
if(attribute_status === 0){
return "passed"
} else if (attribute_status == 1){
} else if (attribute_status & 1 != 0 || attribute_status & 4 != 0 ){
return "failed"
} else if (attribute_status == 2){
} else if (attribute_status & 2 != 0){
return "warn"
}
return