partially working. Some datapoints are failing with panic and are silently ignored.

TODO must fix.

webapp/backend/pkg/database/migrations/m20201107210306/device.go

@@ -29,3 +29,19 @@ type Device struct {
 	DeviceType     string  `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
 	SmartResults   []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
 }
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+func (dv *Device) IsAta() bool {
+	return dv.DeviceProtocol == DeviceProtocolAta
+}
+
+func (dv *Device) IsScsi() bool {
+	return dv.DeviceProtocol == DeviceProtocolScsi
+}
+
+func (dv *Device) IsNvme() bool {
+	return dv.DeviceProtocol == DeviceProtocolNvme
+}

webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go

@@ -6,8 +6,10 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
 	log "github.com/sirupsen/logrus"
 	"strings"
+	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -22,13 +24,9 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin
 	}
 
 	tags, fields := deviceSmartData.Flatten()
-	p := influxdb2.NewPoint("smart",
-		tags,
-		fields,
-		deviceSmartData.Date)
 
 	// write point immediately
-	return deviceSmartData, sr.influxWriteApi.WritePoint(ctx, p)
+	return deviceSmartData, sr.saveDatapoint(sr.influxWriteApi, "smart", tags, fields, deviceSmartData.Date, ctx)
 }
 
 func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) {
@@ -93,6 +91,17 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn
 // Helper Methods
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
+func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error {
+	sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields))
+	p := influxdb2.NewPoint(measurement,
+		tags,
+		fields,
+		date)
+
+	// write point immediately
+	return influxWriteApi.WritePoint(ctx, p)
+}
+
 func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string {
 
 	/*

webapp/backend/pkg/database/scrutiny_repository_migrations.go

@@ -2,11 +2,16 @@ package database
 
 import (
 	"context"
+	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	"github.com/go-gormigrate/gormigrate/v2"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"gorm.io/gorm"
+	"strconv"
+	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -66,14 +71,170 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 				//get a list of all devices:
 				//	get a list of all smart scans in the last 2 weeks:
 				//		get a list of associated smart attribute data:
-				//			translate to a collector.SmartInfo object
+				//			translate to a measurements.Smart{} object
-				//			call scrutinyRepository.SaveSmartAttributes
+				//			call CUSTOM INFLUXDB SAVE FUNCTION (taking bucket as parameter)
 				//	get a list of all smart scans in the last 9 weeks:
 				//		do same as above (select 1 scan per week)
 				//	get a list of all smart scans in the last 25 months:
 				//		do same as above (select 1 scan per month)
 				//	get a list of all smart scans:
 				//		do same as above (select 1 scan per year)
+
+				preDevices := []m20201107210306.Device{} //pre-migration device information
+				if err = tx.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
+					return db.Order("smarts.created_at ASC") //OLD: .Limit(devicesCount)
+				}).Find(&preDevices).Error; err != nil {
+					sr.logger.Errorln("Could not get device summary from DB", err)
+					return err
+				}
+
+				//weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary.
+				weeklyLookup := map[string]bool{}
+				monthlyLookup := map[string]bool{}
+				yearlyLookup := map[string]bool{}
+
+				//calculate bucket oldest dates
+				today := time.Now()
+				dailyBucketMax := today.Add(-RETENTION_PERIOD_15_DAYS_IN_SECONDS * time.Second)     //15 days
+				weeklyBucketMax := today.Add(-RETENTION_PERIOD_9_WEEKS_IN_SECONDS * time.Second)    //9 weeks
+				monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks
+
+				for _, preDevice := range preDevices {
+					for _, preSmartResult := range preDevice.SmartResults { //pre-migration smart results
+
+						//we're looping in ASC mode, so from oldest entry to most current.
+
+						//TODO: skip any results that are outside of the range that we care about for each bucket.
+
+						err, postSmartResults := m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(tx, preDevice, preSmartResult)
+						if err != nil {
+							return err
+						}
+						smartTags, smartFields := postSmartResults.Flatten()
+
+						err, postSmartTemp := m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice, preSmartResult)
+						if err != nil {
+							return err
+						}
+						tempTags, tempFields := postSmartTemp.Flatten()
+						tempTags["device_wwn"] = preDevice.WWN
+
+						year, week := postSmartResults.Date.ISOWeek()
+						month := postSmartResults.Date.Month()
+
+						yearStr := strconv.Itoa(year)
+						yearMonthStr := fmt.Sprintf("%d-%d", year, month)
+						yearWeekStr := fmt.Sprintf("%d-%d", year, week)
+
+						//write data to daily bucket if in the last 15 days
+						if postSmartResults.Date.After(dailyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: daily", preDevice.WWN)
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						//write data to the weekly bucket if in the last 9 weeks, and week has not been processed yet
+						if _, weekExists := weeklyLookup[yearWeekStr]; !weekExists && postSmartResults.Date.After(weeklyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: weekly", preDevice.WWN)
+
+							//this week/year pair has not been processed
+							weeklyLookup[yearWeekStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						//write data to the monthly bucket if in the last 9 weeks, and week has not been processed yet
+						if _, monthExists := monthlyLookup[yearMonthStr]; !monthExists && postSmartResults.Date.After(monthlyBucketMax) {
+							sr.logger.Debugf("device (%s) smart data added to bucket: monthly", preDevice.WWN)
+							//this month/year pair has not been processed
+							monthlyLookup[yearMonthStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+
+						if _, yearExists := yearlyLookup[yearStr]; !yearExists && year != today.Year() {
+							sr.logger.Debugf("device (%s) smart data added to bucket: yearly", preDevice.WWN)
+							//this year has not been processed
+							yearlyLookup[yearStr] = true
+							// write point immediately
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"smart",
+								smartTags,
+								smartFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+
+							err = sr.saveDatapoint(
+								sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))),
+								"temp",
+								tempTags,
+								tempFields,
+								postSmartResults.Date, ctx)
+							if err != nil {
+								return err
+							}
+						}
+					}
+				}
+
 				return nil
 			},
 		},
@@ -104,3 +265,183 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 	sr.logger.Infoln("Database migration completed successfully")
 	return nil
 }
+
+// Deprecated
+func m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.SmartTemperature) {
+	//extract temperature data for every datapoint
+	postSmartTemp := measurements.SmartTemperature{
+		Date: preSmartResult.TestDate,
+		Temp: preSmartResult.Temp,
+	}
+
+	return nil, postSmartTemp
+}
+
+// Deprecated
+func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(database *gorm.DB, preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.Smart) {
+	//create a measurements.Smart object (which we will then push to the InfluxDB)
+	postDeviceSmartData := measurements.Smart{
+		Date:            preSmartResult.TestDate,
+		DeviceWWN:       preDevice.WWN,
+		DeviceProtocol:  preDevice.DeviceProtocol,
+		Temp:            preSmartResult.Temp,
+		PowerOnHours:    preSmartResult.PowerOnHours,
+		PowerCycleCount: preSmartResult.PowerCycleCount,
+
+		// this needs to be populated using measurements.Smart.ProcessAtaSmartInfo, ProcessScsiSmartInfo or ProcessNvmeSmartInfo
+		// because those functions will take into account thresholds (which we didn't consider correctly previously)
+		Attributes: map[string]measurements.SmartAttribute{},
+	}
+
+	result := database.Preload("AtaAttributes").Preload("NvmeAttributes").Preload("ScsiAttributes").Find(&preSmartResult)
+	if result.Error != nil {
+		return result.Error, postDeviceSmartData
+	}
+
+	if preDevice.IsAta() {
+		preAtaSmartAttributesTable := []collector.AtaSmartAttributesTableItem{}
+		for _, preAtaAttribute := range preSmartResult.AtaAttributes {
+			preAtaSmartAttributesTable = append(preAtaSmartAttributesTable, collector.AtaSmartAttributesTableItem{
+				ID:         preAtaAttribute.AttributeId,
+				Name:       preAtaAttribute.Name,
+				Value:      int64(preAtaAttribute.Value),
+				Worst:      int64(preAtaAttribute.Worst),
+				Thresh:     int64(preAtaAttribute.Threshold),
+				WhenFailed: preAtaAttribute.WhenFailed,
+				Flags: struct {
+					Value         int    `json:"value"`
+					String        string `json:"string"`
+					Prefailure    bool   `json:"prefailure"`
+					UpdatedOnline bool   `json:"updated_online"`
+					Performance   bool   `json:"performance"`
+					ErrorRate     bool   `json:"error_rate"`
+					EventCount    bool   `json:"event_count"`
+					AutoKeep      bool   `json:"auto_keep"`
+				}{
+					Value:         0,
+					String:        "",
+					Prefailure:    false,
+					UpdatedOnline: false,
+					Performance:   false,
+					ErrorRate:     false,
+					EventCount:    false,
+					AutoKeep:      false,
+				},
+				Raw: struct {
+					Value  int64  `json:"value"`
+					String string `json:"string"`
+				}{
+					Value:  preAtaAttribute.RawValue,
+					String: preAtaAttribute.RawString,
+				},
+			})
+		}
+
+		postDeviceSmartData.ProcessAtaSmartInfo(preAtaSmartAttributesTable)
+
+	} else if preDevice.IsNvme() {
+		//info collector.SmartInfo
+		postNvmeSmartHealthInformation := collector.NvmeSmartHealthInformationLog{}
+
+		for _, preNvmeAttribute := range preSmartResult.NvmeAttributes {
+			switch preNvmeAttribute.AttributeId {
+			case "critical_warning":
+				postNvmeSmartHealthInformation.CriticalWarning = int64(preNvmeAttribute.Value)
+			case "temperature":
+				postNvmeSmartHealthInformation.Temperature = int64(preNvmeAttribute.Value)
+			case "available_spare":
+				postNvmeSmartHealthInformation.AvailableSpare = int64(preNvmeAttribute.Value)
+			case "available_spare_threshold":
+				postNvmeSmartHealthInformation.AvailableSpareThreshold = int64(preNvmeAttribute.Value)
+			case "percentage_used":
+				postNvmeSmartHealthInformation.PercentageUsed = int64(preNvmeAttribute.Value)
+			case "data_units_read":
+				postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value)
+			case "data_units_written":
+				postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value)
+			case "host_reads":
+				postNvmeSmartHealthInformation.HostReads = int64(preNvmeAttribute.Value)
+			case "host_writes":
+				postNvmeSmartHealthInformation.HostWrites = int64(preNvmeAttribute.Value)
+			case "controller_busy_time":
+				postNvmeSmartHealthInformation.ControllerBusyTime = int64(preNvmeAttribute.Value)
+			case "power_cycles":
+				postNvmeSmartHealthInformation.PowerCycles = int64(preNvmeAttribute.Value)
+			case "power_on_hours":
+				postNvmeSmartHealthInformation.PowerOnHours = int64(preNvmeAttribute.Value)
+			case "unsafe_shutdowns":
+				postNvmeSmartHealthInformation.UnsafeShutdowns = int64(preNvmeAttribute.Value)
+			case "media_errors":
+				postNvmeSmartHealthInformation.MediaErrors = int64(preNvmeAttribute.Value)
+			case "num_err_log_entries":
+				postNvmeSmartHealthInformation.NumErrLogEntries = int64(preNvmeAttribute.Value)
+			case "warning_temp_time":
+				postNvmeSmartHealthInformation.WarningTempTime = int64(preNvmeAttribute.Value)
+			case "critical_comp_time":
+				postNvmeSmartHealthInformation.CriticalCompTime = int64(preNvmeAttribute.Value)
+			}
+		}
+
+		postDeviceSmartData.ProcessNvmeSmartInfo(postNvmeSmartHealthInformation)
+
+	} else if preDevice.IsScsi() {
+		//info collector.SmartInfo
+		var postScsiGrownDefectList int64
+		postScsiErrorCounterLog := collector.ScsiErrorCounterLog{
+			Read: struct {
+				ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+				ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+				ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+				TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+				CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+				GigabytesProcessed               string `json:"gigabytes_processed"`
+				TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+			}{},
+			Write: struct {
+				ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
+				ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
+				ErrorsCorrectedByRereadsRewrites int64  `json:"errors_corrected_by_rereads_rewrites"`
+				TotalErrorsCorrected             int64  `json:"total_errors_corrected"`
+				CorrectionAlgorithmInvocations   int64  `json:"correction_algorithm_invocations"`
+				GigabytesProcessed               string `json:"gigabytes_processed"`
+				TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
+			}{},
+		}
+
+		for _, preScsiAttribute := range preSmartResult.ScsiAttributes {
+			switch preScsiAttribute.AttributeId {
+			case "scsi_grown_defect_list":
+				postScsiGrownDefectList = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_eccfast":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_eccdelayed":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value)
+			case "read.errors_corrected_by_rereads_rewrites":
+				postScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value)
+			case "read.total_errors_corrected":
+				postScsiErrorCounterLog.Read.TotalErrorsCorrected = int64(preScsiAttribute.Value)
+			case "read.correction_algorithm_invocations":
+				postScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value)
+			case "read.total_uncorrected_errors":
+				postScsiErrorCounterLog.Read.TotalUncorrectedErrors = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_eccfast":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_eccdelayed":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value)
+			case "write.errors_corrected_by_rereads_rewrites":
+				postScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value)
+			case "write.total_errors_corrected":
+				postScsiErrorCounterLog.Write.TotalErrorsCorrected = int64(preScsiAttribute.Value)
+			case "write.correction_algorithm_invocations":
+				postScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value)
+			case "write.total_uncorrected_errors":
+				postScsiErrorCounterLog.Write.TotalUncorrectedErrors = int64(preScsiAttribute.Value)
+			}
+		}
+		postDeviceSmartData.ProcessScsiSmartInfo(postScsiGrownDefectList, postScsiErrorCounterLog)
+	} else {
+		return fmt.Errorf("Unknown device protocol: %s", preDevice.DeviceProtocol), postDeviceSmartData
+	}
+
+	return nil, postDeviceSmartData
+}

webapp/backend/pkg/models/collector/smart.go

@@ -135,28 +135,7 @@ type SmartInfo struct {
 	} `json:"ata_sct_temperature_history"`
 	AtaSmartAttributes struct {
 		Revision int                           `json:"revision"`
-		Table    []struct {
+		Table    []AtaSmartAttributesTableItem `json:"table"`
-			ID         int    `json:"id"`
-			Name       string `json:"name"`
-			Value      int64  `json:"value"`
-			Worst      int64  `json:"worst"`
-			Thresh     int64  `json:"thresh"`
-			WhenFailed string `json:"when_failed"`
-			Flags      struct {
-				Value         int    `json:"value"`
-				String        string `json:"string"`
-				Prefailure    bool   `json:"prefailure"`
-				UpdatedOnline bool   `json:"updated_online"`
-				Performance   bool   `json:"performance"`
-				ErrorRate     bool   `json:"error_rate"`
-				EventCount    bool   `json:"event_count"`
-				AutoKeep      bool   `json:"auto_keep"`
-			} `json:"flags"`
-			Raw struct {
-				Value  int64  `json:"value"`
-				String string `json:"string"`
-			} `json:"raw"`
-		} `json:"table"`
 	} `json:"ata_smart_attributes"`
 	AtaSmartErrorLog struct {
 		Summary struct {
@@ -250,7 +229,41 @@ type SmartInfo struct {
 		} `json:"utilization"`
 		FormattedLbaSize int `json:"formatted_lba_size"`
 	} `json:"nvme_namespaces"`
-	NvmeSmartHealthInformationLog struct {
+	NvmeSmartHealthInformationLog NvmeSmartHealthInformationLog `json:"nvme_smart_health_information_log"`
+
+	// SCSI Protocol Specific Fields
+	Vendor              string              `json:"vendor"`
+	Product             string              `json:"product"`
+	ScsiVersion         string              `json:"scsi_version"`
+	ScsiGrownDefectList int64               `json:"scsi_grown_defect_list"`
+	ScsiErrorCounterLog ScsiErrorCounterLog `json:"scsi_error_counter_log"`
+}
+
+//Primary Attribute Structs
+type AtaSmartAttributesTableItem struct {
+	ID         int    `json:"id"`
+	Name       string `json:"name"`
+	Value      int64  `json:"value"`
+	Worst      int64  `json:"worst"`
+	Thresh     int64  `json:"thresh"`
+	WhenFailed string `json:"when_failed"`
+	Flags      struct {
+		Value         int    `json:"value"`
+		String        string `json:"string"`
+		Prefailure    bool   `json:"prefailure"`
+		UpdatedOnline bool   `json:"updated_online"`
+		Performance   bool   `json:"performance"`
+		ErrorRate     bool   `json:"error_rate"`
+		EventCount    bool   `json:"event_count"`
+		AutoKeep      bool   `json:"auto_keep"`
+	} `json:"flags"`
+	Raw struct {
+		Value  int64  `json:"value"`
+		String string `json:"string"`
+	} `json:"raw"`
+}
+
+type NvmeSmartHealthInformationLog struct {
 	CriticalWarning         int64 `json:"critical_warning"`
 	Temperature             int64 `json:"temperature"`
 	AvailableSpare          int64 `json:"available_spare"`
@@ -268,14 +281,9 @@ type SmartInfo struct {
 	NumErrLogEntries        int64 `json:"num_err_log_entries"`
 	WarningTempTime         int64 `json:"warning_temp_time"`
 	CriticalCompTime        int64 `json:"critical_comp_time"`
-	} `json:"nvme_smart_health_information_log"`
+}
 
-	// SCSI Protocol Specific Fields
+type ScsiErrorCounterLog struct {
-	Vendor              string `json:"vendor"`
-	Product             string `json:"product"`
-	ScsiVersion         string `json:"scsi_version"`
-	ScsiGrownDefectList int64  `json:"scsi_grown_defect_list"`
-	ScsiErrorCounterLog struct {
 	Read struct {
 		ErrorsCorrectedByEccfast         int64  `json:"errors_corrected_by_eccfast"`
 		ErrorsCorrectedByEccdelayed      int64  `json:"errors_corrected_by_eccdelayed"`
@@ -294,5 +302,4 @@ type SmartInfo struct {
 		GigabytesProcessed               string `json:"gigabytes_processed"`
 		TotalUncorrectedErrors           int64  `json:"total_uncorrected_errors"`
 	} `json:"write"`
-	} `json:"scsi_error_counter_log"`
 }

webapp/backend/pkg/models/measurements/smart.go

@@ -125,20 +125,20 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er
 	// process ATA/NVME/SCSI protocol data
 	sm.Attributes = map[string]SmartAttribute{}
 	if sm.DeviceProtocol == pkg.DeviceProtocolAta {
-		sm.ProcessAtaSmartInfo(info)
+		sm.ProcessAtaSmartInfo(info.AtaSmartAttributes.Table)
 	} else if sm.DeviceProtocol == pkg.DeviceProtocolNvme {
-		sm.ProcessNvmeSmartInfo(info)
+		sm.ProcessNvmeSmartInfo(info.NvmeSmartHealthInformationLog)
 	} else if sm.DeviceProtocol == pkg.DeviceProtocolScsi {
-		sm.ProcessScsiSmartInfo(info)
+		sm.ProcessScsiSmartInfo(info.ScsiGrownDefectList, info.ScsiErrorCounterLog)
 	}
 
 	return nil
 }
 
 //generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) {
+func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTableItem) {
 	sm.Status = pkg.DeviceStatusPassed
-	for _, collectorAttr := range info.AtaSmartAttributes.Table {
+	for _, collectorAttr := range tableItems {
 		attrModel := SmartAtaAttribute{
 			AttributeId: collectorAttr.ID,
 			Value:       collectorAttr.Value,
@@ -164,25 +164,25 @@ func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) {
 }
 
 //generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) {
+func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.NvmeSmartHealthInformationLog) {
 
 	sm.Attributes = map[string]SmartAttribute{
-		"critical_warning":     (&SmartNvmeAttribute{AttributeId: "critical_warning", Value: info.NvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0}).PopulateAttributeStatus(),
+		"critical_warning":     (&SmartNvmeAttribute{AttributeId: "critical_warning", Value: nvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0}).PopulateAttributeStatus(),
-		"temperature":          (&SmartNvmeAttribute{AttributeId: "temperature", Value: info.NvmeSmartHealthInformationLog.Temperature, Threshold: -1}).PopulateAttributeStatus(),
+		"temperature":          (&SmartNvmeAttribute{AttributeId: "temperature", Value: nvmeSmartHealthInformationLog.Temperature, Threshold: -1}).PopulateAttributeStatus(),
-		"available_spare":      (&SmartNvmeAttribute{AttributeId: "available_spare", Value: info.NvmeSmartHealthInformationLog.AvailableSpare, Threshold: info.NvmeSmartHealthInformationLog.AvailableSpareThreshold}).PopulateAttributeStatus(),
+		"available_spare":      (&SmartNvmeAttribute{AttributeId: "available_spare", Value: nvmeSmartHealthInformationLog.AvailableSpare, Threshold: nvmeSmartHealthInformationLog.AvailableSpareThreshold}).PopulateAttributeStatus(),
-		"percentage_used":      (&SmartNvmeAttribute{AttributeId: "percentage_used", Value: info.NvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100}).PopulateAttributeStatus(),
+		"percentage_used":      (&SmartNvmeAttribute{AttributeId: "percentage_used", Value: nvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100}).PopulateAttributeStatus(),
-		"data_units_read":      (&SmartNvmeAttribute{AttributeId: "data_units_read", Value: info.NvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1}).PopulateAttributeStatus(),
+		"data_units_read":      (&SmartNvmeAttribute{AttributeId: "data_units_read", Value: nvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1}).PopulateAttributeStatus(),
-		"data_units_written":   (&SmartNvmeAttribute{AttributeId: "data_units_written", Value: info.NvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1}).PopulateAttributeStatus(),
+		"data_units_written":   (&SmartNvmeAttribute{AttributeId: "data_units_written", Value: nvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1}).PopulateAttributeStatus(),
-		"host_reads":           (&SmartNvmeAttribute{AttributeId: "host_reads", Value: info.NvmeSmartHealthInformationLog.HostReads, Threshold: -1}).PopulateAttributeStatus(),
+		"host_reads":           (&SmartNvmeAttribute{AttributeId: "host_reads", Value: nvmeSmartHealthInformationLog.HostReads, Threshold: -1}).PopulateAttributeStatus(),
-		"host_writes":          (&SmartNvmeAttribute{AttributeId: "host_writes", Value: info.NvmeSmartHealthInformationLog.HostWrites, Threshold: -1}).PopulateAttributeStatus(),
+		"host_writes":          (&SmartNvmeAttribute{AttributeId: "host_writes", Value: nvmeSmartHealthInformationLog.HostWrites, Threshold: -1}).PopulateAttributeStatus(),
-		"controller_busy_time": (&SmartNvmeAttribute{AttributeId: "controller_busy_time", Value: info.NvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1}).PopulateAttributeStatus(),
+		"controller_busy_time": (&SmartNvmeAttribute{AttributeId: "controller_busy_time", Value: nvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1}).PopulateAttributeStatus(),
-		"power_cycles":         (&SmartNvmeAttribute{AttributeId: "power_cycles", Value: info.NvmeSmartHealthInformationLog.PowerCycles, Threshold: -1}).PopulateAttributeStatus(),
+		"power_cycles":         (&SmartNvmeAttribute{AttributeId: "power_cycles", Value: nvmeSmartHealthInformationLog.PowerCycles, Threshold: -1}).PopulateAttributeStatus(),
-		"power_on_hours":       (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: info.NvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(),
+		"power_on_hours":       (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: nvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(),
-		"unsafe_shutdowns":     (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: info.NvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(),
+		"unsafe_shutdowns":     (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: nvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(),
-		"media_errors":         (&SmartNvmeAttribute{AttributeId: "media_errors", Value: info.NvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(),
+		"media_errors":         (&SmartNvmeAttribute{AttributeId: "media_errors", Value: nvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(),
-		"num_err_log_entries":  (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: info.NvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(),
+		"num_err_log_entries":  (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: nvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(),
-		"warning_temp_time":    (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: info.NvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(),
+		"warning_temp_time":    (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: nvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(),
-		"critical_comp_time":   (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: info.NvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(),
+		"critical_comp_time":   (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: nvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(),
 	}
 
 	//find analyzed attribute status
@@ -194,21 +194,21 @@ func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) {
 }
 
 //generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessScsiSmartInfo(info collector.SmartInfo) {
+func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog) {
 	sm.Attributes = map[string]SmartAttribute{
-		"scsi_grown_defect_list":                     (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: info.ScsiGrownDefectList, Threshold: 0}).PopulateAttributeStatus(),
+		"scsi_grown_defect_list":                     (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: defectGrownList, Threshold: 0}).PopulateAttributeStatus(),
-		"read_errors_corrected_by_eccfast":           (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_eccfast":           (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(),
-		"read_errors_corrected_by_eccdelayed":        (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_eccdelayed":        (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(),
-		"read_errors_corrected_by_rereads_rewrites":  (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(),
+		"read_errors_corrected_by_rereads_rewrites":  (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(),
-		"read_total_errors_corrected":                (&SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Value: info.ScsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(),
+		"read_total_errors_corrected":                (&SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Value: scsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(),
-		"read_correction_algorithm_invocations":      (&SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Value: info.ScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(),
+		"read_correction_algorithm_invocations":      (&SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Value: scsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(),
-		"read_total_uncorrected_errors":              (&SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Value: info.ScsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(),
+		"read_total_uncorrected_errors":              (&SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Value: scsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(),
-		"write_errors_corrected_by_eccfast":          (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_eccfast":          (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(),
-		"write_errors_corrected_by_eccdelayed":       (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_eccdelayed":       (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(),
-		"write_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(),
+		"write_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(),
-		"write_total_errors_corrected":               (&SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Value: info.ScsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(),
+		"write_total_errors_corrected":               (&SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Value: scsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(),
-		"write_correction_algorithm_invocations":     (&SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Value: info.ScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(),
+		"write_correction_algorithm_invocations":     (&SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Value: scsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(),
-		"write_total_uncorrected_errors":             (&SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Value: info.ScsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(),
+		"write_total_uncorrected_errors":             (&SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Value: scsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(),
 	}
 
 	//find analyzed attribute status