gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed device processing in details page. Summary query is still broken.

Browse Source
This commit is contained in:
Jason Kulatunga
2022-05-04 20:13:11 -07:00
parent 8fe0dbed6b
commit fc5a9ba15e
3 changed files with 12 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/backend/pkg/database/scrutiny_repository.go
+1 -1
View File
@@ -368,7 +368,7 @@ func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []str
return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR} return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR}
case DURATION_KEY_FOREVER: case DURATION_KEY_FOREVER:
//data stored before the last year //data stored before the last year
return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR} return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR, DURATION_KEY_FOREVER}
} }
return []string{DURATION_KEY_WEEK} return []string{DURATION_KEY_WEEK}
} }
webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go
+1 -1
View File
@@ -92,7 +92,7 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error { func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error {
sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields)) //sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields))
p := influxdb2.NewPoint(measurement, p := influxdb2.NewPoint(measurement,
tags, tags,
fields, fields,
webapp/backend/pkg/database/scrutiny_repository_migrations.go
+10 -5
View File
@@ -88,11 +88,6 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
return err return err
} }
//weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary.
weeklyLookup := map[string]bool{}
monthlyLookup := map[string]bool{}
yearlyLookup := map[string]bool{}
//calculate bucket oldest dates //calculate bucket oldest dates
today := time.Now() today := time.Now()
dailyBucketMax := today.Add(-RETENTION_PERIOD_15_DAYS_IN_SECONDS * time.Second) //15 days dailyBucketMax := today.Add(-RETENTION_PERIOD_15_DAYS_IN_SECONDS * time.Second) //15 days
@@ -100,6 +95,12 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks
for _, preDevice := range preDevices { for _, preDevice := range preDevices {
sr.logger.Infof("\n====================================\n\nBegin processing device %s\n\n====================================\n", preDevice.WWN)
//weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary.
weeklyLookup := map[string]bool{}
monthlyLookup := map[string]bool{}
yearlyLookup := map[string]bool{}
for _, preSmartResult := range preDevice.SmartResults { //pre-migration smart results for _, preSmartResult := range preDevice.SmartResults { //pre-migration smart results
//we're looping in ASC mode, so from oldest entry to most current. //we're looping in ASC mode, so from oldest entry to most current.
@@ -183,6 +184,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
//write data to the monthly bucket if in the last 9 weeks, and week has not been processed yet //write data to the monthly bucket if in the last 9 weeks, and week has not been processed yet
if _, monthExists := monthlyLookup[yearMonthStr]; !monthExists && postSmartResults.Date.After(monthlyBucketMax) { if _, monthExists := monthlyLookup[yearMonthStr]; !monthExists && postSmartResults.Date.After(monthlyBucketMax) {
sr.logger.Debugf("device (%s) smart data added to bucket: monthly", preDevice.WWN) sr.logger.Debugf("device (%s) smart data added to bucket: monthly", preDevice.WWN)
//this month/year pair has not been processed //this month/year pair has not been processed
monthlyLookup[yearMonthStr] = true monthlyLookup[yearMonthStr] = true
// write point immediately // write point immediately
@@ -209,6 +211,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
if _, yearExists := yearlyLookup[yearStr]; !yearExists && year != today.Year() { if _, yearExists := yearlyLookup[yearStr]; !yearExists && year != today.Year() {
sr.logger.Debugf("device (%s) smart data added to bucket: yearly", preDevice.WWN) sr.logger.Debugf("device (%s) smart data added to bucket: yearly", preDevice.WWN)
//this year has not been processed //this year has not been processed
yearlyLookup[yearStr] = true yearlyLookup[yearStr] = true
// write point immediately // write point immediately
@@ -233,6 +236,8 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
} }
} }
} }
fmt.Printf("finished processing device %s. weekly: %d, monthly: %d, yearly: %d", preDevice.WWN, len(weeklyLookup), len(monthlyLookup), len(yearlyLookup))
} }
return nil return nil