gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix update, using raw flux script.

Browse Source
This commit is contained in:
Jason Kulatunga
2022-07-09 10:42:30 -07:00
parent 04563c0d0d
commit 0f0efac866
2 changed files with 116 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/backend/pkg/database/scrutiny_repository_tasks.go
+35 -28
View File
@@ -11,10 +11,10 @@ import (
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) error { func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) error {
weeklyTaskName := "tsk-weekly-aggr" weeklyTaskName := "tsk-weekly-aggr"
weeklyTaskScript := sr.DownsampleScript("weekly") weeklyTaskScript := sr.DownsampleScript("weekly", weeklyTaskName, "0 1 * * 0")
if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: weeklyTaskName}); findErr == nil && len(found) == 0 { if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: weeklyTaskName}); findErr == nil && len(found) == 0 {
//weekly on Sunday at 1:00am //weekly on Sunday at 1:00am
_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, weeklyTaskName, weeklyTaskScript, "0 1 * * 0", orgID) _, err := sr.influxTaskApi.CreateTaskByFlux(ctx, weeklyTaskScript, orgID)
if err != nil { if err != nil {
return err return err
} }
@@ -32,10 +32,10 @@ func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) err
} }
monthlyTaskName := "tsk-monthly-aggr" monthlyTaskName := "tsk-monthly-aggr"
monthlyTaskScript := sr.DownsampleScript("monthly") monthlyTaskScript := sr.DownsampleScript("monthly", monthlyTaskName, "30 1 1 * *")
if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: monthlyTaskName}); findErr == nil && len(found) == 0 { if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: monthlyTaskName}); findErr == nil && len(found) == 0 {
//monthly on first day of the month at 1:30am //monthly on first day of the month at 1:30am
_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, monthlyTaskName, monthlyTaskScript, "30 1 1 * *", orgID) _, err := sr.influxTaskApi.CreateTaskByFlux(ctx, monthlyTaskScript, orgID)
if err != nil { if err != nil {
return err return err
} }
@@ -53,10 +53,10 @@ func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) err
} }
yearlyTaskName := "tsk-yearly-aggr" yearlyTaskName := "tsk-yearly-aggr"
yearlyTaskScript := sr.DownsampleScript("yearly") yearlyTaskScript := sr.DownsampleScript("yearly", yearlyTaskName, "0 2 1 1 *")
if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: yearlyTaskName}); findErr == nil && len(found) == 0 { if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: yearlyTaskName}); findErr == nil && len(found) == 0 {
//yearly on the first day of the year at 2:00am //yearly on the first day of the year at 2:00am
_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, yearlyTaskName, yearlyTaskScript, "0 2 1 1 *", orgID) _, err := sr.influxTaskApi.CreateTaskByFlux(ctx, yearlyTaskScript, orgID)
if err != nil { if err != nil {
return err return err
} }
@@ -75,7 +75,7 @@ func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) err
return nil return nil
} }
func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string { func (sr *scrutinyRepository) DownsampleScript(aggregationType string, name string, cron string) string {
var sourceBucket string // the source of the data var sourceBucket string // the source of the data
var destBucket string // the destination for the aggregated data var destBucket string // the destination for the aggregated data
var rangeStart string var rangeStart string
@@ -124,30 +124,37 @@ func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
*/ */
return fmt.Sprintf(` return fmt.Sprintf(`
sourceBucket = "%s" option task = {
rangeStart = %s name: "%s",
rangeEnd = %s cron: "%s",
aggWindow = %s }
destBucket = "%s"
destOrg = "%s"
from(bucket: sourceBucket) sourceBucket = "%s"
|> range(start: rangeStart, stop: rangeEnd) rangeStart = %s
|> filter(fn: (r) => r["_measurement"] == "smart" ) rangeEnd = %s
|> group(columns: ["device_wwn", "_field"]) aggWindow = %s
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false) destBucket = "%s"
|> to(bucket: destBucket, org: destOrg) destOrg = "%s"
from(bucket: sourceBucket) from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd) |> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp") |> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn"]) |> group(columns: ["device_wwn", "_field"])
|> toInt() |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false) |> to(bucket: destBucket, org: destOrg)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp") from(bucket: sourceBucket)
|> to(bucket: destBucket, org: destOrg) |> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp")
|> group(columns: ["device_wwn"])
|> toInt()
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp")
|> to(bucket: destBucket, org: destOrg)
`, `,
name,
cron,
sourceBucket, sourceBucket,
rangeStart, rangeStart,
rangeEnd, rangeEnd,
webapp/backend/pkg/database/scrutiny_repository_tasks_test.go
+81 -66
View File
@@ -24,33 +24,38 @@ func Test_DownsampleScript_Weekly(t *testing.T) {
aggregationType := "weekly" aggregationType := "weekly"
//test //test
influxDbScript := deviceRepo.DownsampleScript(aggregationType) influxDbScript := deviceRepo.DownsampleScript(aggregationType, "tsk-weekly-aggr", "0 1 * * 0")
//assert //assert
require.Equal(t, ` require.Equal(t, `
sourceBucket = "metrics" option task = {
rangeStart = -2w name: "tsk-weekly-aggr",
rangeEnd = -1w cron: "0 1 * * 0",
aggWindow = 1w }
destBucket = "metrics_weekly"
destOrg = "scrutiny"
from(bucket: sourceBucket) sourceBucket = "metrics"
|> range(start: rangeStart, stop: rangeEnd) rangeStart = -2w
|> filter(fn: (r) => r["_measurement"] == "smart" ) rangeEnd = -1w
|> group(columns: ["device_wwn", "_field"]) aggWindow = 1w
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false) destBucket = "metrics_weekly"
|> to(bucket: destBucket, org: destOrg) destOrg = "scrutiny"
from(bucket: sourceBucket) from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd) |> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp") |> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn"]) |> group(columns: ["device_wwn", "_field"])
|> toInt() |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false) |> to(bucket: destBucket, org: destOrg)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp") from(bucket: sourceBucket)
|> to(bucket: destBucket, org: destOrg) |> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp")
|> group(columns: ["device_wwn"])
|> toInt()
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp")
|> to(bucket: destBucket, org: destOrg)
`, influxDbScript) `, influxDbScript)
} }
@@ -71,33 +76,38 @@ func Test_DownsampleScript_Monthly(t *testing.T) {
aggregationType := "monthly" aggregationType := "monthly"
//test //test
influxDbScript := deviceRepo.DownsampleScript(aggregationType) influxDbScript := deviceRepo.DownsampleScript(aggregationType, "tsk-monthly-aggr", "30 1 1 * *")
//assert //assert
require.Equal(t, ` require.Equal(t, `
sourceBucket = "metrics_weekly" option task = {
rangeStart = -2mo name: "tsk-monthly-aggr",
rangeEnd = -1mo cron: "30 1 1 * *",
aggWindow = 1mo }
destBucket = "metrics_monthly"
destOrg = "scrutiny"
from(bucket: sourceBucket) sourceBucket = "metrics_weekly"
|> range(start: rangeStart, stop: rangeEnd) rangeStart = -2mo
|> filter(fn: (r) => r["_measurement"] == "smart" ) rangeEnd = -1mo
|> group(columns: ["device_wwn", "_field"]) aggWindow = 1mo
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false) destBucket = "metrics_monthly"
|> to(bucket: destBucket, org: destOrg) destOrg = "scrutiny"
from(bucket: sourceBucket) from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd) |> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp") |> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn"]) |> group(columns: ["device_wwn", "_field"])
|> toInt() |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false) |> to(bucket: destBucket, org: destOrg)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp") from(bucket: sourceBucket)
|> to(bucket: destBucket, org: destOrg) |> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp")
|> group(columns: ["device_wwn"])
|> toInt()
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp")
|> to(bucket: destBucket, org: destOrg)
`, influxDbScript) `, influxDbScript)
} }
@@ -118,32 +128,37 @@ func Test_DownsampleScript_Yearly(t *testing.T) {
aggregationType := "yearly" aggregationType := "yearly"
//test //test
influxDbScript := deviceRepo.DownsampleScript(aggregationType) influxDbScript := deviceRepo.DownsampleScript(aggregationType, "tsk-yearly-aggr", "0 2 1 1 *")
//assert //assert
require.Equal(t, ` require.Equal(t, `
sourceBucket = "metrics_monthly" option task = {
rangeStart = -2y name: "tsk-yearly-aggr",
rangeEnd = -1y cron: "0 2 1 1 *",
aggWindow = 1y }
destBucket = "metrics_yearly"
destOrg = "scrutiny"
from(bucket: sourceBucket) sourceBucket = "metrics_monthly"
|> range(start: rangeStart, stop: rangeEnd) rangeStart = -2y
|> filter(fn: (r) => r["_measurement"] == "smart" ) rangeEnd = -1y
|> group(columns: ["device_wwn", "_field"]) aggWindow = 1y
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false) destBucket = "metrics_yearly"
|> to(bucket: destBucket, org: destOrg) destOrg = "scrutiny"
from(bucket: sourceBucket) from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd) |> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp") |> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn"]) |> group(columns: ["device_wwn", "_field"])
|> toInt() |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false) |> to(bucket: destBucket, org: destOrg)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp") from(bucket: sourceBucket)
|> to(bucket: destBucket, org: destOrg) |> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp")
|> group(columns: ["device_wwn"])
|> toInt()
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp")
|> to(bucket: destBucket, org: destOrg)
`, influxDbScript) `, influxDbScript)
} }