[BROKEN COMMIT]

This code leverages the new `types.isType` functionality introduced in the flux language (https://github.com/influxdata/flux/issues/2159)

This code will fix https://github.com/AnalogJ/scrutiny/issues/22 and all related issues.

Unfortunately this code is broken because the influxdb go client library does not correctly handle import statments in the task defintion.

blocked by
https://github.com/influxdata/influxdb-client-go/issues/322

CONTRIBUTING.md

@@ -76,17 +76,19 @@ If you'd like to populate the database with some test data,  you can run the fol
 
 > NOTE: you may need to update the `local_time` key within the JSON file, any timestamps older than ~3 weeks will be automatically ignored
 > (since the downsampling & retention policy takes effect at 2 weeks)
+> This is done automatically by the `webapp/backend/pkg/models/testdata/helper.go` script
 
 ```
-docker run -p 8086:8086 --rm influxdb:2.0
+docker run -p 8086:8086 --rm influxdb:2.2
 
 
 docker run --rm -p 8086:8086 \
+      -e DOCKER_INFLUXDB_INIT_MODE=setup \
       -e DOCKER_INFLUXDB_INIT_USERNAME=admin \
       -e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
       -e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
       -e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
-      influxdb:2.0
+      influxdb:2.2
 
 
 # curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/web/testdata/register-devices-req.json localhost:8080/api/devices/register

webapp/backend/pkg/database/scrutiny_repository.go

@@ -257,6 +257,37 @@ func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) err
 	return nil
 }
 
+/*
+
+
+sourceBucket = "metrics"
+rangeStart = -2w
+rangeEnd = -1w
+aggWindow = 1w
+destBucket = "metrics_weekly"
+destOrg = "scrutiny"
+
+smart_data = from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "smart" )
+|> filter(fn: (r) => r["_field"] !~ /(_measurement|device_protocol|device_wwn|attribute_id|raw_string|status_reason|when_failed)/)
+|> yield(name: "last")
+
+smart_data
+|> aggregateWindow(fn: mean, every: aggWindow)
+|> to(bucket: destBucket, org: destOrg)
+
+temp_data = from(bucket: sourceBucket)
+|> range(start: rangeStart, stop: rangeEnd)
+|> filter(fn: (r) => r["_measurement"] == "temp")
+|> toInt()
+|> yield(name: "mean")
+
+temp_data
+|> aggregateWindow(fn: mean, every: aggWindow)
+|> to(bucket: destBucket, org: destOrg)
+
+*/
 func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
 	var sourceBucket string // the source of the data
 	var destBucket string   // the destination for the aggregated data
@@ -284,7 +315,7 @@ func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
 		aggWindow = "1y"
 	}
 
-	return fmt.Sprintf(`
+	return fmt.Sprintf(`import "types"
   sourceBucket = "%s"
   rangeStart = %s
   rangeEnd = %s
@@ -295,18 +326,24 @@ func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
   smart_data = from(bucket: sourceBucket)
   |> range(start: rangeStart, stop: rangeEnd)
   |> filter(fn: (r) => r["_measurement"] == "smart" )
-  |> filter(fn: (r) => r["_field"] !~ /(_measurement|device_protocol|device_wwn|attribute_id|raw_string|status_reason|when_failed)/)
+  |> group(columns: ["device_wwn", "_field"])
-  |> yield(name: "last")
 
-  smart_data
+  non_numeric_smart_data = smart_data
-  |> aggregateWindow(fn: mean, every: aggWindow)
+    |> filter(fn: (r) => types.isType(v: r._value, type: "string") or types.isType(v: r._value, type: "bool"))
+    |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
+
+  numeric_smart_data = smart_data
+    |> filter(fn: (r) => types.isType(v: r._value, type: "int") or types.isType(v: r._value, type: "float"))
+    |> aggregateWindow(every: aggWindow, fn: mean, createEmpty: false)
+
+  union(tables: [non_numeric_smart_data, numeric_smart_data])
   |> to(bucket: destBucket, org: destOrg)
 
   temp_data = from(bucket: sourceBucket)
   |> range(start: rangeStart, stop: rangeEnd)
   |> filter(fn: (r) => r["_measurement"] == "temp")
+  |> group(columns: ["device_wwn"])
   |> toInt()
-  |> yield(name: "mean")
 
   temp_data
   |> aggregateWindow(fn: mean, every: aggWindow)
@@ -726,7 +763,9 @@ func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string {
 
 	*/
 
-	partialQueryStr := []string{`import "influxdata/influxdb/schema"`}
+	partialQueryStr := []string{
+		`import "influxdata/influxdb/schema"`,
+	}
 
 	nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey)
 

webapp/backend/pkg/models/testdata/helper.go

@@ -32,7 +32,7 @@ func main() {
 		log.Fatalf("ERROR %v", err)
 	}
 	defer file.Close()
-	_, err = SendPostRequest("http://localhost:8080/api/devices/register", file)
+	_, err = SendPostRequest("http://localhost:9090/api/devices/register", file)
 	if err != nil {
 		log.Fatalf("ERROR %v", err)
 	}
@@ -46,7 +46,7 @@ func main() {
 					log.Fatalf("ERROR %v", err)
 				}
 
-				_, err = SendPostRequest(fmt.Sprintf("http://localhost:8080/api/device/%s/smart", diskId), smartDataReader)
+				_, err = SendPostRequest(fmt.Sprintf("http://localhost:9090/api/device/%s/smart", diskId), smartDataReader)
 				if err != nil {
 					log.Fatalf("ERROR %v", err)
 				}