Merge pull request #162 from AnalogJ/cron

Removed references to Jobber, replaced with Cron.
Created crontab at /etc/cron.d/scrutiny
Running cron in foreground.
Adding timezone support
Working Collector image (untested master).

.dockerignore

@@ -0,0 +1,6 @@
+/dist
+/vendor
+/.idea
+/.github
+/.git
+/webapp/frontend/node_modules

.github/workflows/build.yaml

@@ -11,12 +11,17 @@ jobs:
       PROJECT_PATH: /go/src/github.com/analogj/scrutiny
       CGO_ENABLED: 1
     steps:
+      - name: Git
+        run: |
+          apt-get update && apt-get install -y software-properties-common
+          add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git
+          git --version
       - name: Checkout
         uses: actions/checkout@v2
       - name: Test
         run: |
-          mkdir -p $PROJECT_PATH
+          mkdir -p $(dirname "$PROJECT_PATH")
-          cp -a $GITHUB_WORKSPACE/* $PROJECT_PATH/
+          cp -a $GITHUB_WORKSPACE $PROJECT_PATH
           cd $PROJECT_PATH
 
           go mod vendor

.github/workflows/release.yaml

@@ -21,6 +21,11 @@ jobs:
       PROJECT_PATH: /go/src/github.com/analogj/scrutiny
       CGO_ENABLED: 1
     steps:
+      - name: Git
+        run: |
+          apt-get update && apt-get install -y software-properties-common
+          add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git
+          git --version
       - name: Checkout
         uses: actions/checkout@v2
       - name: Bump version
@@ -32,8 +37,8 @@ jobs:
           github_token: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
       - name: Test
         run: |
-          mkdir -p $PROJECT_PATH
+          mkdir -p $(dirname "$PROJECT_PATH")
-          cp -a $GITHUB_WORKSPACE/* $PROJECT_PATH/
+          cp -a $GITHUB_WORKSPACE $PROJECT_PATH
           cd $PROJECT_PATH
 
           go mod vendor
@@ -130,7 +135,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /buildd/scrutiny-collector-metrics-linux-arm-5
+          asset_path: /build/scrutiny-collector-metrics-linux-arm-5
           asset_name: scrutiny-collector-metrics-linux-arm-5
           asset_content_type: application/octet-stream
 
@@ -151,7 +156,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /buildd/scrutiny-collector-metrics-linux-arm-6
+          asset_path: /build/scrutiny-collector-metrics-linux-arm-6
           asset_name: scrutiny-collector-metrics-linux-arm-6
           asset_content_type: application/octet-stream
 
@@ -172,7 +177,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
-          asset_path: /buildd/scrutiny-collector-metrics-linux-arm-7
+          asset_path: /build/scrutiny-collector-metrics-linux-arm-7
           asset_name: scrutiny-collector-metrics-linux-arm-7
           asset_content_type: application/octet-stream
 

Makefile

@@ -1,5 +1,6 @@
 export CGO_ENABLED = 1
 
+GO_WORKSPACE ?= /go/src/github.com/analogj/scrutiny
 
 BINARY=\
   linux/amd64   \
@@ -16,14 +17,14 @@ $(BINARY): OS = $(word 1,$(subst /, ,$*))
 $(BINARY): ARCH = $(word 2,$(subst /, ,$*))
 $(BINARY): build/scrutiny-web-%:
 	@echo "building web binary (OS = $(OS), ARCH = $(ARCH))"
-	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-web -tags "static netgo sqlite_omit_load_extension" ./webapp/backend/cmd/scrutiny/
+	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-web -tags "static netgo sqlite_omit_load_extension" ${GO_WORKSPACE}/webapp/backend/cmd/scrutiny/
 
 	chmod +x "/build/scrutiny-web-$(OS)-$(ARCH)"
 	file "/build/scrutiny-web-$(OS)-$(ARCH)" || true
 	ldd "/build/scrutiny-web-$(OS)-$(ARCH)" || true
 
 	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
-	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ./collector/cmd/collector-metrics/
+	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
 
 	chmod +x "/build/scrutiny-collector-metrics-$(OS)-$(ARCH)"
 	file "/build/scrutiny-collector-metrics-$(OS)-$(ARCH)" || true
@@ -33,10 +34,10 @@ windows/amd64: export OS = windows
 windows/amd64: export ARCH = amd64
 windows/amd64:
 	@echo "building web binary (OS = $(OS), ARCH = $(ARCH))"
-	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-web -tags "static netgo sqlite_omit_load_extension" ./webapp/backend/cmd/scrutiny/
+	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-web -tags "static netgo sqlite_omit_load_extension" ${GO_WORKSPACE}/webapp/backend/cmd/scrutiny/
 
 	@echo "building collector binary (OS = $(OS), ARCH = $(ARCH))"
-	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ./collector/cmd/collector-metrics/
+	xgo -v --targets="$(OS)/$(ARCH)" -ldflags "-extldflags=-static -X main.goos=$(OS) -X main.goarch=$(ARCH)" -out scrutiny-collector-metrics -tags "static netgo" ${GO_WORKSPACE}/collector/cmd/collector-metrics/
 
 freebsd/amd64: export GOOS = freebsd
 freebsd/amd64: export GOARCH = amd64

docker/Dockerfile

@@ -31,15 +31,13 @@ EXPOSE 8080
 WORKDIR /scrutiny
 ENV PATH="/scrutiny/bin:${PATH}"
 
-ADD https://github.com/dshearer/jobber/releases/download/v1.4.4/jobber_1.4.4-1_amd64.deb /tmp/
+RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates curl && update-ca-certificates
-RUN apt install /tmp/jobber_1.4.4-1_amd64.deb
-
-RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates curl && update-ca-certificates
 
 ADD https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-amd64.tar.gz /tmp/
 RUN tar xzf /tmp/s6-overlay-amd64.tar.gz -C /
 COPY /rootfs /
 
+COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /scrutiny/bin/
@@ -47,11 +45,11 @@ COPY --from=frontendbuild /scrutiny/dist /scrutiny/web
 RUN chmod +x /scrutiny/bin/scrutiny && \
     chmod +x /scrutiny/bin/scrutiny-collector-selftest && \
     chmod +x /scrutiny/bin/scrutiny-collector-metrics && \
+    chmod 0644 /etc/cron.d/scrutiny && \
+    rm -f /etc/cron.daily/* && \
     mkdir -p /scrutiny/web && \
     mkdir -p /scrutiny/config && \
-    mkdir -p /scrutiny/jobber && \
+    chmod -R ugo+rwx /scrutiny/config
-    chmod -R ugo+rwx /scrutiny/config && \
-    chmod -R ugo+rwx /scrutiny/jobber
 
 
 CMD ["/init"]

docker/Dockerfile.collector

@@ -11,20 +11,18 @@ RUN go mod vendor && \
 
 ########
 FROM ubuntu:bionic as runtime
-EXPOSE 8080
 WORKDIR /scrutiny
 ENV PATH="/scrutiny/bin:${PATH}"
 
-ADD https://github.com/dshearer/jobber/releases/download/v1.4.4/jobber_1.4.4-1_amd64.deb /tmp/
+RUN apt-get update && apt-get install -y cron smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates && update-ca-certificates
-RUN apt install /tmp/jobber_1.4.4-1_amd64.deb
-
-RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates && update-ca-certificates
-
-COPY /rootfs/scrutiny /scrutiny
 
+COPY /docker/entrypoint-collector.sh /entrypoint-collector.sh
+COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /scrutiny/bin/
 COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /scrutiny/bin/
 RUN chmod +x /scrutiny/bin/scrutiny-collector-selftest && \
-    chmod +x /scrutiny/bin/scrutiny-collector-metrics
+    chmod +x /scrutiny/bin/scrutiny-collector-metrics && \
+    chmod 0644 /etc/cron.d/scrutiny && \
+    rm -f /etc/cron.daily/apt /etc/cron.daily/dpkg /etc/cron.daily/passwd
 
-CMD ["/usr/lib/x86_64-linux-gnu/jobberrunner", "/scrutiny/jobber/jobber.yaml"]
+CMD ["/entrypoint-collector.sh"]

docker/entrypoint-collector.sh

@@ -0,0 +1,9 @@
+# Cron runs in its own isolated environment (usually using only /etc/environment )
+# So when the container starts up, we will do a dump of the runtime environment into a .env file that we
+# will then source into the crontab file (/etc/cron.d/scrutiny.sh)
+
+printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
+
+# now that we have the env start cron in the foreground
+echo "starting cron"
+cron -f

rootfs/etc/cont-init.d/01-timezone

@@ -0,0 +1,7 @@
+#!/usr/bin/with-contenv bash
+
+if [ -n "${TZ}" ]
+then
+    ln -snf "/usr/share/zoneinfo/${TZ}" /etc/localtime
+    echo "${TZ}" > /etc/timezone
+fi

rootfs/etc/cron.d/scrutiny

@@ -0,0 +1,15 @@
+MAILTO=""
+# Example of job definition:
+# .---------------- minute (0 - 59)
+# |  .------------- hour (0 - 23)
+# |  |  .---------- day of month (1 - 31)
+# |  |  |  .------- month (1 - 12) OR jan,feb,mar,apr ...
+# |  |  |  |  .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
+# |  |  |  |  |
+# *  *  *  *  * user-name command to be executed
+
+# correctly route collector logs (STDOUT & STDERR) to Cron foreground (collectable by Docker STDOUT)
+# cron schedule to run daily at midnight:  '0 0 * * *'
+# System environmental variables are stripped by cron, source our dump of the docker environmental variables before each command (/env.sh)
+0 0 * * * root . /env.sh; /scrutiny/bin/scrutiny-collector-metrics run >/proc/1/fd/1 2>/proc/1/fd/2
+# An empty line is required at the end of this file for a valid cron file.

rootfs/etc/services.d/cron/finish

@@ -1,4 +1,4 @@
 #!/usr/bin/execlineb -S0
 
-echo "jobber/cron exiting"
+echo "cron exiting"
 s6-svscanctl -t /var/run/s6/services

rootfs/etc/services.d/cron/run

@@ -0,0 +1,10 @@
+#!/usr/bin/with-contenv bash
+
+# Cron runs in its own isolated environment (usually using only /etc/environment )
+# So when the container starts up, we will do a dump of the runtime environment into a .env file that we
+# will then source into the crontab file (/etc/cron.d/scrutiny.sh)
+
+printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
+
+echo "starting cron"
+cron -f

rootfs/etc/services.d/jobber/run

@@ -1,5 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-echo "starting jobber/cron"
-
-su -c "/usr/lib/x86_64-linux-gnu/jobberrunner /scrutiny/jobber/jobber.yaml" root

rootfs/scrutiny/jobber/jobber.yaml

@@ -1,30 +0,0 @@
-version: 1.4
-
-prefs:
-  logPath: /scrutiny/jobber/log.log
-  runLog:
-    type: file
-    path: /scrutiny/jobber/runlog
-    maxFileLen: 100m
-    maxHistories: 2
-
-resultSinks:
-  - &filesystemSink
-    type: filesystem
-    path: /scrutiny/jobber
-    data:
-      - stdout
-      - stderr
-    maxAgeDays: 10
-
-jobs:
-  MetricsJob:
-    cmd: /scrutiny/bin/scrutiny-collector-metrics run
-    # run daily at midnight.
-    time: '0 0 0 * * *'
-    onError: Backoff
-    notifyOnSuccess:
-      - *filesystemSink
-    notifyOnFailure:
-      - *filesystemSink
-

webapp/backend/pkg/version/version.go

@@ -2,4 +2,4 @@ package version
 
 // VERSION is the app-global version string, which will be replaced with a
 // new value during packaging
-const VERSION = "0.3.1"
+const VERSION = "0.3.5"