From 1e18086c4d5059c8a2f2307f27af26ef2c93fcd7 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 00:15:23 -0600 Subject: [PATCH 01/13] update, working on a true static binary. --- .github/workflows/build.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a7a1843..b32301a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,6 +28,8 @@ jobs: GOOS: linux GOARCH: amd64 run: | + apt-get update && apt-get install -y file + cd $PROJECT_PATH go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go @@ -35,6 +37,8 @@ jobs: chmod +x scrutiny-web-linux-amd64 chmod +x scrutiny-collector-metrics-linux-amd64 + file scrutiny-web-linux-amd64 + file scrutiny-collector-metrics-linux-amd64 - name: Build arm env: GOOS: linux From f27883ea4cb995c2cf7722d63e26c95b670a4e06 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 00:24:33 -0600 Subject: [PATCH 02/13] remove requestcatcher live test. --- webapp/backend/pkg/web/server_test.go | 47 ++++++++++++++------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index e121a2d..adb7b12 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -164,29 +164,30 @@ func TestPopulateMultiple(t *testing.T) { //assert } -func TestSendTestNotificationRoute(t *testing.T) { - //setup - parentPath, _ := ioutil.TempDir("", "") - defer os.RemoveAll(parentPath) - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) - fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) - fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://scrutiny.requestcatcher.com/test"}) - ae := web.AppEngine{ - Config: fakeConfig, - } - router := ae.Setup(logrus.New()) - - //test - wr := httptest.NewRecorder() - req, _ := http.NewRequest("POST", "/api/health/notify", strings.NewReader("{}")) - router.ServeHTTP(wr, req) - - //assert - require.Equal(t, 200, wr.Code) -} +//TODO: this test should use a recorded request/response playback. +//func TestSendTestNotificationRoute(t *testing.T) { +// //setup +// parentPath, _ := ioutil.TempDir("", "") +// defer os.RemoveAll(parentPath) +// mockCtrl := gomock.NewController(t) +// defer mockCtrl.Finish() +// fakeConfig := mock_config.NewMockInterface(mockCtrl) +// fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) +// fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) +// fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://scrutiny.requestcatcher.com/test"}) +// ae := web.AppEngine{ +// Config: fakeConfig, +// } +// router := ae.Setup(logrus.New()) +// +// //test +// wr := httptest.NewRecorder() +// req, _ := http.NewRequest("POST", "/api/health/notify", strings.NewReader("{}")) +// router.ServeHTTP(wr, req) +// +// //assert +// require.Equal(t, 200, wr.Code) +//} func TestSendTestNotificationRoute_WebhookFailure(t *testing.T) { //setup From 4bd3dd431179a13f43321ec777627d3a0cde2898 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 00:32:13 -0600 Subject: [PATCH 03/13] testinig CGO_ENABLED=0 --- .github/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b32301a..9ba22a1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,6 +27,7 @@ jobs: env: GOOS: linux GOARCH: amd64 + CGO_ENABLED: 0 run: | apt-get update && apt-get install -y file From 7e78fb4e7dd23c3afadcac247d80138ea23c3c8c Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 00:39:04 -0600 Subject: [PATCH 04/13] test -extldflags=-static instead (CGO_ENABLED=0 works, but checking alternatives) --- .github/workflows/build.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9ba22a1..b3f3220 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,13 +27,12 @@ jobs: env: GOOS: linux GOARCH: amd64 - CGO_ENABLED: 0 run: | apt-get update && apt-get install -y file cd $PROJECT_PATH - go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go - go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go chmod +x scrutiny-web-linux-amd64 chmod +x scrutiny-collector-metrics-linux-amd64 From 99ec2eb2dcc3213734a517fe0367e0f46deba49d Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 00:45:26 -0600 Subject: [PATCH 05/13] using CGO_ENABLED=0 again (-extldflags=-static didnt work for webapp). --- .github/workflows/build.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b3f3220..a9fff48 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,12 +27,13 @@ jobs: env: GOOS: linux GOARCH: amd64 + CGO_ENABLED=0 run: | apt-get update && apt-get install -y file cd $PROJECT_PATH - go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go - go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go + go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go + go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go chmod +x scrutiny-web-linux-amd64 chmod +x scrutiny-collector-metrics-linux-amd64 @@ -43,6 +44,7 @@ jobs: env: GOOS: linux GOARCH: arm + CGO_ENABLED=0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=linux -X main.goarch=arm" -o scrutiny-web-linux-arm -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go @@ -55,6 +57,7 @@ jobs: env: GOOS: linux GOARCH: arm64 + CGO_ENABLED=0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=linux -X main.goarch=arm64" -o scrutiny-web-linux-arm64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go @@ -76,6 +79,7 @@ jobs: env: GOOS: freebsd GOARCH: amd64 + CGO_ENABLED=0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=freebsd -X main.goarch=amd64" -o scrutiny-web-freebsd-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go From 9ffc55ba13d5aa1b9caa53685dbdd0459706732c Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 00:55:19 -0600 Subject: [PATCH 06/13] fix --- .github/workflows/build.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a9fff48..c6129a2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,7 +27,7 @@ jobs: env: GOOS: linux GOARCH: amd64 - CGO_ENABLED=0 + CGO_ENABLED: 0 run: | apt-get update && apt-get install -y file @@ -44,7 +44,7 @@ jobs: env: GOOS: linux GOARCH: arm - CGO_ENABLED=0 + CGO_ENABLED: 0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=linux -X main.goarch=arm" -o scrutiny-web-linux-arm -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go @@ -57,7 +57,7 @@ jobs: env: GOOS: linux GOARCH: arm64 - CGO_ENABLED=0 + CGO_ENABLED: 0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=linux -X main.goarch=arm64" -o scrutiny-web-linux-arm64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go @@ -79,7 +79,7 @@ jobs: env: GOOS: freebsd GOARCH: amd64 - CGO_ENABLED=0 + CGO_ENABLED: 0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=freebsd -X main.goarch=amd64" -o scrutiny-web-freebsd-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go From aff9a0b25881423f93216eceed547018fcc2bc62 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 17:54:24 -0600 Subject: [PATCH 07/13] static build attempt 2. --- .github/workflows/build.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c6129a2..9139e07 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,13 +27,13 @@ jobs: env: GOOS: linux GOARCH: amd64 - CGO_ENABLED: 0 + CGO_ENABLED: 1 run: | apt-get update && apt-get install -y file cd $PROJECT_PATH - go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go - go build -ldflags "-X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go chmod +x scrutiny-web-linux-amd64 chmod +x scrutiny-collector-metrics-linux-amd64 @@ -44,7 +44,6 @@ jobs: env: GOOS: linux GOARCH: arm - CGO_ENABLED: 0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=linux -X main.goarch=arm" -o scrutiny-web-linux-arm -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go @@ -57,7 +56,6 @@ jobs: env: GOOS: linux GOARCH: arm64 - CGO_ENABLED: 0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=linux -X main.goarch=arm64" -o scrutiny-web-linux-arm64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go @@ -79,7 +77,6 @@ jobs: env: GOOS: freebsd GOARCH: amd64 - CGO_ENABLED: 0 run: | cd $PROJECT_PATH go build -ldflags "-X main.goos=freebsd -X main.goarch=amd64" -o scrutiny-web-freebsd-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go From 1d35b14c9d2729f6b685e77edcc2414bd193537d Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 17:58:47 -0600 Subject: [PATCH 08/13] update collector tags. --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9139e07..fc87271 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,7 +33,7 @@ jobs: cd $PROJECT_PATH go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go - go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go chmod +x scrutiny-web-linux-amd64 chmod +x scrutiny-collector-metrics-linux-amd64 From f410e60235c040ffa4d2c2421e52c073a9bf3a0b Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 18:51:38 -0600 Subject: [PATCH 09/13] print ldd info as well. --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fc87271..68a22ab 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,7 +39,9 @@ jobs: chmod +x scrutiny-collector-metrics-linux-amd64 file scrutiny-web-linux-amd64 + ldd scrutiny-web-linux-amd64 file scrutiny-collector-metrics-linux-amd64 + ldd scrutiny-collector-metrics-linux-amd64 - name: Build arm env: GOOS: linux From 9e10f3bf50a024b7c1e0f15cc6a7513ef64f2f42 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 18:56:28 -0600 Subject: [PATCH 10/13] using netgo --- .github/workflows/build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 68a22ab..37b7916 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -32,7 +32,9 @@ jobs: apt-get update && apt-get install -y file cd $PROJECT_PATH - go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go + echo "###### Build Web ######" + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-web-linux-amd64 -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go + echo "###### Build Collector ######" go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=amd64" -o scrutiny-collector-metrics-linux-amd64 -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go chmod +x scrutiny-web-linux-amd64 From 0a55a7076ff9ef278ea2969f760eaf7258231755 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 18:59:35 -0600 Subject: [PATCH 11/13] remove ldd command. --- .github/workflows/build.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 37b7916..6fa4a64 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -41,9 +41,7 @@ jobs: chmod +x scrutiny-collector-metrics-linux-amd64 file scrutiny-web-linux-amd64 - ldd scrutiny-web-linux-amd64 file scrutiny-collector-metrics-linux-amd64 - ldd scrutiny-collector-metrics-linux-amd64 - name: Build arm env: GOOS: linux From ac6b02808cfdf332727c3cb96db1ff13a89440ee Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 18:59:56 -0600 Subject: [PATCH 12/13] fix ldd command. --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6fa4a64..98a5007 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -41,7 +41,9 @@ jobs: chmod +x scrutiny-collector-metrics-linux-amd64 file scrutiny-web-linux-amd64 + ldd scrutiny-web-linux-amd64 || true file scrutiny-collector-metrics-linux-amd64 + ldd scrutiny-collector-metrics-linux-amd64 || true - name: Build arm env: GOOS: linux From e32fc5947e40ab09f5df6299e43bf6eb911472c3 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 8 Oct 2020 19:28:36 -0600 Subject: [PATCH 13/13] static binaries for CI. --- .github/workflows/build.yaml | 12 ++++++------ docs/INSTALL_MANUAL.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 98a5007..acc9635 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -50,8 +50,8 @@ jobs: GOARCH: arm run: | cd $PROJECT_PATH - go build -ldflags "-X main.goos=linux -X main.goarch=arm" -o scrutiny-web-linux-arm -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go - go build -ldflags "-X main.goos=linux -X main.goarch=arm" -o scrutiny-collector-metrics-linux-arm -tags "static" collector/cmd/collector-metrics/collector-metrics.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=arm" -o scrutiny-web-linux-arm -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=arm" -o scrutiny-collector-metrics-linux-arm -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go chmod +x scrutiny-web-linux-arm chmod +x scrutiny-collector-metrics-linux-arm @@ -62,8 +62,8 @@ jobs: GOARCH: arm64 run: | cd $PROJECT_PATH - go build -ldflags "-X main.goos=linux -X main.goarch=arm64" -o scrutiny-web-linux-arm64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go - go build -ldflags "-X main.goos=linux -X main.goarch=arm64" -o scrutiny-collector-metrics-linux-arm64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=arm64" -o scrutiny-web-linux-arm64 -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go + go build -ldflags "-extldflags=-static -X main.goos=linux -X main.goarch=arm64" -o scrutiny-collector-metrics-linux-arm64 -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go chmod +x scrutiny-web-linux-arm64 chmod +x scrutiny-collector-metrics-linux-arm64 @@ -83,8 +83,8 @@ jobs: GOARCH: amd64 run: | cd $PROJECT_PATH - go build -ldflags "-X main.goos=freebsd -X main.goarch=amd64" -o scrutiny-web-freebsd-amd64 -tags "static" webapp/backend/cmd/scrutiny/scrutiny.go - go build -ldflags "-X main.goos=freebsd -X main.goarch=amd64" -o scrutiny-collector-metrics-freebsd-amd64 -tags "static" collector/cmd/collector-metrics/collector-metrics.go + go build -ldflags "-extldflags=-static -X main.goos=freebsd -X main.goarch=amd64" -o scrutiny-web-freebsd-amd64 -tags "static netgo sqlite_omit_load_extension" webapp/backend/cmd/scrutiny/scrutiny.go + go build -ldflags "-extldflags=-static -X main.goos=freebsd -X main.goarch=amd64" -o scrutiny-collector-metrics-freebsd-amd64 -tags "static netgo" collector/cmd/collector-metrics/collector-metrics.go chmod +x scrutiny-web-freebsd-amd64 chmod +x scrutiny-collector-metrics-freebsd-amd64 diff --git a/docs/INSTALL_MANUAL.md b/docs/INSTALL_MANUAL.md index cbb95df..e6eb846 100644 --- a/docs/INSTALL_MANUAL.md +++ b/docs/INSTALL_MANUAL.md @@ -100,7 +100,7 @@ So you'll need to install the v7+ version using one of the following commands: - **Centos8:** - `dnf install https://extras.getpagespeed.com/release-el8-latest.rpm` - `dnf install smartmontools` - +- **FreeBSD:** `pkg install smartmontools` ### Directory Structure