From e2670298fff8397d9504305ed5e58c29467fabf0 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 8 Feb 2022 12:05:28 -0800 Subject: [PATCH 01/15] ci: added README Signed-off-by: Sam Alba --- ci/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ci/README.md diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..0ad8a3d7 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,7 @@ +# Dagger CI + +WORK IN PROGRESS + +This is the home of the new CI + +Reference: https://github.com/dagger/dagger/issues/1549 From 7ecf0a97ad6f280c9b1a2428fc6f9cb437402b83 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 9 Feb 2022 20:48:22 -0800 Subject: [PATCH 02/15] ci: base implementation for building dagger binary Signed-off-by: Sam Alba --- ci/.gitignore | 1 + ci/base.cue | 34 +++++++++++++++++++++++++++ ci/main.cue | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 ci/.gitignore create mode 100644 ci/base.cue create mode 100644 ci/main.cue diff --git a/ci/.gitignore b/ci/.gitignore new file mode 100644 index 00000000..a007feab --- /dev/null +++ b/ci/.gitignore @@ -0,0 +1 @@ +build/* diff --git a/ci/base.cue b/ci/base.cue new file mode 100644 index 00000000..4e359b9e --- /dev/null +++ b/ci/base.cue @@ -0,0 +1,34 @@ +package main + +import ( + "universe.dagger.io/docker" +) + +let GoVersion = "1.17" + +// Base container images used for the CI +images: { + + // base image to build go binaries + goBuilder: docker.#Build & { + _packages: ["bash", "git"] + + steps: [ + docker.#Pull & { + source: "index.docker.io/golang:\(GoVersion)-alpine" + }, + for pkg in _packages { + docker.#Run & { + command: { + name: "apk" + args: ["add", pkg] + flags: { + "-U": true + "--no-cache": true + } + } + } + }, + ] + } +} diff --git a/ci/main.cue b/ci/main.cue new file mode 100644 index 00000000..b06fd4c7 --- /dev/null +++ b/ci/main.cue @@ -0,0 +1,64 @@ +package main + +import ( + // "dagger.io/dagger" + "dagger.io/dagger/engine" + + "universe.dagger.io/bash" +) + +engine.#Plan & { + inputs: { + params: { + // FIXME: implement condition actions using params + } + + directories: { + // dagger repository + source: path: "../" + } + } + + outputs: directories: "go binaries": { + contents: actions.build.export.directories["/build"].contents + dest: "./build" + } + + actions: { + goModCache: engine.#CacheDir & { + id: "go mod cache" + } + + build: bash.#Run & { + input: images.goBuilder.output + + script: contents: #""" + export GOMODCACHE=/gomodcache + mkdir -p /build + git_revision=$(git rev-parse --short HEAD) + GO_ENABLED=0 \ + go build -v -o /build/dagger \ + -ldflags '-s -w -X go.dagger.io/dagger/version.Revision='${git_revision} \ + ./cmd/dagger/ + """# + + export: directories: "/build": _ + workdir: "/usr/src/dagger" + env: GOMODCACHE: "/gomodcache" + + mounts: { + "dagger source code": { + contents: inputs.directories.source.contents + dest: "/usr/src/dagger" + } + + "go mod cache": { + dest: "/gomodcache" + contents: goModCache + } + } + + workdir: mounts["dagger source code"].dest + } + } +} From 679bb301482e7a2da0c3746da434f10b36734dd6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 10 Feb 2022 16:09:08 -0800 Subject: [PATCH 03/15] ci: fixed typos + params for os/arch Signed-off-by: Sam Alba --- ci/main.cue | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ci/main.cue b/ci/main.cue index b06fd4c7..af7bde66 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -1,6 +1,8 @@ package main import ( + "strings" + // "dagger.io/dagger" "dagger.io/dagger/engine" @@ -10,6 +12,10 @@ import ( engine.#Plan & { inputs: { params: { + // FIXME: until we support a better way + os: string | *"darwin" + arch: string | *"amd64" + // FIXME: implement condition actions using params } @@ -32,20 +38,21 @@ engine.#Plan & { build: bash.#Run & { input: images.goBuilder.output + env: { + GOMODCACHE: mounts["go mod cache"].dest + GOOS: strings.ToLower(inputs.params.os) + GOARCH: strings.ToLower(inputs.params.arch) + } + script: contents: #""" - export GOMODCACHE=/gomodcache mkdir -p /build git_revision=$(git rev-parse --short HEAD) - GO_ENABLED=0 \ + CGO_ENABLED=0 \ go build -v -o /build/dagger \ -ldflags '-s -w -X go.dagger.io/dagger/version.Revision='${git_revision} \ ./cmd/dagger/ """# - export: directories: "/build": _ - workdir: "/usr/src/dagger" - env: GOMODCACHE: "/gomodcache" - mounts: { "dagger source code": { contents: inputs.directories.source.contents @@ -59,6 +66,7 @@ engine.#Plan & { } workdir: mounts["dagger source code"].dest + export: directories: "/build": _ } } } From f17c470feb7d8882c60ba8f2562bd5f376c08bd1 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 10 Feb 2022 17:18:57 -0800 Subject: [PATCH 04/15] ci: implemented go lint Signed-off-by: Sam Alba --- ci/base.cue | 11 +++++++++++ ci/main.cue | 34 +++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/ci/base.cue b/ci/base.cue index 4e359b9e..73fea364 100644 --- a/ci/base.cue +++ b/ci/base.cue @@ -5,6 +5,7 @@ import ( ) let GoVersion = "1.17" +let GolangCILintVersion = "1.44.0" // Base container images used for the CI images: { @@ -31,4 +32,14 @@ images: { }, ] } + + // base image for the Go linter + // https://golangci-lint.run/usage/install/#docker + goLinter: docker.#Build & { + steps: [ + docker.#Pull & { + source: "index.docker.io/golangci/golangci-lint:v\(GolangCILintVersion)" + }, + ] + } } diff --git a/ci/main.cue b/ci/main.cue index af7bde66..84269e57 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -3,9 +3,7 @@ package main import ( "strings" - // "dagger.io/dagger" "dagger.io/dagger/engine" - "universe.dagger.io/bash" ) @@ -13,7 +11,7 @@ engine.#Plan & { inputs: { params: { // FIXME: until we support a better way - os: string | *"darwin" + os: string | *"darwin" arch: string | *"amd64" // FIXME: implement condition actions using params @@ -26,8 +24,8 @@ engine.#Plan & { } outputs: directories: "go binaries": { - contents: actions.build.export.directories["/build"].contents - dest: "./build" + contents: actions.build.export.directories["/build"].contents + dest: "./build" } actions: { @@ -35,13 +33,19 @@ engine.#Plan & { id: "go mod cache" } + source: "dagger source code": { + contents: inputs.directories.source.contents + dest: "/usr/src/dagger" + } + + // FIXME: build only if the linter passed build: bash.#Run & { input: images.goBuilder.output env: { GOMODCACHE: mounts["go mod cache"].dest - GOOS: strings.ToLower(inputs.params.os) - GOARCH: strings.ToLower(inputs.params.arch) + GOOS: strings.ToLower(inputs.params.os) + GOARCH: strings.ToLower(inputs.params.arch) } script: contents: #""" @@ -54,10 +58,7 @@ engine.#Plan & { """# mounts: { - "dagger source code": { - contents: inputs.directories.source.contents - dest: "/usr/src/dagger" - } + source "go mod cache": { dest: "/gomodcache" @@ -68,5 +69,16 @@ engine.#Plan & { workdir: mounts["dagger source code"].dest export: directories: "/build": _ } + + goLint: bash.#Run & { + input: images.goLinter.output + + // FIXME: the source volume is too slow, taking >3m on docker for mac (vs < 2sec on the host machine) + script: contents: "golangci-lint run -v --timeout 5m" + workdir: mounts["dagger source code"].dest + mounts: { + source + } + } } } From 8e2f3eb21d892dd50f3ec5f6d0824930463a678e Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 17 Feb 2022 11:58:31 -0800 Subject: [PATCH 05/15] ci: cleaner pattern for sharing base images across actions Signed-off-by: Sam Alba --- ci/{base.cue => images.cue} | 8 +++++--- ci/main.cue | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) rename ci/{base.cue => images.cue} (84%) diff --git a/ci/base.cue b/ci/images.cue similarity index 84% rename from ci/base.cue rename to ci/images.cue index 73fea364..5753a435 100644 --- a/ci/base.cue +++ b/ci/images.cue @@ -8,10 +8,11 @@ let GoVersion = "1.17" let GolangCILintVersion = "1.44.0" // Base container images used for the CI -images: { +#Images: { // base image to build go binaries - goBuilder: docker.#Build & { + goBuilder: _goBuilder.output + _goBuilder: docker.#Build & { _packages: ["bash", "git"] steps: [ @@ -35,7 +36,8 @@ images: { // base image for the Go linter // https://golangci-lint.run/usage/install/#docker - goLinter: docker.#Build & { + goLinter: _goLinter.output + _goLinter: docker.#Build & { steps: [ docker.#Pull & { source: "index.docker.io/golangci/golangci-lint:v\(GolangCILintVersion)" diff --git a/ci/main.cue b/ci/main.cue index 84269e57..a59d7363 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -24,8 +24,8 @@ engine.#Plan & { } outputs: directories: "go binaries": { - contents: actions.build.export.directories["/build"].contents - dest: "./build" + contents: actions.build.export.directories["/build"].contents + dest: "./build" } actions: { @@ -33,6 +33,8 @@ engine.#Plan & { id: "go mod cache" } + baseImages: #Images + source: "dagger source code": { contents: inputs.directories.source.contents dest: "/usr/src/dagger" @@ -40,7 +42,7 @@ engine.#Plan & { // FIXME: build only if the linter passed build: bash.#Run & { - input: images.goBuilder.output + input: baseImages.goBuilder env: { GOMODCACHE: mounts["go mod cache"].dest @@ -71,7 +73,7 @@ engine.#Plan & { } goLint: bash.#Run & { - input: images.goLinter.output + input: baseImages.goLinter // FIXME: the source volume is too slow, taking >3m on docker for mac (vs < 2sec on the host machine) script: contents: "golangci-lint run -v --timeout 5m" From 2a9cf1ce7c5f944cc0fb61bf54a86cb84bad6eb5 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 17 Feb 2022 20:39:40 -0800 Subject: [PATCH 06/15] ci: fixed performance issues, converted more fields to hidden Signed-off-by: Sam Alba --- ci/images.cue | 8 ++------ ci/main.cue | 37 +++++++++++++++++++++---------------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/ci/images.cue b/ci/images.cue index 5753a435..be8615cb 100644 --- a/ci/images.cue +++ b/ci/images.cue @@ -37,11 +37,7 @@ let GolangCILintVersion = "1.44.0" // base image for the Go linter // https://golangci-lint.run/usage/install/#docker goLinter: _goLinter.output - _goLinter: docker.#Build & { - steps: [ - docker.#Pull & { - source: "index.docker.io/golangci/golangci-lint:v\(GolangCILintVersion)" - }, - ] + _goLinter: docker.#Pull & { + source: "index.docker.io/golangci/golangci-lint:v\(GolangCILintVersion)" } } diff --git a/ci/main.cue b/ci/main.cue index a59d7363..ba877a59 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -8,13 +8,19 @@ import ( ) engine.#Plan & { + + // FIXME: Ideally we would want to automatically set the platform's arch identical to the host + // to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86) + platform: "linux/aarch64" + inputs: { params: { // FIXME: until we support a better way os: string | *"darwin" arch: string | *"amd64" - // FIXME: implement condition actions using params + // FIXME: implement condition actions using params until we have a + // better way to select specific actions } directories: { @@ -29,20 +35,23 @@ engine.#Plan & { } actions: { - goModCache: engine.#CacheDir & { - id: "go mod cache" + _goModCache: "go mod cache": { + dest: "/gomodcache" + contents: engine.#CacheDir & { + id: "go mod cache" + } } - baseImages: #Images + _baseImages: #Images - source: "dagger source code": { + _source: "dagger source code": { contents: inputs.directories.source.contents dest: "/usr/src/dagger" } // FIXME: build only if the linter passed build: bash.#Run & { - input: baseImages.goBuilder + input: _baseImages.goBuilder env: { GOMODCACHE: mounts["go mod cache"].dest @@ -59,27 +68,23 @@ engine.#Plan & { ./cmd/dagger/ """# + workdir: mounts["dagger source code"].dest mounts: { - source - - "go mod cache": { - dest: "/gomodcache" - contents: goModCache - } + _source + _goModCache } - workdir: mounts["dagger source code"].dest export: directories: "/build": _ } goLint: bash.#Run & { - input: baseImages.goLinter + input: _baseImages.goLinter - // FIXME: the source volume is too slow, taking >3m on docker for mac (vs < 2sec on the host machine) script: contents: "golangci-lint run -v --timeout 5m" workdir: mounts["dagger source code"].dest mounts: { - source + _source + _goModCache } } } From 297fded56dd15f97f678392b34e5504ad450ed31 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 23 Feb 2022 15:15:44 -0800 Subject: [PATCH 07/15] ci: ported to new Plan syntax + implemented CUE fmt + go linter + code cleanup Signed-off-by: Sam Alba --- ci/images.cue | 37 +++++++++++++++++++++++++++++++++++++ ci/main.cue | 41 ++++++++++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/ci/images.cue b/ci/images.cue index be8615cb..8263b1c7 100644 --- a/ci/images.cue +++ b/ci/images.cue @@ -6,6 +6,7 @@ import ( let GoVersion = "1.17" let GolangCILintVersion = "1.44.0" +let CUEVersion = "0.4.2" // Base container images used for the CI #Images: { @@ -40,4 +41,40 @@ let GolangCILintVersion = "1.44.0" _goLinter: docker.#Pull & { source: "index.docker.io/golangci/golangci-lint:v\(GolangCILintVersion)" } + + // base image for CUE cli + alpine distrib + cue: _cue._alpine.output + _cue: { + _cueBinary: docker.#Pull & { + source: "index.docker.io/cuelang/cue:\(CUEVersion)" + } + + _alpine: docker.#Build & { + _packages: ["bash", "git"] + + steps: [ + docker.#Pull & { + source: "index.docker.io/alpine:3" + }, + for pkg in _packages { + docker.#Run & { + command: { + name: "apk" + args: ["add", pkg] + flags: { + "-U": true + "--no-cache": true + } + } + } + }, + docker.#Copy & { + // input: _alpine.output + contents: _cueBinary.output.rootfs + source: "/usr/bin/cue" + dest: "/usr/bin/cue" + }, + ] + } + } } diff --git a/ci/main.cue b/ci/main.cue index ba877a59..27704095 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -3,11 +3,11 @@ package main import ( "strings" - "dagger.io/dagger/engine" + "dagger.io/dagger" "universe.dagger.io/bash" ) -engine.#Plan & { +dagger.#Plan & { // FIXME: Ideally we would want to automatically set the platform's arch identical to the host // to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86) @@ -29,22 +29,25 @@ engine.#Plan & { } } - outputs: directories: "go binaries": { - contents: actions.build.export.directories["/build"].contents - dest: "./build" - } + // FIXME? + // FTL failed to load plan: outputs.directories."go binaries".contents: undefined field: contents: + // + // outputs: directories: "go binaries": { + // contents: actions.build.export.directories["/build"].contents + // dest: "./build" + // } actions: { _goModCache: "go mod cache": { dest: "/gomodcache" - contents: engine.#CacheDir & { + contents: dagger.#CacheDir & { id: "go mod cache" } } _baseImages: #Images - _source: "dagger source code": { + _sourceCode: "dagger source code": { contents: inputs.directories.source.contents dest: "/usr/src/dagger" } @@ -63,14 +66,14 @@ engine.#Plan & { mkdir -p /build git_revision=$(git rev-parse --short HEAD) CGO_ENABLED=0 \ - go build -v -o /build/dagger \ - -ldflags '-s -w -X go.dagger.io/dagger/version.Revision='${git_revision} \ - ./cmd/dagger/ + go build -v -o /build/dagger \ + -ldflags '-s -w -X go.dagger.io/dagger/version.Revision='${git_revision} \ + ./cmd/dagger/ """# workdir: mounts["dagger source code"].dest mounts: { - _source + _sourceCode _goModCache } @@ -83,9 +86,21 @@ engine.#Plan & { script: contents: "golangci-lint run -v --timeout 5m" workdir: mounts["dagger source code"].dest mounts: { - _source + _sourceCode _goModCache } } + + cueFmt: bash.#Run & { + input: _baseImages.cue + + script: contents: #""" + find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -n 1 -P 8 cue fmt -s + """# + workdir: mounts["dagger source code"].dest + mounts: { + _sourceCode + } + } } } From ba0f565dbffe039cc05f83fbe8a3544ecc6788d4 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 23 Feb 2022 16:21:51 -0800 Subject: [PATCH 08/15] ci: fixed output + implemented cue linter Signed-off-by: Sam Alba --- ci/main.cue | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ci/main.cue b/ci/main.cue index 27704095..7f480c1a 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -29,13 +29,10 @@ dagger.#Plan & { } } - // FIXME? - // FTL failed to load plan: outputs.directories."go binaries".contents: undefined field: contents: - // - // outputs: directories: "go binaries": { - // contents: actions.build.export.directories["/build"].contents - // dest: "./build" - // } + outputs: directories: "go binaries": { + contents: actions.build.export.directories["/build"] + dest: "./build" + } actions: { _goModCache: "go mod cache": { @@ -91,11 +88,12 @@ dagger.#Plan & { } } - cueFmt: bash.#Run & { + cueLint: bash.#Run & { input: _baseImages.cue script: contents: #""" find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -n 1 -P 8 cue fmt -s + test -z "$$(git status -s . | grep -e "^ M" | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)" """# workdir: mounts["dagger source code"].dest mounts: { From 492a7ff9c3ecdc558759f09d836830e988a189d2 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 23 Feb 2022 19:14:04 -0800 Subject: [PATCH 09/15] ci: fix cue linter Signed-off-by: Sam Alba --- ci/main.cue | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/main.cue b/ci/main.cue index 7f480c1a..e5f17b61 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -92,8 +92,10 @@ dagger.#Plan & { input: _baseImages.cue script: contents: #""" + # Format the cue code find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -n 1 -P 8 cue fmt -s - test -z "$$(git status -s . | grep -e "^ M" | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)" + # Check that all formatted files where committed + test -z $(git status -s . | grep -e '^ M' | grep .cue | cut -d ' ' -f3) """# workdir: mounts["dagger source code"].dest mounts: { From 555966d040a27eef2f19a5b82b2609583b36bb97 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 28 Feb 2022 17:09:33 -0800 Subject: [PATCH 10/15] ci: explicit dependency between goBuild and goLint Signed-off-by: Sam Alba --- ci/main.cue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/main.cue b/ci/main.cue index e5f17b61..80c5b7f2 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -49,7 +49,6 @@ dagger.#Plan & { dest: "/usr/src/dagger" } - // FIXME: build only if the linter passed build: bash.#Run & { input: _baseImages.goBuilder @@ -57,6 +56,8 @@ dagger.#Plan & { GOMODCACHE: mounts["go mod cache"].dest GOOS: strings.ToLower(inputs.params.os) GOARCH: strings.ToLower(inputs.params.arch) + // Makes sure the linter completes before starting the build + "__depends": "\(goLint.exit)" } script: contents: #""" From 029a0d270fa4927618ab4e65147d5460acc00a6c Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 28 Feb 2022 17:46:29 -0800 Subject: [PATCH 11/15] ci: simplify code and reduce boiler plate + fixed caching Signed-off-by: Sam Alba --- ci/main.cue | 57 +++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/ci/main.cue b/ci/main.cue index 80c5b7f2..78c69d67 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -25,7 +25,10 @@ dagger.#Plan & { directories: { // dagger repository - source: path: "../" + source: { + path: "../" + exclude: ["./ci"] + } } } @@ -35,27 +38,35 @@ dagger.#Plan & { } actions: { - _goModCache: "go mod cache": { - dest: "/gomodcache" - contents: dagger.#CacheDir & { - id: "go mod cache" + _mountGoCache: { + mounts: "go mod cache": { + dest: "/root/.gocache" + contents: dagger.#CacheDir & { + id: "go mod cache" + } } + env: GOMODCACHE: mounts["go mod cache"].dest + } + + _mountSourceCode: { + mounts: "dagger source code": { + contents: inputs.directories.source.contents + dest: "/usr/src/dagger" + } + workdir: mounts["dagger source code"].dest } _baseImages: #Images - _sourceCode: "dagger source code": { - contents: inputs.directories.source.contents - dest: "/usr/src/dagger" - } - build: bash.#Run & { + _mountSourceCode + _mountGoCache + input: _baseImages.goBuilder env: { - GOMODCACHE: mounts["go mod cache"].dest - GOOS: strings.ToLower(inputs.params.os) - GOARCH: strings.ToLower(inputs.params.arch) + GOOS: strings.ToLower(inputs.params.os) + GOARCH: strings.ToLower(inputs.params.arch) // Makes sure the linter completes before starting the build "__depends": "\(goLint.exit)" } @@ -69,27 +80,21 @@ dagger.#Plan & { ./cmd/dagger/ """# - workdir: mounts["dagger source code"].dest - mounts: { - _sourceCode - _goModCache - } - export: directories: "/build": _ } goLint: bash.#Run & { + _mountSourceCode + _mountGoCache + input: _baseImages.goLinter script: contents: "golangci-lint run -v --timeout 5m" - workdir: mounts["dagger source code"].dest - mounts: { - _sourceCode - _goModCache - } } cueLint: bash.#Run & { + _mountSourceCode + input: _baseImages.cue script: contents: #""" @@ -98,10 +103,6 @@ dagger.#Plan & { # Check that all formatted files where committed test -z $(git status -s . | grep -e '^ M' | grep .cue | cut -d ' ' -f3) """# - workdir: mounts["dagger source code"].dest - mounts: { - _sourceCode - } } } } From 646a4d153d6f7893eee350095977dbbe56ceaf3e Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 1 Mar 2022 14:48:50 -0800 Subject: [PATCH 12/15] ci: implemented go test Signed-off-by: Sam Alba --- ci/images.cue | 2 +- ci/main.cue | 32 ++++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/ci/images.cue b/ci/images.cue index 8263b1c7..6e2296f7 100644 --- a/ci/images.cue +++ b/ci/images.cue @@ -14,7 +14,7 @@ let CUEVersion = "0.4.2" // base image to build go binaries goBuilder: _goBuilder.output _goBuilder: docker.#Build & { - _packages: ["bash", "git"] + _packages: ["bash", "git", "alpine-sdk"] steps: [ docker.#Pull & { diff --git a/ci/main.cue b/ci/main.cue index 78c69d67..d4df297e 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -33,7 +33,7 @@ dagger.#Plan & { } outputs: directories: "go binaries": { - contents: actions.build.export.directories["/build"] + contents: actions.goBuild.export.directories["/build"] dest: "./build" } @@ -58,24 +58,27 @@ dagger.#Plan & { _baseImages: #Images - build: bash.#Run & { + // Go build the dagger binary + // depends on goLint and goTest to complete successfully + goBuild: bash.#Run & { _mountSourceCode _mountGoCache input: _baseImages.goBuilder env: { - GOOS: strings.ToLower(inputs.params.os) - GOARCH: strings.ToLower(inputs.params.arch) - // Makes sure the linter completes before starting the build - "__depends": "\(goLint.exit)" + GOOS: strings.ToLower(inputs.params.os) + GOARCH: strings.ToLower(inputs.params.arch) + CGO_ENABLED: "0" + // Makes sure the linter and unit tests complete before starting the build + "__depends_lint": "\(goLint.exit)" + "__depends_tests": "\(goTest.exit)" } script: contents: #""" mkdir -p /build git_revision=$(git rev-parse --short HEAD) - CGO_ENABLED=0 \ - go build -v -o /build/dagger \ + go build -v -o /build/dagger \ -ldflags '-s -w -X go.dagger.io/dagger/version.Revision='${git_revision} \ ./cmd/dagger/ """# @@ -83,20 +86,29 @@ dagger.#Plan & { export: directories: "/build": _ } + // Go unit tests + goTest: bash.#Run & { + _mountSourceCode + _mountGoCache + + input: _baseImages.goBuilder + script: contents: "go test -race -v ./..." + } + + // Go lint using golangci-lint goLint: bash.#Run & { _mountSourceCode _mountGoCache input: _baseImages.goLinter - script: contents: "golangci-lint run -v --timeout 5m" } + // CUE lint cueLint: bash.#Run & { _mountSourceCode input: _baseImages.cue - script: contents: #""" # Format the cue code find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -n 1 -P 8 cue fmt -s From 576772147807fac4b4a77abd298d2a7d22c32ecd Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 1 Mar 2022 14:49:09 -0800 Subject: [PATCH 13/15] ci: added github workflow for running the tests Signed-off-by: Sam Alba --- .github/workflows/dagger-ci.yml | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/dagger-ci.yml diff --git a/.github/workflows/dagger-ci.yml b/.github/workflows/dagger-ci.yml new file mode 100644 index 00000000..e1e8599d --- /dev/null +++ b/.github/workflows/dagger-ci.yml @@ -0,0 +1,42 @@ +name: "Dagger CI" + +on: + push: + branches: [main] + paths: + - '**.sh' + - '**.bash' + - '**.go' + - '**.cue' + - '**.bats' + - 'Makefile' + - 'go.mod' + - 'go.sum' + - '.github/workflows/dagger-ci.yml' + pull_request: + branches: [main] + paths: + - '**.sh' + - '**.bash' + - '**.go' + - '**.cue' + - '**.bats' + - 'Makefile' + - 'go.mod' + - 'go.sum' + - '.github/workflows/dagger-ci.yml' + + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Dagger + uses: dagger/dagger-for-github@v1 + with: + age-key: ${{ secrets.DAGGER_AGE_KEY }} + args: up ./ci From 637a3c9efd16a368993376df482ded0e16b15654 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 9 Mar 2022 16:22:49 -0800 Subject: [PATCH 14/15] ci: ported config to new client API Signed-off-by: Sam Alba --- ci/cue.mod/module.cue | 1 + ci/cue.mod/pkg/dagger.io | 1 + ci/cue.mod/pkg/universe.dagger.io | 1 + ci/main.cue | 40 ++++++++++--------------------- 4 files changed, 15 insertions(+), 28 deletions(-) create mode 100644 ci/cue.mod/module.cue create mode 120000 ci/cue.mod/pkg/dagger.io create mode 120000 ci/cue.mod/pkg/universe.dagger.io diff --git a/ci/cue.mod/module.cue b/ci/cue.mod/module.cue new file mode 100644 index 00000000..b57d0104 --- /dev/null +++ b/ci/cue.mod/module.cue @@ -0,0 +1 @@ +module: "" \ No newline at end of file diff --git a/ci/cue.mod/pkg/dagger.io b/ci/cue.mod/pkg/dagger.io new file mode 120000 index 00000000..61c69ed9 --- /dev/null +++ b/ci/cue.mod/pkg/dagger.io @@ -0,0 +1 @@ +../../../pkg/dagger.io \ No newline at end of file diff --git a/ci/cue.mod/pkg/universe.dagger.io b/ci/cue.mod/pkg/universe.dagger.io new file mode 120000 index 00000000..9b1eece6 --- /dev/null +++ b/ci/cue.mod/pkg/universe.dagger.io @@ -0,0 +1 @@ +../../../pkg/universe.dagger.io \ No newline at end of file diff --git a/ci/main.cue b/ci/main.cue index d4df297e..ebaf5f79 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -1,8 +1,6 @@ package main import ( - "strings" - "dagger.io/dagger" "universe.dagger.io/bash" ) @@ -13,28 +11,14 @@ dagger.#Plan & { // to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86) platform: "linux/aarch64" - inputs: { - params: { - // FIXME: until we support a better way - os: string | *"darwin" - arch: string | *"amd64" - - // FIXME: implement condition actions using params until we have a - // better way to select specific actions - } - - directories: { - // dagger repository - source: { - path: "../" - exclude: ["./ci"] - } - } - } - - outputs: directories: "go binaries": { - contents: actions.goBuild.export.directories["/build"] - dest: "./build" + client: filesystem: { + "../": read: exclude: [ + "ci", + "node_modules", + "cmd/dagger/dagger", + "cmd/dagger/dagger-debug", + ] + "./build": write: contents: actions.build.export.directories["/build"] } actions: { @@ -50,7 +34,7 @@ dagger.#Plan & { _mountSourceCode: { mounts: "dagger source code": { - contents: inputs.directories.source.contents + contents: client.filesystem."../".read.contents dest: "/usr/src/dagger" } workdir: mounts["dagger source code"].dest @@ -60,15 +44,15 @@ dagger.#Plan & { // Go build the dagger binary // depends on goLint and goTest to complete successfully - goBuild: bash.#Run & { + build: bash.#Run & { _mountSourceCode _mountGoCache input: _baseImages.goBuilder env: { - GOOS: strings.ToLower(inputs.params.os) - GOARCH: strings.ToLower(inputs.params.arch) + GOOS: client.platform.os + GOARCH: client.platform.arch CGO_ENABLED: "0" // Makes sure the linter and unit tests complete before starting the build "__depends_lint": "\(goLint.exit)" From eec7bc0a95dbf97a53d39017fd4f58940fe8fe1e Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 9 Mar 2022 16:31:14 -0800 Subject: [PATCH 15/15] ci: fixed github action Signed-off-by: Sam Alba --- .github/workflows/dagger-ci.yml | 12 +++++++----- ci/main.cue | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dagger-ci.yml b/.github/workflows/dagger-ci.yml index e1e8599d..03d3f90b 100644 --- a/.github/workflows/dagger-ci.yml +++ b/.github/workflows/dagger-ci.yml @@ -26,17 +26,19 @@ on: - 'go.sum' - '.github/workflows/dagger-ci.yml' +env: + DAGGER_LOG_FORMAT: plain jobs: - deploy: + build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - - name: Dagger - uses: dagger/dagger-for-github@v1 + name: Dagger CI + uses: dagger/dagger-for-github@v2 with: - age-key: ${{ secrets.DAGGER_AGE_KEY }} - args: up ./ci + workdir: ci + args: do build diff --git a/ci/main.cue b/ci/main.cue index ebaf5f79..a80281fa 100644 --- a/ci/main.cue +++ b/ci/main.cue @@ -9,7 +9,9 @@ dagger.#Plan & { // FIXME: Ideally we would want to automatically set the platform's arch identical to the host // to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86) - platform: "linux/aarch64" + // Uncomment if running locally on Mac M1 to bypass qemu + // platform: "linux/aarch64" + platform: "linux/amd64" client: filesystem: { "../": read: exclude: [