From f4c712e373c0a292dfa19732ac63a4131a2bf220 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 15 Apr 2021 12:09:01 -0700 Subject: [PATCH 1/5] pipeline: implemented op.#Exec.always correctly Signed-off-by: Sam Alba --- dagger/pipeline.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dagger/pipeline.go b/dagger/pipeline.go index 7cddcfd0..e90b7f6d 100644 --- a/dagger/pipeline.go +++ b/dagger/pipeline.go @@ -385,13 +385,8 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value, st llb.State) ( } // always? - // FIXME: initialize once for an entire compute job, to avoid cache misses if cmd.Always { - cacheBuster, err := randomID(8) - if err != nil { - return st, err - } - opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster)) + opts = append(opts, llb.IgnoreCache) } // mounts if mounts := op.Lookup("mount"); mounts.Exists() { From cfa9f08bc92cdd832d75e1b8fb4be302d107fac6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 15 Apr 2021 12:10:14 -0700 Subject: [PATCH 2/5] tests: added test to ensure an op.#Exec with no cache will not execute twice Signed-off-by: Sam Alba --- tests/compute/success/exec-nocache/main.cue | 25 +++++++++++++++++++++ tests/test-compute.sh | 2 ++ 2 files changed, 27 insertions(+) create mode 100644 tests/compute/success/exec-nocache/main.cue diff --git a/tests/compute/success/exec-nocache/main.cue b/tests/compute/success/exec-nocache/main.cue new file mode 100644 index 00000000..0a4152e8 --- /dev/null +++ b/tests/compute/success/exec-nocache/main.cue @@ -0,0 +1,25 @@ +package main + +import ( + "dagger.io/alpine" + "dagger.io/dagger/op" +) + +rand: { + string + + #up: [ + op.#Load & {from: alpine.#Image}, + op.#Exec & { + always: true + args: ["sh", "-c", """ + tr -dc A-Za-z0-9 /id + """] + }, + op.#Export & {source: "/id"}, + ] +} + +// If rand is executed twice, cue won't validate +ref1: rand +ref2: ref1 diff --git a/tests/test-compute.sh b/tests/test-compute.sh index 324d9a64..7d5c22e2 100644 --- a/tests/test-compute.sh +++ b/tests/test-compute.sh @@ -35,6 +35,8 @@ test::compute::simple(){ "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/compute/success/overload/flat test::one "Compute: overloading #Component should work" --exit=0 \ "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/compute/success/overload/wrapped + test::one "Compute: Disabling the cache on exec should not compute twice when referenced by another key" --exit=0 \ + "$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/compute/success/exec-nocache } test::compute::dependencies(){ From 3a195afa31b50b4f4922f071efbb5b76de77a84a Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 15 Apr 2021 12:12:01 -0700 Subject: [PATCH 3/5] stdlib/aws/elb: RulePriority compute can now safely bypass the cache Signed-off-by: Sam Alba --- stdlib/aws/elb/elb.cue | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stdlib/aws/elb/elb.cue b/stdlib/aws/elb/elb.cue index c5077065..cd6a0851 100644 --- a/stdlib/aws/elb/elb.cue +++ b/stdlib/aws/elb/elb.cue @@ -19,9 +19,7 @@ import ( out: string aws.#Script & { - // FIXME: we should not rely on the cache for this but it's being - // executed several times if enabled: https://github.com/dagger/dagger/issues/42 - // always: true + always: true files: { "/inputs/listenerArn": listenerArn From 468ac1220a89d9d6c83bc519fc95d304d8d2abd9 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 15 Apr 2021 12:34:15 -0700 Subject: [PATCH 4/5] added comment about `always` flag + cue fmt Signed-off-by: Sam Alba --- dagger/pipeline.go | 2 ++ stdlib/dagger/op/op.cue | 1 + tests/compute/success/exec-nocache/main.cue | 26 ++++++++++----------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/dagger/pipeline.go b/dagger/pipeline.go index e90b7f6d..ca1e6876 100644 --- a/dagger/pipeline.go +++ b/dagger/pipeline.go @@ -386,6 +386,8 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value, st llb.State) ( // always? if cmd.Always { + // FIXME: also disables persistent cache directories + // There's an ongoing proposal that would fix this: https://github.com/moby/buildkit/issues/1213 opts = append(opts, llb.IgnoreCache) } // mounts diff --git a/stdlib/dagger/op/op.cue b/stdlib/dagger/op/op.cue index 6bdcc8a4..92ec4c53 100644 --- a/stdlib/dagger/op/op.cue +++ b/stdlib/dagger/op/op.cue @@ -45,6 +45,7 @@ package op do: "exec" args: [...string] env?: [string]: string + // `true` means also ignoring the mount cache volumes always?: true | *false dir: string | *"/" mount: [string]: "tmpfs" | "cache" | {from: _, path: string | *"/"} diff --git a/tests/compute/success/exec-nocache/main.cue b/tests/compute/success/exec-nocache/main.cue index 0a4152e8..fb9eef6b 100644 --- a/tests/compute/success/exec-nocache/main.cue +++ b/tests/compute/success/exec-nocache/main.cue @@ -1,23 +1,23 @@ package main import ( - "dagger.io/alpine" - "dagger.io/dagger/op" + "dagger.io/alpine" + "dagger.io/dagger/op" ) rand: { - string + string - #up: [ - op.#Load & {from: alpine.#Image}, - op.#Exec & { - always: true - args: ["sh", "-c", """ - tr -dc A-Za-z0-9 /id - """] - }, - op.#Export & {source: "/id"}, - ] + #up: [ + op.#Load & {from: alpine.#Image}, + op.#Exec & { + always: true + args: ["sh", "-c", """ + tr -dc A-Za-z0-9 /id + """] + }, + op.#Export & {source: "/id"}, + ] } // If rand is executed twice, cue won't validate From d8a632064b7dd4cce9444213fdd92cf8ef529c6e Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 15 Apr 2021 12:37:17 -0700 Subject: [PATCH 5/5] removed dead code Signed-off-by: Sam Alba --- dagger/utils.go | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 dagger/utils.go diff --git a/dagger/utils.go b/dagger/utils.go deleted file mode 100644 index 211516c8..00000000 --- a/dagger/utils.go +++ /dev/null @@ -1,15 +0,0 @@ -package dagger - -import ( - "crypto/rand" - "fmt" -) - -func randomID(size int) (string, error) { - b := make([]byte, size) - _, err := rand.Read(b) - if err != nil { - return "", err - } - return fmt.Sprintf("%x", b), nil -}