From 089d464802ad911a4d03624113492c06a2d3076e Mon Sep 17 00:00:00 2001 From: Gerhard Lazu Date: Tue, 1 Mar 2022 19:17:13 +0000 Subject: [PATCH 1/2] Extract yarn install into a separate step If multiple yarn.#Run commands run in parallel, they will corrupt each other's yarn cache mount. Because we extract yarn install into a separate step, LLB will dedup the yarn install step and only run it once, regardless how many yarn.#Run commands run in parallel. Fixes https://github.com/dagger/dagger/issues/1670 Signed-off-by: Gerhard Lazu --- pkg/universe.dagger.io/yarn/yarn.cue | 140 ++++++++++++++++----------- 1 file changed, 81 insertions(+), 59 deletions(-) diff --git a/pkg/universe.dagger.io/yarn/yarn.cue b/pkg/universe.dagger.io/yarn/yarn.cue index a4ff1d15..1f934f9d 100644 --- a/pkg/universe.dagger.io/yarn/yarn.cue +++ b/pkg/universe.dagger.io/yarn/yarn.cue @@ -16,7 +16,7 @@ import ( script: *"build" | string } -// Build a Yarn package +// Run a Yarn command #Run: { // Custom name for the build. // When building different apps in the same plan, assign @@ -50,71 +50,93 @@ import ( // FIXME: not implemented. Are they needed? secrets: [string]: dagger.#Secret - // FIXME: Yarn's version depends on Alpine's version - // Yarn version - // yarnVersion: *"=~1.22" | string - - // FIXME: custom base image not supported - _buildImage: alpine.#Build & { - packages: { - bash: {} - yarn: {} - } - } - - // Run yarn in a docker container - container: bash.#Run & { - input: *_buildImage.output | docker.#Image - - // FIXME: move shell script to its own file - script: contents: #""" - # Create $ENVFILE_NAME file if set - [ -n "$ENVFILE_NAME" ] && echo "$ENVFILE" > "$ENVFILE_NAME" - - yarn --cwd "$YARN_CWD" install --production false - - opts=( $(echo $YARN_ARGS) ) - yarn --cwd "$YARN_CWD" run "$YARN_BUILD_SCRIPT" ${opts[@]} - if [ ! -z "${YARN_BUILD_DIRECTORY:-}" ]; then - mv "$YARN_BUILD_DIRECTORY" /build - else - mkdir /build - fi - """# - - mounts: { - "yarn cache": { - dest: "/cache/yarn" - contents: dagger.#CacheDir & { - // FIXME: are there character limitations in cache ID? - id: "universe.dagger.io/yarn.#Build \(name)" + _build: docker.#Build & { + steps: [ + // FIXME: Yarn's version depends on Alpine's version + // Yarn version + // yarnVersion: *"=~1.22" | string + // FIXME: custom base image not supported + alpine.#Build & { + packages: { + bash: {} + yarn: {} } - } - "package source": { + }, + + docker.#Copy & { dest: "/src" contents: source - } - } + }, - export: directories: "/build": _ + bash.#Run & { + // FIXME: move shell script to its own file + script: contents: #""" + yarn --cwd "$YARN_CWD" install --production false + """# - env: { - YARN_BUILD_SCRIPT: yarnScript - YARN_ARGS: strings.Join(args, "\n") - YARN_CACHE_FOLDER: "/cache/yarn" - YARN_CWD: cwd - if buildDir != _|_ { - YARN_BUILD_DIRECTORY: buildDir - } - if writeEnvFile != "" { - ENVFILE_NAME: writeEnvFile - ENVFILE: strings.Join([ for k, v in env {"\(k)=\(v)"}], "\n") - } - } + mounts: "yarn cache": { + dest: "/cache/yarn" + contents: dagger.#CacheDir & { + // FIXME: are there character limitations in cache ID? + id: "universe.dagger.io/yarn.#Run \(name)" + } + } - workdir: "/src" + env: { + YARN_CACHE_FOLDER: "/cache/yarn" + YARN_CWD: cwd + } + + workdir: "/src" + }, + + bash.#Run & { + // FIXME: move shell script to its own file + script: contents: #""" + # Create $ENVFILE_NAME file if set + [ -n "$ENVFILE_NAME" ] && echo "$ENVFILE" > "$ENVFILE_NAME" + + opts=( $(echo $YARN_ARGS) ) + yarn --cwd "$YARN_CWD" run "$YARN_BUILD_SCRIPT" ${opts[@]} + if [ ! -z "${YARN_BUILD_DIRECTORY:-}" ]; then + mv "$YARN_BUILD_DIRECTORY" /build + else + mkdir /build + fi + """# + + mounts: "yarn cache": { + dest: "/cache/yarn" + contents: dagger.#CacheDir & { + // FIXME: are there character limitations in cache ID? + id: "universe.dagger.io/yarn.#Run \(name)" + } + } + + env: { + YARN_BUILD_SCRIPT: yarnScript + YARN_ARGS: strings.Join(args, "\n") + YARN_CACHE_FOLDER: "/cache/yarn" + YARN_CWD: cwd + if buildDir != _|_ { + YARN_BUILD_DIRECTORY: buildDir + } + if writeEnvFile != "" { + ENVFILE_NAME: writeEnvFile + ENVFILE: strings.Join([ for k, v in env {"\(k)=\(v)"}], "\n") + } + } + + workdir: "/src" + }, + ] } // The final contents of the package after build - output: container.export.directories."/build" + _output: dagger.#Subdir & { + input: _build.output.rootfs + path: "/build" + } + + output: _output.output } From 26d24e6ac00d23a9c0ef411dc4d755d11da5acd5 Mon Sep 17 00:00:00 2001 From: Gerhard Lazu Date: Thu, 3 Mar 2022 12:39:32 +0000 Subject: [PATCH 2/2] First attempt at restoring the container.input interface The goal is to preserve the container.input interface so that custom images can be specified, as per yarn/test/test.cue while keeping the initial fix for https://github.com/dagger/dagger/issues/1670 Signed-off-by: Gerhard Lazu --- pkg/universe.dagger.io/yarn/test/test.cue | 2 +- pkg/universe.dagger.io/yarn/yarn.cue | 48 ++++++++++++----------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/pkg/universe.dagger.io/yarn/test/test.cue b/pkg/universe.dagger.io/yarn/test/test.cue index 574523d5..2c822339 100644 --- a/pkg/universe.dagger.io/yarn/test/test.cue +++ b/pkg/universe.dagger.io/yarn/test/test.cue @@ -66,7 +66,7 @@ dagger.#Plan & { build: yarn.#Build & { source: common.data - container: input: buildImage.output + container: #input: buildImage.output } } } diff --git a/pkg/universe.dagger.io/yarn/yarn.cue b/pkg/universe.dagger.io/yarn/yarn.cue index 1f934f9d..d3d9c9b9 100644 --- a/pkg/universe.dagger.io/yarn/yarn.cue +++ b/pkg/universe.dagger.io/yarn/yarn.cue @@ -18,26 +18,24 @@ import ( // Run a Yarn command #Run: { - // Custom name for the build. - // When building different apps in the same plan, assign - // different names for optimal caching. + // Custom name for this command. + // Assign an app-specific name if there are multiple apps in the same plan. name: string | *"" - // Application source code + // App source code source: dagger.#FS - // working directory to use + // Working directory to use cwd: *"." | string - // Write the contents of `environment` to this file, - // in the "envfile" format + // Write the contents of `environment` to this file, in the "envfile" format writeEnvFile: string | *"" - // Read build output from this directory - // (path must be relative to working directory) + // Optional: Read build output from this directory + // Must be relative to working directory, cwd buildDir?: string - // Run this yarn script + // Yarn script to run for this command. script: string // Fix for shadowing issues @@ -50,18 +48,22 @@ import ( // FIXME: not implemented. Are they needed? secrets: [string]: dagger.#Secret - _build: docker.#Build & { + container: #input: docker.#Image | *{ + // FIXME: Yarn's version depends on Alpine's version + // Yarn version + // yarnVersion: *"=~1.22" | string + // FIXME: custom base image not supported + alpine.#Build & { + packages: { + bash: {} + yarn: {} + } + } + } + + _run: docker.#Build & { steps: [ - // FIXME: Yarn's version depends on Alpine's version - // Yarn version - // yarnVersion: *"=~1.22" | string - // FIXME: custom base image not supported - alpine.#Build & { - packages: { - bash: {} - yarn: {} - } - }, + container.#input, docker.#Copy & { dest: "/src" @@ -132,9 +134,9 @@ import ( ] } - // The final contents of the package after build + // The final contents of the package after run _output: dagger.#Subdir & { - input: _build.output.rootfs + input: _run.output.rootfs path: "/build" }