From 98b3951c7386a8bb62d8a986b9d27a4940367a3d Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 6 May 2021 06:53:04 +0000 Subject: [PATCH] stdlib: os package Signed-off-by: Solomon Hykes --- stdlib/dagger/dagger.cue | 10 +-- stdlib/dagger/op/op.cue | 5 +- stdlib/docker/{image.cue => docker.cue} | 19 +++-- stdlib/go/go.cue | 9 +-- stdlib/io/io.cue | 1 - stdlib/js/react/react.cue | 75 ++++++++--------- stdlib/netlify/netlify.cue | 89 +++++++++++---------- stdlib/netlify/netlify.sh.cue | 13 +-- stdlib/{docker => os}/container.cue | 102 ++++++++---------------- stdlib/os/{os.cue => dir.cue} | 47 +++-------- stdlib/os/file.cue | 36 +++++++++ tests/stdlib/js/react/react.cue | 19 +---- 12 files changed, 193 insertions(+), 232 deletions(-) rename stdlib/docker/{image.cue => docker.cue} (66%) rename stdlib/{docker => os}/container.cue (54%) rename stdlib/os/{os.cue => dir.cue} (58%) create mode 100644 stdlib/os/file.cue diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index 8fa8571c..2f66e734 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -1,16 +1,14 @@ package dagger import ( - "dagger.io/dagger/op" ) +// "dagger.io/dagger/op" // An artifact such as source code checkout, container image, binary archive... // May be passed as user input, or computed by a buildkit pipeline -#Artifact: { - #up: [...op.#Op] - _ - ... -} + +// FIXME (perf). See https://github.com/dagger/dagger/issues/445 +#Artifact: _ // Secret value // FIXME: currently aliased as a string to mark secrets diff --git a/stdlib/dagger/op/op.cue b/stdlib/dagger/op/op.cue index 278259c1..736be630 100644 --- a/stdlib/dagger/op/op.cue +++ b/stdlib/dagger/op/op.cue @@ -48,7 +48,10 @@ package op // `true` means also ignoring the mount cache volumes always?: true | *false dir: string | *"/" - mount: [string]: "tmpfs" | "cache" | {from: _, path: string | *"/"} + // FIXME (perf): complex schema in low-level ops causes explosive perf issues + // see https://github.com/dagger/dagger/issues/445 + // mount: [string]: "tmpfs" | "cache" | {from: _, path: string | *"/"} + mount: [string]: _ } #DockerLogin: { diff --git a/stdlib/docker/image.cue b/stdlib/docker/docker.cue similarity index 66% rename from stdlib/docker/image.cue rename to stdlib/docker/docker.cue index 7b65b32c..15c04efd 100644 --- a/stdlib/docker/image.cue +++ b/stdlib/docker/docker.cue @@ -17,22 +17,21 @@ import ( } -// Build a Docker image from source, using included Dockerfile -// FIXME: DEPRECATED by #Build -#ImageFromSource: #Build - -// Fetch an image from a remote registry -#ImageFromRegistry: { - ref: string +// Pull a docker container +#Pull: { + // Remote ref (example: "index.docker.io/alpine:latest") + from: string #up: [ - op.#FetchContainer & { - "ref": ref - }, + op.#FetchContainer & {ref: from}, ] } +// FIXME: #Push +// FIXME: #Run + // Build a Docker image from the provided Dockerfile contents +// FIXME: incorporate into #Build #ImageFromDockerfile: { dockerfile: string context: dagger.#Artifact diff --git a/stdlib/go/go.cue b/stdlib/go/go.cue index b1315d6b..dbd72114 100644 --- a/stdlib/go/go.cue +++ b/stdlib/go/go.cue @@ -4,7 +4,7 @@ import ( "dagger.io/dagger" "dagger.io/dagger/op" - "dagger.io/docker" + "dagger.io/os" ) // A standalone go environment @@ -13,15 +13,14 @@ import ( version: *"1.16" | string source: dagger.#Artifact - docker.#Container & { + os.#Container & { env: { GOMODCACHE: volume.goCache.dest CGO_ENABLED: "0" } - image: docker.#ImageFromRegistry & { - ref: "docker.io/golang:\(version)-alpine" - } + from: "docker.io/golang:\(version)-alpine" + volume: { goSource: { from: source diff --git a/stdlib/io/io.cue b/stdlib/io/io.cue index 3c142ab0..5804cb49 100644 --- a/stdlib/io/io.cue +++ b/stdlib/io/io.cue @@ -21,7 +21,6 @@ package io format: "string" | "json" | "yaml" | "lines" data: { string - ... } } ... diff --git a/stdlib/js/react/react.cue b/stdlib/js/react/react.cue index a3fbc74e..03c58475 100644 --- a/stdlib/js/react/react.cue +++ b/stdlib/js/react/react.cue @@ -4,11 +4,12 @@ import ( "strings" "dagger.io/dagger" - "dagger.io/dagger/op" "dagger.io/alpine" + "dagger.io/os" ) // A ReactJS application +// FIXME: move this to a 'yarn' package for clarity #App: { // Application source code source: dagger.#Artifact @@ -30,48 +31,36 @@ import ( script: string | *"build" } - build: #up: [ - op.#Load & { - from: alpine.#Image & { - package: bash: "=~5.1" - package: yarn: "=~1.22" + build: os.#Dir & { + from: ctr + path: "/build" + } + + ctr: os.#Container & { + image: alpine.#Image & { + package: { + bash: "=~5.1" + yarn: "=~1.22" } - }, - op.#Exec & { - args: [ - "/bin/bash", - "--noprofile", - "--norc", - "-eo", - "pipefail", - "-c", - """ - [ -n "$ENVFILE_NAME" ] && echo "$ENVFILE" > "$ENVFILE_NAME" - yarn install --production false - yarn run "$YARN_BUILD_SCRIPT" - mv "$YARN_BUILD_DIRECTORY" /build - """, - ] - if env != _|_ { - "env": env + } + shell: path: "/bin/bash" + command: """ + [ -n "$ENVFILE_NAME" ] && echo "$ENVFILE" > "$ENVFILE_NAME" + yarn install --production false + yarn run "$YARN_BUILD_SCRIPT" + mv "$YARN_BUILD_DIRECTORY" /build + """ + "env": env & { + YARN_BUILD_SCRIPT: yarn.script + YARN_CACHE_FOLDER: "/cache/yarn" + YARN_BUILD_DIRECTORY: yarn.buildDir + if writeEnvFile != "" { + ENVFILE_NAME: writeEnvFile + ENVFILE: strings.Join([ for k, v in env {"\(k)=\(v)"}], "\n") } - "env": { - YARN_BUILD_SCRIPT: yarn.script - YARN_CACHE_FOLDER: "/cache/yarn" - YARN_BUILD_DIRECTORY: yarn.buildDir - if writeEnvFile != "" { - ENVFILE_NAME: writeEnvFile - ENVFILE: strings.Join([ for k, v in env {"\(k)=\(v)"}], "\n") - } - } - dir: "/src" - mount: { - "/src": from: source - "/cache/yarn": "cache" - } - }, - op.#Subdir & { - dir: "/build" - }, - ] + } + dir: "/src" + mount: "/src": from: source + cache: "/cache/yarn": true + } } diff --git a/stdlib/netlify/netlify.cue b/stdlib/netlify/netlify.cue index 04d41761..be977f5e 100644 --- a/stdlib/netlify/netlify.cue +++ b/stdlib/netlify/netlify.cue @@ -3,7 +3,7 @@ package netlify import ( "dagger.io/dagger" "dagger.io/alpine" - "dagger.io/dagger/op" + "dagger.io/os" ) // A Netlify account @@ -34,53 +34,56 @@ import ( create: bool | *true // Website url - url: string + url: { + os.#File & { + from: ctr + path: "/netlify/url" + } + }.read.data // Unique Deploy URL - deployUrl: string + deployUrl: { + os.#File & { + from: ctr + path: "/netlify/deployUrl" + } + }.read.data // Logs URL for this deployment - logsUrl: string + logsUrl: { + os.#File & { + from: ctr + path: "/netlify/logsUrl" + } + }.read.data - #up: [ - op.#Load & { - from: alpine.#Image & { - package: bash: "=~5.1" - package: jq: "=~1.6" - package: curl: "=~7.76" - package: yarn: "=~1.22" + ctr: os.#Container & { + image: alpine.#Image & { + package: { + bash: "=~5.1" + jq: "=~1.6" + curl: "=~7.76" + yarn: "=~1.22" } - }, - op.#Exec & { - args: ["yarn", "global", "add", "netlify-cli@2.47.0"] - }, - op.#Exec & { - args: [ - "/bin/bash", - "--noprofile", - "--norc", - "-eo", - "pipefail", - "-c", - #code, - ] - env: { - NETLIFY_SITE_NAME: name - if (create) { - NETLIFY_SITE_CREATE: "1" - } - if customDomain != _|_ { - NETLIFY_DOMAIN: customDomain - } - NETLIFY_ACCOUNT: account.name - NETLIFY_AUTH_TOKEN: account.token + } + setup: [ + "yarn global add netlify-cli@2.47.0", + ] + // set in netlify.sh.cue + // FIXME: use embedding once cue supports it + command: _ + env: { + NETLIFY_SITE_NAME: name + if (create) { + NETLIFY_SITE_CREATE: "1" } - dir: "/src" - mount: "/src": from: contents - }, - op.#Export & { - source: "/output.json" - format: "json" - }, - ] + if customDomain != _|_ { + NETLIFY_DOMAIN: customDomain + } + NETLIFY_ACCOUNT: account.name + NETLIFY_AUTH_TOKEN: account.token + } + dir: "/src" + mount: "/src": from: contents + } } diff --git a/stdlib/netlify/netlify.sh.cue b/stdlib/netlify/netlify.sh.cue index 34d89ff1..2130de71 100644 --- a/stdlib/netlify/netlify.sh.cue +++ b/stdlib/netlify/netlify.sh.cue @@ -1,6 +1,6 @@ package netlify -#code: #""" +#Site: ctr: command: #""" create_site() { url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites" @@ -41,9 +41,10 @@ package netlify deployUrl=$( /output.json + # Write output files + mkdir -p /netlify + echo "$url" > /netlify/url + echo "$deployUrl" > /netlify/deployUrl + echo "$logsUrl" > /netlify/logsUrl + """# diff --git a/stdlib/docker/container.cue b/stdlib/os/container.cue similarity index 54% rename from stdlib/docker/container.cue rename to stdlib/os/container.cue index a5fefd44..92fddc52 100644 --- a/stdlib/docker/container.cue +++ b/stdlib/os/container.cue @@ -1,7 +1,4 @@ -// docker: build and run Docker containers -// https://docker.com - -package docker +package os import ( "strings" @@ -18,11 +15,17 @@ import ( // see https://github.com/dagger/dagger/issues/304 #DefaultImage: alpine.#Image.#up -// Run a Docker container +// Built-in container implementation, using buildkit #Container: { // Container image image: dagger.#Artifact | *#DefaultImage + // { + // // Optionally fetch a remote image + // // eg. "index.docker.io/alpine" + // from: string + // image: #up: [op.#FetchContainer & { "ref": from }] + // } | {} // Independently cacheable setup commands setup: [...string] @@ -36,43 +39,28 @@ import ( // Directory in which the command is executed dir: string | *"/" - // Directory to expose as the output. - // By default the root filesystem is the output. - outputDir: string | *"/" - // If true, the command is never cached. // (false by default). always: true | *false - // External volumes. There are 4 types: - // - // 1. "mount": mount any artifact. - // Changes are not included in the final output. - // - // 2. "copy": copy any artifact. - // Changes are included in the final output. - // - // 3. "tmpfs": create a temporary directory. - // - // 4. "cache": create a persistent cache diretory. - // - volume: [name=string]: { - // Destination path - dest: string | *"/" + // Copy contents from other artifacts + copy: [string]: from: dagger.#Artifact - *{ - type: "mount" - from: dagger.#Artifact - source: string | *"/" - } | { - type: "copy" - from: dagger.#Artifact - source: string | *"/" - } | { - type: "tmpfs" | "cache" - } + // Mount contents from other artifacts. + // Mount is active when executing `command`, but not `setup`. + + mount: [string]: { + // FIXME(perf) should be #Artifact. See https://github.com/dagger/dagger/issues/445 + from: {...} + // FIXME: support source path } + // Mount persistent cache directories + cache: [string]: true + + // Mount temporary directories + tmpfs: [string]: true + // Configure the shell which executes all commands. shell: { // Path of the shell to execute @@ -94,20 +82,14 @@ import ( } env: PATH: string | *strings.Join([ for p, v in shell.search if v {p}], ":") - // Export values from the container to the cue configuration - export: *null | { - source: string - format: op.#Export.format - } - #up: [ op.#Load & {from: image}, // Copy volumes with type=copy - for _, v in volume if v.type == "copy" { + for dest, o in copy { op.#Copy & { - from: v.from - dest: v.dest - src: v.source + "dest": dest + from: o.from + // FIXME: support source path } }, // Execute setup commands, without volumes @@ -126,33 +108,19 @@ import ( "env": env "dir": dir "always": always - mount: { - // FIXME: fix perf issue - for _, v in volume if v.type == "cache" { - "\(v.dest)": "cache" + "mount": { + for dest, o in mount { + "\(dest)": from: o.from + // FIXME: support source path } - for _, v in volume if v.type == "tmpfs" { - "\(v.dest)": "tmpfs" + for dest in cache { + "\(dest)": "cache" } - for _, v in volume if v.type == "mount" { - "\(v.dest)": { - from: v.from - path: v.source - } + for dest in tmpfs { + "\(dest)": "tmpfs" } } } }, - // FIXME: is subdir deprecated by dagger.io/io.#Dir ? - op.#Subdir & { - dir: outputDir - }, - // FIXME: is export deprecated by dagger.io/io.#File ? - if export != null { - op.#Export & { - source: export.source - format: export.format - } - }, ] } diff --git a/stdlib/os/os.cue b/stdlib/os/dir.cue similarity index 58% rename from stdlib/os/os.cue rename to stdlib/os/dir.cue index 09360ee0..ed47aaa4 100644 --- a/stdlib/os/os.cue +++ b/stdlib/os/dir.cue @@ -1,13 +1,23 @@ package os import ( - "dagger.io/io" - "dagger.io/dagger" "dagger.io/dagger/op" ) -#Dir: io.#Dir & { +#Dir: { + from: dagger.#Artifact + path: string | *"/" + + #up: [ + op.#Load & {"from": from}, + op.#Subdir & { + dir: path + }, + ] +} + +#ReadDir: { from: dagger.#Artifact path: string @@ -32,34 +42,3 @@ import ( ] } } - -#File: { - from: dagger.#Artifact - path: string - - io.#Reader & { - read: { - format: _ - data: { - _ - #up: [ - op.#Load & {"from": from}, - op.#Export & {source: path, "format": format}, - ] - } - } - } - - io.#Writer & { - write: *null | { - data: _ - #up: [ - op.#Load & {"from": from}, - op.#WriteFile & { - dest: path - contents: data - }, - ] - } - } -} diff --git a/stdlib/os/file.cue b/stdlib/os/file.cue new file mode 100644 index 00000000..b484374f --- /dev/null +++ b/stdlib/os/file.cue @@ -0,0 +1,36 @@ +package os + +import ( + "dagger.io/dagger" + "dagger.io/dagger/op" +) + +// Built-in file implementation, using buildkit +#File: { + from: dagger.#Artifact + path: string + + read: { + // FIXME: support different data schemas for different formats + format: "string" + data: { + string + #up: [ + op.#Load & {"from": from}, + op.#Export & {source: path, "format": format}, + ] + } + } + + write: *null | { + // FIXME: support encoding in different formats + data: string + #up: [ + op.#Load & {"from": from}, + op.#WriteFile & { + dest: path + contents: data + }, + ] + } +} diff --git a/tests/stdlib/js/react/react.cue b/tests/stdlib/js/react/react.cue index 5fb6530c..60244b01 100644 --- a/tests/stdlib/js/react/react.cue +++ b/tests/stdlib/js/react/react.cue @@ -4,7 +4,7 @@ import ( "dagger.io/dagger" "dagger.io/js/react" "dagger.io/alpine" - "dagger.io/docker" + "dagger.io/os" ) TestData: dagger.#Artifact @@ -14,26 +14,13 @@ TestReact: { source: TestData } - test: docker.#Container & { + test: os.#Container & { image: alpine.#Image & { package: bash: "=5.1.0-r0" } - volume: build: { - from: app.build - dest: "/build" - } + mount: "/build": from: app.build command: """ test "$(cat /build/test)" = "output" """ - shell: { - path: "/bin/bash" - args: [ - "--noprofile", - "--norc", - "-eo", - "pipefail", - "-c", - ] - } } }