From 5a1d4bff62f0f439692276405c9e1394e7a6b245 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 12 Oct 2021 13:42:17 -0700 Subject: [PATCH 1/5] Support loading artifacts into a Docker Engine This adds support to loading artifacts (e.g. docker.#Build, os.#Container, ...) into any arbitrary docker engine (through a dagger.#Stream for UNIX sockets or SSH for a remote engine) Implementation: - Add op.#SaveImage which serializes an artifact into an arbitrary path (docker tarball format) - Add docker.#Load which uses op.#SaveImage to serialize to disk and executes `docker load` to load it back Caveats: Because we're doing this in userspace rather than letting dagger itself load the image, the performance is pretty bad. The buildkit API is meant for streaming (get a stream of a docker image pipe it into docker load). Because of userspace, we have to load the entire docker image into memory, then serialize it in a single WriteFile LLB operation. Example: ```cue package main import ( "alpha.dagger.io/dagger" "alpha.dagger.io/docker" ) source: dagger.#Input & dagger.#Artifact dockersocket: dagger.#Input & dagger.#Stream build: docker.#Build & { "source": source } load: docker.#Load & { source: build tag: "testimage" socket: dockersocket } ``` Signed-off-by: Andrea Luzzardi --- docs/reference/dagger/op.md | 10 ++++ docs/reference/docker/README.md | 20 ++++++++ environment/pipeline.go | 67 +++++++++++++++++++++++++++ stdlib/dagger/op/op.cue | 6 +++ stdlib/docker/docker.cue | 81 +++++++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+) diff --git a/docs/reference/dagger/op.md b/docs/reference/dagger/op.md index 91c8953b..776ca893 100644 --- a/docs/reference/dagger/op.md +++ b/docs/reference/dagger/op.md @@ -144,6 +144,16 @@ _No input._ _No output._ +## op.#SaveImage + +### op.#SaveImage Inputs + +_No input._ + +### op.#SaveImage Outputs + +_No output._ + ## op.#Subdir ### op.#Subdir Inputs diff --git a/docs/reference/docker/README.md b/docs/reference/docker/README.md index d96f3365..585a3bbb 100644 --- a/docs/reference/docker/README.md +++ b/docs/reference/docker/README.md @@ -40,6 +40,26 @@ A container image that can run any docker command _No output._ +## docker.#Load + +Load a docker image into a docker engine + +### docker.#Load Inputs + +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*tag* | `string` |Name and optionally a tag in the 'name:tag' format | +|*source* | `dagger.#Artifact` |Image source | +|*load.command* | `"docker load -i /src/image.tar"` |Command to execute | +|*load.registries* | `[]` |Image registries | + +### docker.#Load Outputs + +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*ref* | `string` |Image ref | +|*digest* | `string` |Image digest | + ## docker.#Pull Pull a docker container diff --git a/environment/pipeline.go b/environment/pipeline.go index f3ab066a..1f7854e3 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "io/fs" "net" "net/url" @@ -261,6 +262,8 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value, st llb.State) ( return p.FetchContainer(ctx, op, st) case "push-container": return p.PushContainer(ctx, op, st) + case "save-image": + return p.SaveImage(ctx, op, st) case "fetch-git": return p.FetchGit(ctx, op, st) case "fetch-http": @@ -872,6 +875,70 @@ func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value, st llb return st, err } +func (p *Pipeline) SaveImage(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { + tag, err := op.Lookup("tag").String() + if err != nil { + return st, err + } + + dest, err := op.Lookup("dest").String() + if err != nil { + return st, err + } + + pipeR, pipeW := io.Pipe() + var ( + errCh = make(chan error) + image []byte + ) + go func() { + image, err = io.ReadAll(pipeR) + errCh <- err + }() + + resp, err := p.s.Export(ctx, p.State(), &p.image, bk.ExportEntry{ + Type: bk.ExporterDocker, + Attrs: map[string]string{ + "name": tag, + }, + Output: func(_ map[string]string) (io.WriteCloser, error) { + return pipeW, nil + }, + }) + + if err != nil { + return st, err + } + + if err := <-errCh; err != nil { + return st, err + } + + if digest, ok := resp.ExporterResponse["containerimage.digest"]; ok { + imageRef := fmt.Sprintf( + "%s@%s", + resp.ExporterResponse["image.name"], + digest, + ) + + st = st.File( + llb.Mkdir("/dagger", fs.FileMode(0755)), + llb.WithCustomName(p.vertexNamef("Mkdir /dagger")), + ).File( + llb.Mkfile("/dagger/image_digest", fs.FileMode(0644), []byte(digest)), + llb.WithCustomName(p.vertexNamef("Storing image digest to /dagger/image_digest")), + ).File( + llb.Mkfile("/dagger/image_ref", fs.FileMode(0644), []byte(imageRef)), + llb.WithCustomName(p.vertexNamef("Storing image ref to /dagger/image_ref")), + ) + } + + return st.File( + llb.Mkfile(dest, 0644, image), + llb.WithCustomName(p.vertexNamef("SaveImage %s", dest)), + ), nil +} + func getSecretID(secretField *compiler.Value) (string, error) { if !secretField.HasAttr("secret") { return "", fmt.Errorf("invalid secret %q: not a secret", secretField.Path().String()) diff --git a/stdlib/dagger/op/op.cue b/stdlib/dagger/op/op.cue index 81cee4c8..cf4d62c0 100644 --- a/stdlib/dagger/op/op.cue +++ b/stdlib/dagger/op/op.cue @@ -81,6 +81,12 @@ package op ref: string } +#SaveImage: { + do: "save-image" + tag: string + dest: string +} + #FetchGit: { do: "fetch-git" remote: string diff --git a/stdlib/docker/docker.cue b/stdlib/docker/docker.cue index 0236ed8b..198a12b3 100644 --- a/stdlib/docker/docker.cue +++ b/stdlib/docker/docker.cue @@ -101,6 +101,87 @@ import ( } & dagger.#Output } +// Load a docker image into a docker engine +#Load: { + // Connect to a remote SSH server + ssh?: { + // ssh host + host: dagger.#Input & {string} + + // ssh user + user: dagger.#Input & {string} + + // ssh port + port: dagger.#Input & {*22 | int} + + // private key + key: dagger.#Input & {dagger.#Secret} + + // fingerprint + fingerprint?: dagger.#Input & {string} + + // ssh key passphrase + keyPassphrase?: dagger.#Input & {dagger.#Secret} + } + + // Mount local docker socket + socket?: dagger.#Stream & dagger.#Input + + // Name and optionally a tag in the 'name:tag' format + tag: dagger.#Input & {string} + + // Image source + source: dagger.#Input & {dagger.#Artifact} + + save: #up: [ + op.#Load & {from: source}, + + op.#SaveImage & { + "tag": tag + dest: "/image.tar" + }, + ] + + load: #Command & { + if ssh != _|_ { + "ssh": ssh + } + if socket != _|_ { + "socket": socket + } + + copy: "/src": from: save + + command: "docker load -i /src/image.tar" + } + + // Image ref + ref: { + string + + #up: [ + op.#Load & {from: save}, + + op.#Export & { + source: "/dagger/image_ref" + }, + ] + } & dagger.#Output + + // Image digest + digest: { + string + + #up: [ + op.#Load & {from: save}, + + op.#Export & { + source: "/dagger/image_digest" + }, + ] + } & dagger.#Output +} + #Run: { // Connect to a remote SSH server ssh?: { From 50d55a8885495737db892526241b7a9fad2366cc Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 12 Oct 2021 15:37:01 -0700 Subject: [PATCH 2/5] op.#SaveImage: export the image ID Signed-off-by: Andrea Luzzardi --- docs/reference/docker/README.md | 3 +-- environment/pipeline.go | 15 +++------------ stdlib/docker/docker.cue | 19 +++---------------- 3 files changed, 7 insertions(+), 30 deletions(-) diff --git a/docs/reference/docker/README.md b/docs/reference/docker/README.md index 585a3bbb..fef82bae 100644 --- a/docs/reference/docker/README.md +++ b/docs/reference/docker/README.md @@ -57,8 +57,7 @@ Load a docker image into a docker engine | Name | Type | Description | | ------------- |:-------------: |:-------------: | -|*ref* | `string` |Image ref | -|*digest* | `string` |Image digest | +|*id* | `string` |Image ID | ## docker.#Pull diff --git a/environment/pipeline.go b/environment/pipeline.go index 1f7854e3..ee751939 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -914,22 +914,13 @@ func (p *Pipeline) SaveImage(ctx context.Context, op *compiler.Value, st llb.Sta return st, err } - if digest, ok := resp.ExporterResponse["containerimage.digest"]; ok { - imageRef := fmt.Sprintf( - "%s@%s", - resp.ExporterResponse["image.name"], - digest, - ) - + if id, ok := resp.ExporterResponse["containerimage.config.digest"]; ok { st = st.File( llb.Mkdir("/dagger", fs.FileMode(0755)), llb.WithCustomName(p.vertexNamef("Mkdir /dagger")), ).File( - llb.Mkfile("/dagger/image_digest", fs.FileMode(0644), []byte(digest)), - llb.WithCustomName(p.vertexNamef("Storing image digest to /dagger/image_digest")), - ).File( - llb.Mkfile("/dagger/image_ref", fs.FileMode(0644), []byte(imageRef)), - llb.WithCustomName(p.vertexNamef("Storing image ref to /dagger/image_ref")), + llb.Mkfile("/dagger/image_id", fs.FileMode(0644), []byte(id)), + llb.WithCustomName(p.vertexNamef("Storing image id to /dagger/image_id")), ) } diff --git a/stdlib/docker/docker.cue b/stdlib/docker/docker.cue index 198a12b3..b2c9882b 100644 --- a/stdlib/docker/docker.cue +++ b/stdlib/docker/docker.cue @@ -155,28 +155,15 @@ import ( command: "docker load -i /src/image.tar" } - // Image ref - ref: { + // Image ID + id: { string #up: [ op.#Load & {from: save}, op.#Export & { - source: "/dagger/image_ref" - }, - ] - } & dagger.#Output - - // Image digest - digest: { - string - - #up: [ - op.#Load & {from: save}, - - op.#Export & { - source: "/dagger/image_digest" + source: "/dagger/image_id" }, ] } & dagger.#Output From b2c4fea73dcf2d90316ab49ae739efb0e9b323ef Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 12 Oct 2021 15:38:09 -0700 Subject: [PATCH 3/5] tests: add docker load test Signed-off-by: Andrea Luzzardi --- stdlib/.dagger/env/docker-load/.gitignore | 2 ++ stdlib/.dagger/env/docker-load/values.yaml | 30 ++++++++++++++++++ stdlib/docker/tests/load/load.cue | 32 ++++++++++++++++++++ stdlib/docker/tests/load/testdata/Dockerfile | 2 ++ stdlib/universe.bats | 6 +++- 5 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 stdlib/.dagger/env/docker-load/.gitignore create mode 100644 stdlib/.dagger/env/docker-load/values.yaml create mode 100644 stdlib/docker/tests/load/load.cue create mode 100644 stdlib/docker/tests/load/testdata/Dockerfile diff --git a/stdlib/.dagger/env/docker-load/.gitignore b/stdlib/.dagger/env/docker-load/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/stdlib/.dagger/env/docker-load/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/stdlib/.dagger/env/docker-load/values.yaml b/stdlib/.dagger/env/docker-load/values.yaml new file mode 100644 index 00000000..9cc52e3a --- /dev/null +++ b/stdlib/.dagger/env/docker-load/values.yaml @@ -0,0 +1,30 @@ +plan: + package: ./docker/tests/load +name: docker-load +inputs: + dockersocket: + socket: + unix: /var/run/docker.sock + source: + dir: + path: ./docker/tests/load/testdata +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzZCt6S3VGQkM4ZU9TamhQ + c2J0RDZpdzNQZUJ6V1BBdHh3M1NhTmJLeXpFCldaOVNUYVhWQW5ma3JGUk9XZWZ2 + YjZIM0tMUGRoSk1QYTFkalA4S2N3UzAKLS0tIFhMeXMvaGM4UFllYWFCNWVUZFd5 + U09jNHNlK094NGoyZnRlSk56T1N6K1EKJs5D3S2zPWNrGCyLWDDjq7Iif0m2JoL6 + gqEjofnPSD7SjgfNKIpeOWcQ1sI7wmI4GGgaTpdhd431XxOn/fU44w== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-10-12T22:18:14Z" + mac: ENC[AES256_GCM,data:HjlY0FzB5hsg/VfyBbVTNWDCYV41lhPeyHOVrMdRWiOKJnV1aKGV1826fMXKcmgJK39kovcEXaXmVYOv3qtpDTlIMcOdoEflbDY/fhZmaDxxNq6QSkYfmadO84YWvS1FyouCPVZzRDe9tBdNyIT5cCx8CrgZ5bebh9aNem3lqRs=,iv:juMn1S06I5mZP8fWytn2eUkOvjNQepn6MAzDWvLxrWM=,tag:0BxrbK6EaoB44RgUHD+dpQ==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 diff --git a/stdlib/docker/tests/load/load.cue b/stdlib/docker/tests/load/load.cue new file mode 100644 index 00000000..bedd18d9 --- /dev/null +++ b/stdlib/docker/tests/load/load.cue @@ -0,0 +1,32 @@ +package docker + +import ( + "alpha.dagger.io/dagger" + "alpha.dagger.io/random" +) + +dockersocket: dagger.#Stream & dagger.#Input + +source: dagger.#Artifact & dagger.#Input + +TestLoad: { + suffix: random.#String & { + seed: "" + } + + image: #Build & { + "source": source + } + + load: #Load & { + tag: "daggerci-image-load-\(suffix.out)" + source: image + socket: dockersocket + } + + run: #Run & { + name: "daggerci-container-load-\(suffix.out)" + ref: load.id + socket: dockersocket + } +} diff --git a/stdlib/docker/tests/load/testdata/Dockerfile b/stdlib/docker/tests/load/testdata/Dockerfile new file mode 100644 index 00000000..987ce9be --- /dev/null +++ b/stdlib/docker/tests/load/testdata/Dockerfile @@ -0,0 +1,2 @@ +FROM alpine +RUN echo test >> /test.txt \ No newline at end of file diff --git a/stdlib/universe.bats b/stdlib/universe.bats index 0a19361e..7c11b275 100644 --- a/stdlib/universe.bats +++ b/stdlib/universe.bats @@ -126,6 +126,10 @@ setup() { assert_failure } +@test "docker load" { + dagger -e docker-load up +} + @test "docker compose" { dagger -e docker-compose up } @@ -177,7 +181,7 @@ setup() { } @test "google cloud: gke" { - dagger -e google-gke up + dagger -e google-gke up } @test "google cloud: secretmanager" { From e5c571b8a95c2a4579b06de6b04230073f48d8cb Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 12 Oct 2021 16:05:43 -0700 Subject: [PATCH 4/5] op.#SaveImage: fix data race Signed-off-by: Andrea Luzzardi --- environment/pipeline.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment/pipeline.go b/environment/pipeline.go index ee751939..e2801b0e 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -892,6 +892,8 @@ func (p *Pipeline) SaveImage(ctx context.Context, op *compiler.Value, st llb.Sta image []byte ) go func() { + var err error + image, err = io.ReadAll(pipeR) errCh <- err }() From 322997b74a054988787e80ccd09a706018cb9310 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 12 Oct 2021 17:10:46 -0700 Subject: [PATCH 5/5] stdlib: docker: clean up docker.#Command inputs Signed-off-by: Andrea Luzzardi --- docs/reference/docker/README.md | 17 +++++------------ docs/reference/docker/compose.md | 12 ++++-------- stdlib/docker/command.cue | 18 +++++++++--------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/docs/reference/docker/README.md b/docs/reference/docker/README.md index fef82bae..82724b27 100644 --- a/docs/reference/docker/README.md +++ b/docs/reference/docker/README.md @@ -31,10 +31,7 @@ A container image that can run any docker command ### docker.#Command Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*command* | `string` |Command to execute | -|*registries* | `[]` |Image registries | +_No input._ ### docker.#Command Outputs @@ -46,12 +43,10 @@ Load a docker image into a docker engine ### docker.#Load Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*tag* | `string` |Name and optionally a tag in the 'name:tag' format | -|*source* | `dagger.#Artifact` |Image source | -|*load.command* | `"docker load -i /src/image.tar"` |Command to execute | -|*load.registries* | `[]` |Image registries | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*tag* | `string` |Name and optionally a tag in the 'name:tag' format | +|*source* | `dagger.#Artifact` |Image source | ### docker.#Load Outputs @@ -98,9 +93,7 @@ Push a docker image to a remote registry | Name | Type | Description | | ------------- |:-------------: |:-------------: | |*ref* | `string` |Image reference (e.g: nginx:alpine) | -|*run.command* | `"""\n # Run detach container\n OPTS=""\n \n if [ ! -z "$CONTAINER_NAME" ]; then\n \tOPTS="$OPTS --name $CONTAINER_NAME"\n fi\n \n if [ ! -z "$CONTAINER_PORTS" ]; then\n \tOPTS="$OPTS -p $CONTAINER_PORTS"\n fi\n \n docker container run -d $OPTS "$IMAGE_REF"\n """` |Command to execute | |*run.env.IMAGE_REF* | `string` |- | -|*run.registries* | `[]` |Image registries | ### docker.#Run Outputs diff --git a/docs/reference/docker/compose.md b/docs/reference/docker/compose.md index 2a74308c..afea1ec0 100644 --- a/docs/reference/docker/compose.md +++ b/docs/reference/docker/compose.md @@ -14,14 +14,10 @@ import "alpha.dagger.io/docker/compose" ### compose.#App Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*name* | `*"source" \| string` |App name (use as COMPOSE_PROJECT_NAME) | -|*registries* | `[]` |Image registries | -|*run.command* | `"""\n if [ -n "$DOCKER_HOSTNAME" ]; then\n \tssh -i /key -fNT -o "StreamLocalBindUnlink=yes" -L "$(pwd)"/docker.sock:/var/run/docker.sock -p "$DOCKER_PORT" "$DOCKER_USERNAME"@"$DOCKER_HOSTNAME"\n \texport DOCKER_HOST="unix://$(pwd)/docker.sock"\n fi\n \n # Extend session duration\n echo "Host *\\nServerAliveInterval 240" \>\> "$HOME"/.ssh/config\n chmod 600 "$HOME"/.ssh/config\n \n # Move compose\n if [ -d "$SOURCE_DIR" ]; then\n \tif [ -f docker-compose.yaml ]; then\n \t\tcp docker-compose.yaml "$SOURCE_DIR"/docker-compose.yaml\n \tfi\n \tcd "$SOURCE_DIR"\n fi\n \n docker-compose build\n docker-compose up -d\n """` |Command to execute | -|*run.env.COMPOSE_PROJECT_NAME* | `*"source" \| string` |- | -|*run.package."docker-compose"* | `true` |- | -|*run.registries* | `[]` |Image registries | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*name* | `*"source" \| string` |App name (use as COMPOSE_PROJECT_NAME) | +|*registries* | `[]` |Image registries | ### compose.#App Outputs diff --git a/stdlib/docker/command.cue b/stdlib/docker/command.cue index ac28e152..5d69a2fa 100644 --- a/stdlib/docker/command.cue +++ b/stdlib/docker/command.cue @@ -31,11 +31,11 @@ import ( } // Command to execute - command: string @dagger(input) + command: string // Environment variables shared by all commands env: { - [string]: string @dagger(input) + [string]: string } // Mount content from other artifacts @@ -44,17 +44,17 @@ import ( from: dagger.#Artifact } | { secret: dagger.#Secret - } @dagger(input) + } } // Mount persistent cache directories cache: { - [string]: true @dagger(input) + [string]: true } // Mount temporary directories tmpfs: { - [string]: true @dagger(input) + [string]: true } // Mount docker socket @@ -62,7 +62,7 @@ import ( // Additional packages to install package: { - [string]: true | false | string @dagger(input) + [string]: true | false | string } // Image registries @@ -70,7 +70,7 @@ import ( target?: string username: string secret: dagger.#Secret - }] @dagger(input) + }] // Copy contents from other artifacts copy: [string]: from: dagger.#Artifact @@ -98,7 +98,7 @@ import ( ssh-add /key > /dev/null if [ "$?" != 0 ]; then exit 1 - fi + fi fi if [[ ! -z $FINGERPRINT ]]; then @@ -131,7 +131,7 @@ import ( for registry in registries { op.#Exec & { args: ["/bin/bash", "-c", #""" - echo "$TARGER_HOST" | docker login --username "$DOCKER_USERNAME" --password-stdin "$(cat /password)" + echo "$TARGER_HOST" | docker login --username "$DOCKER_USERNAME" --password-stdin "$(cat /password)" """#, ] env: {