From 7ca84282be8d30ac8bd60ef8a8239c6ee438e225 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 20 Dec 2021 12:47:42 -0700 Subject: [PATCH 01/13] ported op.#FetchGit to engine.#GitPull Signed-off-by: Richard Jones --- plan/task/gitPull.go | 80 +++++++++++++++++++++++++++++ stdlib/europa/dagger/engine/git.cue | 6 ++- tests/tasks.bats | 4 ++ tests/tasks/gitPull/exists.cue | 10 ++++ 4 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 plan/task/gitPull.go create mode 100644 tests/tasks/gitPull/exists.cue diff --git a/plan/task/gitPull.go b/plan/task/gitPull.go new file mode 100644 index 00000000..657fae86 --- /dev/null +++ b/plan/task/gitPull.go @@ -0,0 +1,80 @@ +package task + +import ( + "context" + "net/url" + + "github.com/moby/buildkit/client/llb" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("GitPull", func() Task { return &gitPullTask{} }) +} + +type gitPullTask struct { +} + +func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + remote, err := v.Lookup("remote").String() + if err != nil { + return nil, err + } + ref, err := v.Lookup("ref").String() + if err != nil { + return nil, err + } + + remoteRedacted := remote + if u, err := url.Parse(remote); err == nil { + remoteRedacted = u.Redacted() + } + + gitOpts := []llb.GitOption{} + var opts struct { + KeepGitDir bool + } + + if err := v.Decode(&opts); err != nil { + return nil, err + } + + if opts.KeepGitDir { + gitOpts = append(gitOpts, llb.KeepGitDir()) + } + // Secret + if authToken := v.Lookup("auth.token"); authToken.Exists() { + authTokenSecret, err := pctx.Secrets.FromValue(authToken) + if err != nil { + return nil, err + } + gitOpts = append(gitOpts, llb.AuthTokenSecret(authTokenSecret.ID())) + } + if authHeader := v.Lookup("auth.header"); authHeader.Exists() { + authHeaderSecret, err := pctx.Secrets.FromValue(authHeader) + if err != nil { + return nil, err + } + gitOpts = append(gitOpts, llb.AuthHeaderSecret(authHeaderSecret.ID())) + } + + gitOpts = append(gitOpts, withCustomName(v, "FetchGit %s@%s", remoteRedacted, ref)) + + st := llb.Git( + remote, + ref, + gitOpts..., + ) + + result, err := s.Solve(ctx, st, pctx.Platform.Get()) + if err != nil { + return nil, err + } + + fs := pctx.FS.New(result) + return compiler.NewValue().FillFields(map[string]interface{}{ + "contents": fs.MarshalCUE(), + }) +} diff --git a/stdlib/europa/dagger/engine/git.cue b/stdlib/europa/dagger/engine/git.cue index f27db9ca..957c4091 100644 --- a/stdlib/europa/dagger/engine/git.cue +++ b/stdlib/europa/dagger/engine/git.cue @@ -12,10 +12,12 @@ package engine // Pull a directory from a git remote #GitPull: { - @dagger(notimplemented) $dagger: task: _name: "GitPull" - remote: string ref: string + auth: { + token?: #Secret + header?: #Secret + } output: #FS } diff --git a/tests/tasks.bats b/tests/tasks.bats index dda02fb9..9d893739 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -93,3 +93,7 @@ setup() { "$DAGGER" --europa up ./scratch.cue -l debug } +@test "task: #GitPull" { + cd "$TESTDIR"/tasks/gitPull/ + "$DAGGER" --europa up ./exists.cue +} diff --git a/tests/tasks/gitPull/exists.cue b/tests/tasks/gitPull/exists.cue new file mode 100644 index 00000000..ef6a7ded --- /dev/null +++ b/tests/tasks/gitPull/exists.cue @@ -0,0 +1,10 @@ +package main + +import "alpha.dagger.io/europa/dagger/engine" + +engine.#Plan & { + actions: gitPull: engine.#GitPull & { + remote: "https://github.com/blocklayerhq/acme-clothing.git" + ref: "master" + } +} From ee938111c699c559f746e481a6e3de7f71b94c04 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 20 Dec 2021 13:30:35 -0700 Subject: [PATCH 02/13] conforming to llb api Signed-off-by: Richard Jones --- plan/task/gitPull.go | 14 ++++++-------- stdlib/europa/dagger/engine/git.cue | 13 ++++++------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/plan/task/gitPull.go b/plan/task/gitPull.go index 657fae86..789412ec 100644 --- a/plan/task/gitPull.go +++ b/plan/task/gitPull.go @@ -44,15 +44,17 @@ func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solve if opts.KeepGitDir { gitOpts = append(gitOpts, llb.KeepGitDir()) } + // Secret - if authToken := v.Lookup("auth.token"); authToken.Exists() { + if authToken := v.Lookup("authToken"); authToken.Exists() { authTokenSecret, err := pctx.Secrets.FromValue(authToken) if err != nil { return nil, err } gitOpts = append(gitOpts, llb.AuthTokenSecret(authTokenSecret.ID())) } - if authHeader := v.Lookup("auth.header"); authHeader.Exists() { + + if authHeader := v.Lookup("authHeader"); authHeader.Exists() { authHeaderSecret, err := pctx.Secrets.FromValue(authHeader) if err != nil { return nil, err @@ -62,11 +64,7 @@ func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solve gitOpts = append(gitOpts, withCustomName(v, "FetchGit %s@%s", remoteRedacted, ref)) - st := llb.Git( - remote, - ref, - gitOpts..., - ) + st := llb.Git(remote, ref, gitOpts...) result, err := s.Solve(ctx, st, pctx.Platform.Get()) if err != nil { @@ -75,6 +73,6 @@ func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solve fs := pctx.FS.New(result) return compiler.NewValue().FillFields(map[string]interface{}{ - "contents": fs.MarshalCUE(), + "output": fs.MarshalCUE(), }) } diff --git a/stdlib/europa/dagger/engine/git.cue b/stdlib/europa/dagger/engine/git.cue index 957c4091..c21a0b68 100644 --- a/stdlib/europa/dagger/engine/git.cue +++ b/stdlib/europa/dagger/engine/git.cue @@ -13,11 +13,10 @@ package engine // Pull a directory from a git remote #GitPull: { $dagger: task: _name: "GitPull" - remote: string - ref: string - auth: { - token?: #Secret - header?: #Secret - } - output: #FS + remote: string + ref: string + authToken?: #Secret + authHeader?: #Secret + keepGitDir: true | *false + output: #FS } From 6848755a428c33c5482bd31763140d24e9d5b2ea Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 20 Dec 2021 13:30:55 -0700 Subject: [PATCH 03/13] added gitdir test Signed-off-by: Richard Jones --- tests/tasks.bats | 1 + tests/tasks/gitPull/gitdir.cue | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/tasks/gitPull/gitdir.cue diff --git a/tests/tasks.bats b/tests/tasks.bats index 9d893739..6b77d043 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -96,4 +96,5 @@ setup() { @test "task: #GitPull" { cd "$TESTDIR"/tasks/gitPull/ "$DAGGER" --europa up ./exists.cue + "$DAGGER" --europa up ./gitdir.cue } diff --git a/tests/tasks/gitPull/gitdir.cue b/tests/tasks/gitPull/gitdir.cue new file mode 100644 index 00000000..93861b02 --- /dev/null +++ b/tests/tasks/gitPull/gitdir.cue @@ -0,0 +1,36 @@ +package testing + +import "alpha.dagger.io/europa/dagger/engine" + +engine.#Plan & { + actions: { + repo1: engine.#GitPull & { + remote: "https://github.com/blocklayerhq/acme-clothing.git" + ref: "master" + } + + repo2: engine.#GitPull & { + remote: "https://github.com/blocklayerhq/acme-clothing.git" + ref: "master" + keepGitDir: true + } + + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + verify: engine.#Exec & { + input: image.output + args: ["sh", "-c", """ + set -eu + [ ! -d /repo1/.git ] + [ -d /repo2/.git ] + """] + mounts: { + repo_1: {dest: "/repo1", contents: repo1.output} + repo_2: {dest: "/repo2", contents: repo2.output} + } + } + + } +} From 1384b025b7cc7f6368683bd71bee4adf446a4aaa Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 20 Dec 2021 14:08:05 -0700 Subject: [PATCH 04/13] more tests Signed-off-by: Richard Jones --- tests/tasks.bats | 8 ++++++++ tests/tasks/gitPull/badref.cue | 10 ++++++++++ tests/tasks/gitPull/badremote.cue | 10 ++++++++++ tests/tasks/gitPull/invalid.cue | 7 +++++++ 4 files changed, 35 insertions(+) create mode 100644 tests/tasks/gitPull/badref.cue create mode 100644 tests/tasks/gitPull/badremote.cue create mode 100644 tests/tasks/gitPull/invalid.cue diff --git a/tests/tasks.bats b/tests/tasks.bats index 6b77d043..72d8f90f 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -97,4 +97,12 @@ setup() { cd "$TESTDIR"/tasks/gitPull/ "$DAGGER" --europa up ./exists.cue "$DAGGER" --europa up ./gitdir.cue + run "$DAGGER" --europa up ./invalid.cue + assert_failure + run "$DAGGER" --europa up ./badremote.cue + assert_failure + run "$DAGGER" --europa up ./badref.cue + assert_failure + + } diff --git a/tests/tasks/gitPull/badref.cue b/tests/tasks/gitPull/badref.cue new file mode 100644 index 00000000..f49fc236 --- /dev/null +++ b/tests/tasks/gitPull/badref.cue @@ -0,0 +1,10 @@ +package main + +import "alpha.dagger.io/europa/dagger/engine" + +engine.#Plan & { + actions: badref: engine.#GitPull & { + remote: "https://github.com/blocklayerhq/acme-clothing.git" + ref: "lalalalal" + } +} diff --git a/tests/tasks/gitPull/badremote.cue b/tests/tasks/gitPull/badremote.cue new file mode 100644 index 00000000..b926ed97 --- /dev/null +++ b/tests/tasks/gitPull/badremote.cue @@ -0,0 +1,10 @@ +package main + +import "alpha.dagger.io/europa/dagger/engine" + +engine.#Plan & { + actions: badremote: engine.#GitPull & { + remote: "https://github.com/blocklayerhq/lalalala.git" + ref: "master" + } +} diff --git a/tests/tasks/gitPull/invalid.cue b/tests/tasks/gitPull/invalid.cue new file mode 100644 index 00000000..61d83638 --- /dev/null +++ b/tests/tasks/gitPull/invalid.cue @@ -0,0 +1,7 @@ +package main + +import "alpha.dagger.io/europa/dagger/engine" + +engine.#Plan & { + actions: invalid: engine.#GitPull & {} +} From a49d7944098d27b0b72a044fbe8f57f17121d211 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 20 Dec 2021 14:58:35 -0700 Subject: [PATCH 05/13] added private repo test Signed-off-by: Richard Jones --- tests/tasks.bats | 4 ++- tests/tasks/gitPull/privateRepo.cue | 31 ++++++++++++++++++++++++ tests/tasks/gitPull/privateRepo.enc.yaml | 20 +++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 tests/tasks/gitPull/privateRepo.cue create mode 100644 tests/tasks/gitPull/privateRepo.enc.yaml diff --git a/tests/tasks.bats b/tests/tasks.bats index 72d8f90f..02b5fec9 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -97,7 +97,9 @@ setup() { cd "$TESTDIR"/tasks/gitPull/ "$DAGGER" --europa up ./exists.cue "$DAGGER" --europa up ./gitdir.cue - run "$DAGGER" --europa up ./invalid.cue + "$DAGGER" --europa up ./privateRepo.cue + + run "$DAGGER" --europa up ./invalid.cue assert_failure run "$DAGGER" --europa up ./badremote.cue assert_failure diff --git a/tests/tasks/gitPull/privateRepo.cue b/tests/tasks/gitPull/privateRepo.cue new file mode 100644 index 00000000..bafdda4d --- /dev/null +++ b/tests/tasks/gitPull/privateRepo.cue @@ -0,0 +1,31 @@ +package main + +import "alpha.dagger.io/europa/dagger/engine" + +engine.#Plan & { + inputs: secrets: TestPAT: command: { + name: "sops" + args: ["exec-env", "./privateRepo.enc.yaml", "echo $data"] + } + actions: { + alpine: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + testRepo: engine.#GitPull & { + remote: "https://github.com/dagger/dagger.git" + ref: "main" + authToken: inputs.secrets.TestPAT.contents + } + + testContent: engine.#Exec & { + input: alpine.output + always: true + args: ["ls", "-l", "/input/repo | grep 'universe -> stdlib'"] + mounts: inputRepo: { + dest: "/input/repo" + contents: testRepo.output + } + } + } +} diff --git a/tests/tasks/gitPull/privateRepo.enc.yaml b/tests/tasks/gitPull/privateRepo.enc.yaml new file mode 100644 index 00000000..d3a8e516 --- /dev/null +++ b/tests/tasks/gitPull/privateRepo.enc.yaml @@ -0,0 +1,20 @@ +{ + "data": "ENC[AES256_GCM,data:bdiZfZl5DUcmEeQz5FYmY7LKMaFqu8tRuImNtM2kofw1JVeFeoyDRA==,iv:ovbCLds/NAzl9dfkLSrkV5qWsQG9uJJjlw8psXLgJ+w=,tag:QvmQeMmPv6+JtOC5/wSr0Q==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": [ + { + "recipient": "age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRc1VaM281bjRUand4V2hz\nMnNIbW8rbUt5Q05LS0RuYVJ2Q09JZU9lZWcwCk8yTVRWYlQvR1Rvb3d4Skx0K2I5\nRXB6MmxnTEtFaThRWU13L05hQWdGblUKLS0tIHpUTll5aFgwSXdzVnMvWG11MDZ4\nbXBNWVVraDI5TmZHMlFSc2VSWSsrTkEKT6gyiklm5eQ04Q2p81+FACg+aRlfgaK6\nqT3aI/pLBrQXIKp5oYpyvY+5WPvM5uQHm5wH7FKXxoxxiZOaIOwgIQ==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2021-12-20T21:36:38Z", + "mac": "ENC[AES256_GCM,data:dxvXu9DDdWOG8pMJlh85OBSe5LePSFy8E1PYw8fQMGQFBVl9dHifTQuZ6eQeGluXtbqetgwa1ZHek1E7UO/WjtWYBtU1tBmlVqmcJ0MIJ67ULFwMHUb5yQqCh7JgymmsHRynqOQ4S2azIbqjmwz6eOyxxqNyJDA6l1NHsrVSeEU=,iv:UC02CgdM30+KOO0KDPkSFKM1HR+t2tenu0j5gYqx1FY=,tag:QB89K6zbWHjk6DCgMz8Lwg==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } +} \ No newline at end of file From 0295dc63407a2369abf2961a2c42641221e9e44c Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 22 Dec 2021 11:02:52 -0700 Subject: [PATCH 06/13] added support for username:password Signed-off-by: Richard Jones --- plan/task/gitPull.go | 62 ++++++++++++++++++----------- stdlib/europa/dagger/engine/git.cue | 21 +++++++--- tests/tasks/gitPull/privateRepo.cue | 16 ++++---- 3 files changed, 62 insertions(+), 37 deletions(-) diff --git a/plan/task/gitPull.go b/plan/task/gitPull.go index 789412ec..6922f1f9 100644 --- a/plan/task/gitPull.go +++ b/plan/task/gitPull.go @@ -3,8 +3,10 @@ package task import ( "context" "net/url" + "strings" "github.com/moby/buildkit/client/llb" + "github.com/rs/zerolog/log" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" @@ -18,53 +20,65 @@ type gitPullTask struct { } func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { - remote, err := v.Lookup("remote").String() - if err != nil { - return nil, err - } - ref, err := v.Lookup("ref").String() - if err != nil { - return nil, err + var gitPull struct { + Remote string + Ref string + KeepGitDir bool + Username string } - remoteRedacted := remote - if u, err := url.Parse(remote); err == nil { - remoteRedacted = u.Redacted() + if err := v.Decode(&gitPull); err != nil { + return nil, err } gitOpts := []llb.GitOption{} - var opts struct { - KeepGitDir bool - } - if err := v.Decode(&opts); err != nil { - return nil, err - } + lg := log.Ctx(ctx) - if opts.KeepGitDir { + if gitPull.KeepGitDir { + lg.Debug().Str("keepGitDir", "true").Msg("adding git option") gitOpts = append(gitOpts, llb.KeepGitDir()) } - // Secret - if authToken := v.Lookup("authToken"); authToken.Exists() { + if gitPull.Username != "" { + pwd := v.Lookup("password") + + pwdSecret, err := pctx.Secrets.FromValue(pwd) + if err != nil { + return nil, err + } + + remote, err := url.Parse(gitPull.Remote) + if err != nil { + return nil, err + } + + remote.User = url.UserPassword(gitPull.Username, strings.TrimSpace(pwdSecret.PlainText())) + gitPull.Remote = remote.String() + } else if authToken := v.Lookup("authToken"); plancontext.IsSecretValue(authToken) { authTokenSecret, err := pctx.Secrets.FromValue(authToken) if err != nil { return nil, err } + lg.Debug().Str("authToken", "***").Msg("adding git option") gitOpts = append(gitOpts, llb.AuthTokenSecret(authTokenSecret.ID())) - } - - if authHeader := v.Lookup("authHeader"); authHeader.Exists() { + } else if authHeader := v.Lookup("authHeader"); plancontext.IsSecretValue(authHeader) { authHeaderSecret, err := pctx.Secrets.FromValue(authHeader) if err != nil { return nil, err } + lg.Debug().Str("authHeader", "***").Msg("adding git option") gitOpts = append(gitOpts, llb.AuthHeaderSecret(authHeaderSecret.ID())) } - gitOpts = append(gitOpts, withCustomName(v, "FetchGit %s@%s", remoteRedacted, ref)) + remoteRedacted := gitPull.Remote + if u, err := url.Parse(gitPull.Remote); err == nil { + remoteRedacted = u.Redacted() + } - st := llb.Git(remote, ref, gitOpts...) + gitOpts = append(gitOpts, withCustomName(v, "GitPull %s@%s", remoteRedacted, gitPull.Ref)) + + st := llb.Git(gitPull.Remote, gitPull.Ref, gitOpts...) result, err := s.Solve(ctx, st, pctx.Platform.Get()) if err != nil { diff --git a/stdlib/europa/dagger/engine/git.cue b/stdlib/europa/dagger/engine/git.cue index c21a0b68..83e3fa8c 100644 --- a/stdlib/europa/dagger/engine/git.cue +++ b/stdlib/europa/dagger/engine/git.cue @@ -11,12 +11,21 @@ package engine } // Pull a directory from a git remote +// Note: do not add credentials to the remote url: e.g: https://username:password@github.com +// as this will expose those in logs. By using username and password (as #Secret) Dagger will +// url encode them for you #GitPull: { $dagger: task: _name: "GitPull" - remote: string - ref: string - authToken?: #Secret - authHeader?: #Secret - keepGitDir: true | *false - output: #FS + remote: string + ref: string + keepGitDir: true | *false + { + username: string + password: #Secret // can be password or personal access token + } | { + authToken: #Secret + } | { + authHeader: #Secret + } + output: #FS } diff --git a/tests/tasks/gitPull/privateRepo.cue b/tests/tasks/gitPull/privateRepo.cue index bafdda4d..2aa2be53 100644 --- a/tests/tasks/gitPull/privateRepo.cue +++ b/tests/tasks/gitPull/privateRepo.cue @@ -3,27 +3,29 @@ package main import "alpha.dagger.io/europa/dagger/engine" engine.#Plan & { - inputs: secrets: TestPAT: command: { + inputs: secrets: token: command: { name: "sops" args: ["exec-env", "./privateRepo.enc.yaml", "echo $data"] } + actions: { alpine: engine.#Pull & { - source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + source: "alpine:3.15.0" } testRepo: engine.#GitPull & { - remote: "https://github.com/dagger/dagger.git" - ref: "main" - authToken: inputs.secrets.TestPAT.contents + remote: "https://github.com/dagger/dagger.git" + ref: "main" + username: "dagger-test" + password: inputs.secrets.token.contents } testContent: engine.#Exec & { input: alpine.output always: true - args: ["ls", "-l", "/input/repo | grep 'universe -> stdlib'"] + args: ["ls", "-l", "/repo"] mounts: inputRepo: { - dest: "/input/repo" + dest: "/repo" contents: testRepo.output } } From feb685cf814d7170570c590c803d33d976112b63 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 22 Dec 2021 11:56:58 -0700 Subject: [PATCH 07/13] fixed issue with disjunction that wouldnt resolve Signed-off-by: Richard Jones --- plan/task/gitPull.go | 14 ++++++++------ stdlib/europa/dagger/engine/git.cue | 2 +- tests/tasks/gitPull/gitdir.cue | 6 +++--- tests/tasks/gitPull/privateRepo.cue | 10 ++++++---- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/plan/task/gitPull.go b/plan/task/gitPull.go index 6922f1f9..5cf1f2ad 100644 --- a/plan/task/gitPull.go +++ b/plan/task/gitPull.go @@ -24,7 +24,9 @@ func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solve Remote string Ref string KeepGitDir bool - Username string + Auth struct { + Username string + } } if err := v.Decode(&gitPull); err != nil { @@ -40,8 +42,8 @@ func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solve gitOpts = append(gitOpts, llb.KeepGitDir()) } - if gitPull.Username != "" { - pwd := v.Lookup("password") + if gitPull.Auth.Username != "" { + pwd := v.Lookup("auth.password") pwdSecret, err := pctx.Secrets.FromValue(pwd) if err != nil { @@ -53,16 +55,16 @@ func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solve return nil, err } - remote.User = url.UserPassword(gitPull.Username, strings.TrimSpace(pwdSecret.PlainText())) + remote.User = url.UserPassword(gitPull.Auth.Username, strings.TrimSpace(pwdSecret.PlainText())) gitPull.Remote = remote.String() - } else if authToken := v.Lookup("authToken"); plancontext.IsSecretValue(authToken) { + } else if authToken := v.Lookup("auth.authToken"); plancontext.IsSecretValue(authToken) { authTokenSecret, err := pctx.Secrets.FromValue(authToken) if err != nil { return nil, err } lg.Debug().Str("authToken", "***").Msg("adding git option") gitOpts = append(gitOpts, llb.AuthTokenSecret(authTokenSecret.ID())) - } else if authHeader := v.Lookup("authHeader"); plancontext.IsSecretValue(authHeader) { + } else if authHeader := v.Lookup("auth.authHeader"); plancontext.IsSecretValue(authHeader) { authHeaderSecret, err := pctx.Secrets.FromValue(authHeader) if err != nil { return nil, err diff --git a/stdlib/europa/dagger/engine/git.cue b/stdlib/europa/dagger/engine/git.cue index 83e3fa8c..aaa8fd17 100644 --- a/stdlib/europa/dagger/engine/git.cue +++ b/stdlib/europa/dagger/engine/git.cue @@ -19,7 +19,7 @@ package engine remote: string ref: string keepGitDir: true | *false - { + auth?: { username: string password: #Secret // can be password or personal access token } | { diff --git a/tests/tasks/gitPull/gitdir.cue b/tests/tasks/gitPull/gitdir.cue index 93861b02..3baef549 100644 --- a/tests/tasks/gitPull/gitdir.cue +++ b/tests/tasks/gitPull/gitdir.cue @@ -16,7 +16,7 @@ engine.#Plan & { } image: engine.#Pull & { - source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + source: "alpine:3.15.0" } verify: engine.#Exec & { @@ -27,8 +27,8 @@ engine.#Plan & { [ -d /repo2/.git ] """] mounts: { - repo_1: {dest: "/repo1", contents: repo1.output} - repo_2: {dest: "/repo2", contents: repo2.output} + a: {dest: "/repo1", contents: repo1.output} + b: {dest: "/repo2", contents: repo2.output} } } diff --git a/tests/tasks/gitPull/privateRepo.cue b/tests/tasks/gitPull/privateRepo.cue index 2aa2be53..84c5ee4b 100644 --- a/tests/tasks/gitPull/privateRepo.cue +++ b/tests/tasks/gitPull/privateRepo.cue @@ -14,10 +14,12 @@ engine.#Plan & { } testRepo: engine.#GitPull & { - remote: "https://github.com/dagger/dagger.git" - ref: "main" - username: "dagger-test" - password: inputs.secrets.token.contents + remote: "https://github.com/dagger/dagger.git" + ref: "main" + auth: { + username: "dagger-test" + password: inputs.secrets.token.contents + } } testContent: engine.#Exec & { From c3d3ca47ffe9631429d7039aaca77489c060af85 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 22 Dec 2021 11:58:35 -0700 Subject: [PATCH 08/13] fixed indentation Signed-off-by: Richard Jones --- tests/tasks.bats | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/tasks.bats b/tests/tasks.bats index 02b5fec9..ae66f6a3 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -97,14 +97,14 @@ setup() { cd "$TESTDIR"/tasks/gitPull/ "$DAGGER" --europa up ./exists.cue "$DAGGER" --europa up ./gitdir.cue - "$DAGGER" --europa up ./privateRepo.cue - - run "$DAGGER" --europa up ./invalid.cue - assert_failure - run "$DAGGER" --europa up ./badremote.cue - assert_failure - run "$DAGGER" --europa up ./badref.cue - assert_failure + "$DAGGER" --europa up ./privateRepo.cue + + run "$DAGGER" --europa up ./invalid.cue + assert_failure + run "$DAGGER" --europa up ./badremote.cue + assert_failure + run "$DAGGER" --europa up ./badref.cue + assert_failure } From b3118628b01df5e885e6fbe678a23d05b5c78f09 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 22 Dec 2021 12:01:57 -0700 Subject: [PATCH 09/13] docs Signed-off-by: Richard Jones --- docs/reference/europa/dagger/engine.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/europa/dagger/engine.md b/docs/reference/europa/dagger/engine.md index 05f964ea..40c11901 100644 --- a/docs/reference/europa/dagger/engine.md +++ b/docs/reference/europa/dagger/engine.md @@ -92,7 +92,7 @@ _No output._ ## engine.#GitPull -Pull a directory from a git remote +Pull a directory from a git remote Note: do not add credentials to the remote url: e.g: https://username:password@github.com as this will expose those in logs. By using username and password (as #Secret) Dagger will url encode them for you ### engine.#GitPull Inputs From 455ccc460f4def6c1c3d6243b21e41bb4bc3966e Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 22 Dec 2021 12:07:07 -0700 Subject: [PATCH 10/13] removed URL from docs Signed-off-by: Richard Jones --- docs/reference/europa/dagger/engine.md | 2 +- stdlib/europa/dagger/engine/git.cue | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/reference/europa/dagger/engine.md b/docs/reference/europa/dagger/engine.md index 40c11901..c7f64d24 100644 --- a/docs/reference/europa/dagger/engine.md +++ b/docs/reference/europa/dagger/engine.md @@ -92,7 +92,7 @@ _No output._ ## engine.#GitPull -Pull a directory from a git remote Note: do not add credentials to the remote url: e.g: https://username:password@github.com as this will expose those in logs. By using username and password (as #Secret) Dagger will url encode them for you +Pull a directory from a git remote Warning: do NOT embed credentials in the remote url as this will expose them in logs. By using username and password Dagger will handle this for you in a secure manner. ### engine.#GitPull Inputs diff --git a/stdlib/europa/dagger/engine/git.cue b/stdlib/europa/dagger/engine/git.cue index aaa8fd17..034f16b0 100644 --- a/stdlib/europa/dagger/engine/git.cue +++ b/stdlib/europa/dagger/engine/git.cue @@ -11,9 +11,8 @@ package engine } // Pull a directory from a git remote -// Note: do not add credentials to the remote url: e.g: https://username:password@github.com -// as this will expose those in logs. By using username and password (as #Secret) Dagger will -// url encode them for you +// Warning: do NOT embed credentials in the remote url as this will expose them in logs. +// By using username and password Dagger will handle this for you in a secure manner. #GitPull: { $dagger: task: _name: "GitPull" remote: string From 2b95bf15363c76cc6824365f75784c14cbac4235 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 22 Dec 2021 15:29:22 -0700 Subject: [PATCH 11/13] renamed files to avoid camelCase; moved token to common secrets file Signed-off-by: Richard Jones --- plan/task/{gitPull.go => gitpull.go} | 0 tests/secrets_sops.yaml | 21 +++++++++++++++++++ tests/tasks.bats | 14 ++++++------- .../tasks/gitPull/{badref.cue => bad_ref.cue} | 0 .../gitPull/{badremote.cue => bad_remote.cue} | 0 .../tasks/gitPull/{gitdir.cue => git_dir.cue} | 0 tests/tasks/gitPull/privateRepo.enc.yaml | 20 ------------------ .../{privateRepo.cue => private_repo.cue} | 2 +- 8 files changed, 29 insertions(+), 28 deletions(-) rename plan/task/{gitPull.go => gitpull.go} (100%) create mode 100644 tests/secrets_sops.yaml rename tests/tasks/gitPull/{badref.cue => bad_ref.cue} (100%) rename tests/tasks/gitPull/{badremote.cue => bad_remote.cue} (100%) rename tests/tasks/gitPull/{gitdir.cue => git_dir.cue} (100%) delete mode 100644 tests/tasks/gitPull/privateRepo.enc.yaml rename tests/tasks/gitPull/{privateRepo.cue => private_repo.cue} (90%) diff --git a/plan/task/gitPull.go b/plan/task/gitpull.go similarity index 100% rename from plan/task/gitPull.go rename to plan/task/gitpull.go diff --git a/tests/secrets_sops.yaml b/tests/secrets_sops.yaml new file mode 100644 index 00000000..63bdf7e1 --- /dev/null +++ b/tests/secrets_sops.yaml @@ -0,0 +1,21 @@ +TestPAT: ENC[AES256_GCM,data:KYPnJTTCaEbEiBwODMDmOmZGx/Vu/4mOZPfRSjhBc239fPfHzDH75w==,iv:j9UFKfdRMfYg/3xw4dCoWbs0Zoy3czqRznlQrRvf4Sc=,tag:Pvd4UMFDOj8rLOwZJjsYpA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnUEhWbjV3M29oUUJyWk81 + Wk1WQ1E0cmtuVlhNSGxkWUM3WmJXdUYvbzAwCjlFWW9IVmtmTjY1aU1LR2lxWFlT + am9RemNqSDRWK2FDYk1xeGNiTFlWMFUKLS0tIFVrSzBCMERQbnhYb09ReVpFK00v + TG5YUDlFVzlRRFBCdEhsNVlVK1dMRTgKx1TPZWWQiaU8iMni03/ekG+m4rFCcaa4 + JI+ED2d+8411BgZtlss/ukQtwskidvYTvetyWw2jes6o1lhfDv5q2A== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-12-22T22:23:53Z" + mac: ENC[AES256_GCM,data:twjo6LyD0QFJoD1h5qh8SFy4XC5+2JHl6mUiUxOjLnSBwft1Ntnto3+2tzZBU6+o+v/Uo++vRoSrIkLARFt74x2xA7jZjc81e5yCkcS79bWSpZ8bA7e8/5hgkyYP5SlMAYsWKXPodmhgN7bwCa6vjZb4ZlFkJuBZrHtLeziVY1E=,iv:kgDj5lnLTgLPK38CcUVYdtE11d/gpFdHPP9hgoZQO9U=,tag:tP2E3t/jKoh2Fx6Z2X/+Tw==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.1 diff --git a/tests/tasks.bats b/tests/tasks.bats index ae66f6a3..d17593fb 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -94,16 +94,16 @@ setup() { } @test "task: #GitPull" { - cd "$TESTDIR"/tasks/gitPull/ - "$DAGGER" --europa up ./exists.cue - "$DAGGER" --europa up ./gitdir.cue - "$DAGGER" --europa up ./privateRepo.cue + cd "$TESTDIR" + "$DAGGER" --europa up ./tasks/gitPull/exists.cue + "$DAGGER" --europa up ./tasks/gitPull/git_dir.cue + "$DAGGER" --europa up ./tasks/gitPull/private_repo.cue - run "$DAGGER" --europa up ./invalid.cue + run "$DAGGER" --europa up ./tasks/gitPull/invalid.cue assert_failure - run "$DAGGER" --europa up ./badremote.cue + run "$DAGGER" --europa up ./tasks/gitPull/bad_remote.cue assert_failure - run "$DAGGER" --europa up ./badref.cue + run "$DAGGER" --europa up ./tasks/gitPull/bad_ref.cue assert_failure diff --git a/tests/tasks/gitPull/badref.cue b/tests/tasks/gitPull/bad_ref.cue similarity index 100% rename from tests/tasks/gitPull/badref.cue rename to tests/tasks/gitPull/bad_ref.cue diff --git a/tests/tasks/gitPull/badremote.cue b/tests/tasks/gitPull/bad_remote.cue similarity index 100% rename from tests/tasks/gitPull/badremote.cue rename to tests/tasks/gitPull/bad_remote.cue diff --git a/tests/tasks/gitPull/gitdir.cue b/tests/tasks/gitPull/git_dir.cue similarity index 100% rename from tests/tasks/gitPull/gitdir.cue rename to tests/tasks/gitPull/git_dir.cue diff --git a/tests/tasks/gitPull/privateRepo.enc.yaml b/tests/tasks/gitPull/privateRepo.enc.yaml deleted file mode 100644 index d3a8e516..00000000 --- a/tests/tasks/gitPull/privateRepo.enc.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{ - "data": "ENC[AES256_GCM,data:bdiZfZl5DUcmEeQz5FYmY7LKMaFqu8tRuImNtM2kofw1JVeFeoyDRA==,iv:ovbCLds/NAzl9dfkLSrkV5qWsQG9uJJjlw8psXLgJ+w=,tag:QvmQeMmPv6+JtOC5/wSr0Q==,type:str]", - "sops": { - "kms": null, - "gcp_kms": null, - "azure_kv": null, - "hc_vault": null, - "age": [ - { - "recipient": "age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRc1VaM281bjRUand4V2hz\nMnNIbW8rbUt5Q05LS0RuYVJ2Q09JZU9lZWcwCk8yTVRWYlQvR1Rvb3d4Skx0K2I5\nRXB6MmxnTEtFaThRWU13L05hQWdGblUKLS0tIHpUTll5aFgwSXdzVnMvWG11MDZ4\nbXBNWVVraDI5TmZHMlFSc2VSWSsrTkEKT6gyiklm5eQ04Q2p81+FACg+aRlfgaK6\nqT3aI/pLBrQXIKp5oYpyvY+5WPvM5uQHm5wH7FKXxoxxiZOaIOwgIQ==\n-----END AGE ENCRYPTED FILE-----\n" - } - ], - "lastmodified": "2021-12-20T21:36:38Z", - "mac": "ENC[AES256_GCM,data:dxvXu9DDdWOG8pMJlh85OBSe5LePSFy8E1PYw8fQMGQFBVl9dHifTQuZ6eQeGluXtbqetgwa1ZHek1E7UO/WjtWYBtU1tBmlVqmcJ0MIJ67ULFwMHUb5yQqCh7JgymmsHRynqOQ4S2azIbqjmwz6eOyxxqNyJDA6l1NHsrVSeEU=,iv:UC02CgdM30+KOO0KDPkSFKM1HR+t2tenu0j5gYqx1FY=,tag:QB89K6zbWHjk6DCgMz8Lwg==,type:str]", - "pgp": null, - "unencrypted_suffix": "_unencrypted", - "version": "3.7.1" - } -} \ No newline at end of file diff --git a/tests/tasks/gitPull/privateRepo.cue b/tests/tasks/gitPull/private_repo.cue similarity index 90% rename from tests/tasks/gitPull/privateRepo.cue rename to tests/tasks/gitPull/private_repo.cue index 84c5ee4b..3d93d486 100644 --- a/tests/tasks/gitPull/privateRepo.cue +++ b/tests/tasks/gitPull/private_repo.cue @@ -5,7 +5,7 @@ import "alpha.dagger.io/europa/dagger/engine" engine.#Plan & { inputs: secrets: token: command: { name: "sops" - args: ["exec-env", "./privateRepo.enc.yaml", "echo $data"] + args: ["exec-env", "./secrets_sops.yaml", "echo $TestPAT"] } actions: { From 044c464289ef86b82b3f2b3586cd5809d54d0029 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 22 Dec 2021 15:43:45 -0700 Subject: [PATCH 12/13] added debug message Signed-off-by: Richard Jones --- plan/task/gitpull.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plan/task/gitpull.go b/plan/task/gitpull.go index 5cf1f2ad..cb8532bd 100644 --- a/plan/task/gitpull.go +++ b/plan/task/gitpull.go @@ -55,6 +55,7 @@ func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solve return nil, err } + lg.Debug().Str("username", gitPull.Auth.Username).Str("password", "***").Msg("using username:password auth") remote.User = url.UserPassword(gitPull.Auth.Username, strings.TrimSpace(pwdSecret.PlainText())) gitPull.Remote = remote.String() } else if authToken := v.Lookup("auth.authToken"); plancontext.IsSecretValue(authToken) { From bd005aeb43c51df1d33659a072d8b765bc6e5b43 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 22 Dec 2021 15:44:04 -0700 Subject: [PATCH 13/13] improved test case Signed-off-by: Richard Jones --- tests/tasks/gitPull/private_repo.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tasks/gitPull/private_repo.cue b/tests/tasks/gitPull/private_repo.cue index 3d93d486..cce02b5a 100644 --- a/tests/tasks/gitPull/private_repo.cue +++ b/tests/tasks/gitPull/private_repo.cue @@ -25,7 +25,7 @@ engine.#Plan & { testContent: engine.#Exec & { input: alpine.output always: true - args: ["ls", "-l", "/repo"] + args: ["ls", "-l", "/repo/README.md"] mounts: inputRepo: { dest: "/repo" contents: testRepo.output