From 86bf3bad86ee1539ed73798e1fce9f8186b5f867 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Tue, 4 Jan 2022 20:27:54 -0700 Subject: [PATCH 1/5] transform secret Signed-off-by: Richard Jones --- .../dagger/engine/transformsecret.cue | 17 +++++++ plan/task/transformsecret.go | 47 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 pkg/dagger.io/dagger/engine/transformsecret.cue create mode 100644 plan/task/transformsecret.go diff --git a/pkg/dagger.io/dagger/engine/transformsecret.cue b/pkg/dagger.io/dagger/engine/transformsecret.cue new file mode 100644 index 00000000..1eae9459 --- /dev/null +++ b/pkg/dagger.io/dagger/engine/transformsecret.cue @@ -0,0 +1,17 @@ +package engine + +// Securely apply a CUE transformation on the contents of a secret +#TransformSecret: { + $dagger: task: _name: "TransformSecret" + // The original secret + input: #Secret + // A new secret with the transformation applied + output: #Secret + // Transformation function + #function: { + // Full contents of the input secret (only available to the function) + input: string + // New contents of the output secret (must provided by the caller) + output: string + } +} diff --git a/plan/task/transformsecret.go b/plan/task/transformsecret.go new file mode 100644 index 00000000..4184d863 --- /dev/null +++ b/plan/task/transformsecret.go @@ -0,0 +1,47 @@ +package task + +import ( + "context" + "errors" + + "cuelang.org/go/cue" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("TransformSecret", func() Task { return &transformSecretTask{} }) +} + +type transformSecretTask struct { +} + +func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { + lg := log.Ctx(ctx) + lg.Debug().Msg("transforming secret") + + input := v.Lookup("input") + if !plancontext.IsSecretValue(input) { + return nil, errors.New("#TransformSecret requires input: #Secret") + } + + inputSecret, err := pctx.Secrets.FromValue(input) + if err != nil { + return nil, err + } + + function := v.Lookup("#function") + function.FillPath(cue.ParsePath("input"), inputSecret.PlainText()) + + outputPlaintext, err := function.Lookup("output").String() + if err != nil { + return nil, err + } + + outputSecret := pctx.Secrets.New(outputPlaintext) + return compiler.NewValue().FillFields(map[string]interface{}{ + "output": outputSecret.MarshalCUE(), + }) +} From 86ae2302614d26be5dff7fc66ac91e87178cce76 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Tue, 4 Jan 2022 20:30:58 -0700 Subject: [PATCH 2/5] refactored existing tests to use #TransformSecret Signed-off-by: Richard Jones --- tests/tasks/build/build_auth.cue | 36 ++++++++++++++++++---------- tests/tasks/gitpull/private_repo.cue | 20 ++++++++++++---- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/tests/tasks/build/build_auth.cue b/tests/tasks/build/build_auth.cue index 79d2fa05..d7505a0d 100644 --- a/tests/tasks/build/build_auth.cue +++ b/tests/tasks/build/build_auth.cue @@ -2,26 +2,38 @@ package testing import ( "dagger.io/dagger/engine" + "encoding/yaml" ) engine.#Plan & { inputs: { directories: testdata: path: "./testdata" - secrets: dockerHubToken: command: { + secrets: sops: command: { name: "sops" - args: ["exec-env", "../../secrets_sops.yaml", "echo $DOCKERHUB_TOKEN"] + args: ["-d", "../../secrets_sops.yaml"] } } - actions: build: engine.#Build & { - source: inputs.directories.testdata.contents - auth: [{ - target: "daggerio/ci-test:private-pull" - username: "daggertest" - secret: inputs.secrets.dockerHubToken.contents - }] - dockerfile: contents: """ - FROM daggerio/ci-test:private-pull@sha256:c74f1b1166784193ea6c8f9440263b9be6cae07dfe35e32a5df7a31358ac2060 - """ + actions: { + dockerHubToken: engine.#TransformSecret & { + input: inputs.secrets.sops.contents + #function: { + input: _ + output: yaml.Unmarshal(input).DOCKERHUB_TOKEN + } + } + + build: engine.#Build & { + source: inputs.directories.testdata.contents + auth: [{ + target: "daggerio/ci-test:private-pull" + username: "daggertest" + + secret: dockerHubToken.output + }] + dockerfile: contents: """ + FROM daggerio/ci-test:private-pull@sha256:c74f1b1166784193ea6c8f9440263b9be6cae07dfe35e32a5df7a31358ac2060 + """ + } } } diff --git a/tests/tasks/gitpull/private_repo.cue b/tests/tasks/gitpull/private_repo.cue index 028852a2..b9b84dc1 100644 --- a/tests/tasks/gitpull/private_repo.cue +++ b/tests/tasks/gitpull/private_repo.cue @@ -1,24 +1,36 @@ package main -import "dagger.io/dagger/engine" +import ( + "encoding/yaml" + "dagger.io/dagger/engine" +) engine.#Plan & { - inputs: secrets: token: command: { + inputs: secrets: sops: command: { name: "sops" - args: ["exec-env", "../../secrets_sops.yaml", "echo $TestPAT"] + args: ["-d", "../../secrets_sops.yaml"] } actions: { + alpine: engine.#Pull & { source: "alpine:3.15.0" } + repoPassword: engine.#TransformSecret & { + input: inputs.secrets.sops.contents + #function: { + input: _ + output: yaml.Unmarshal(input).TestPAT + } + } + testRepo: engine.#GitPull & { remote: "https://github.com/dagger/dagger.git" ref: "main" auth: { username: "dagger-test" - password: inputs.secrets.token.contents + password: repoPassword.output } } From 05820f3a67f6966e05242ec7ca249e26991eb2fa Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 6 Jan 2022 13:13:51 -0700 Subject: [PATCH 3/5] supports map of secrets; errors redact plaintext Signed-off-by: Richard Jones --- go.mod | 1 + .../dagger/engine/transformsecret.cue | 9 ++-- plan/task/transformsecret.go | 41 ++++++++++++++----- tests/tasks/build/build_auth.cue | 4 +- tests/tasks/gitpull/private_repo.cue | 4 +- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 326eb10c..3e569ff5 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2 github.com/rs/zerolog v1.26.0 + github.com/sergi/go-diff v1.1.0 // indirect github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 diff --git a/pkg/dagger.io/dagger/engine/transformsecret.cue b/pkg/dagger.io/dagger/engine/transformsecret.cue index 1eae9459..35da8b82 100644 --- a/pkg/dagger.io/dagger/engine/transformsecret.cue +++ b/pkg/dagger.io/dagger/engine/transformsecret.cue @@ -5,13 +5,14 @@ package engine $dagger: task: _name: "TransformSecret" // The original secret input: #Secret - // A new secret with the transformation applied - output: #Secret + // A new secret or (map of secrets) with the transformation applied + output: #Secret | {[string]: output} // Transformation function #function: { // Full contents of the input secret (only available to the function) - input: string + input: string + _functionOutput: string | {[string]: _functionOutput} // New contents of the output secret (must provided by the caller) - output: string + output: _functionOutput } } diff --git a/plan/task/transformsecret.go b/plan/task/transformsecret.go index 4184d863..516f802b 100644 --- a/plan/task/transformsecret.go +++ b/plan/task/transformsecret.go @@ -3,9 +3,11 @@ package task import ( "context" "errors" + "strings" "cuelang.org/go/cue" "github.com/rs/zerolog/log" + "github.com/sergi/go-diff/diffmatchpatch" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" @@ -23,9 +25,6 @@ func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context lg.Debug().Msg("transforming secret") input := v.Lookup("input") - if !plancontext.IsSecretValue(input) { - return nil, errors.New("#TransformSecret requires input: #Secret") - } inputSecret, err := pctx.Secrets.FromValue(input) if err != nil { @@ -33,15 +32,37 @@ func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context } function := v.Lookup("#function") - function.FillPath(cue.ParsePath("input"), inputSecret.PlainText()) - - outputPlaintext, err := function.Lookup("output").String() + inputSecretPlaintext := inputSecret.PlainText() + err = function.FillPath(cue.ParsePath("input"), inputSecretPlaintext) if err != nil { - return nil, err + dmp := diffmatchpatch.New() + errStr := err.Error() + diffs := dmp.DiffMain(inputSecretPlaintext, err.Error(), false) + for _, diff := range diffs { + if diff.Type == diffmatchpatch.DiffEqual { + diffText := strings.ReplaceAll(diff.Text, ":", "") + errStr = strings.ReplaceAll(errStr, diffText, "") + } + } + + return nil, errors.New(errStr) } - outputSecret := pctx.Secrets.New(outputPlaintext) - return compiler.NewValue().FillFields(map[string]interface{}{ - "output": outputSecret.MarshalCUE(), + output := compiler.NewValue() + // users could yaml.Unmarshal(input) and return a map + // or yaml.Unmarshal(input).someKey and return a string + // walk will ensure we convert every leaf + functionPathSelectors := function.Path().Selectors() + function.Lookup("output").Walk(nil, func(v *compiler.Value) { + if v.Kind() == cue.StringKind { + plaintext, _ := v.String() + secret := pctx.Secrets.New(plaintext) + newLeafSelectors := v.Path().Selectors()[len(functionPathSelectors):] + newLeafSelectors = append(newLeafSelectors, cue.Str("contents")) + newLeafPath := cue.MakePath(newLeafSelectors...) + output.FillPath(newLeafPath, secret.MarshalCUE()) + } }) + + return output, nil } diff --git a/tests/tasks/build/build_auth.cue b/tests/tasks/build/build_auth.cue index d7505a0d..8b34138a 100644 --- a/tests/tasks/build/build_auth.cue +++ b/tests/tasks/build/build_auth.cue @@ -19,7 +19,7 @@ engine.#Plan & { input: inputs.secrets.sops.contents #function: { input: _ - output: yaml.Unmarshal(input).DOCKERHUB_TOKEN + output: yaml.Unmarshal(input) } } @@ -29,7 +29,7 @@ engine.#Plan & { target: "daggerio/ci-test:private-pull" username: "daggertest" - secret: dockerHubToken.output + secret: dockerHubToken.output.DOCKERHUB_TOKEN.contents }] dockerfile: contents: """ FROM daggerio/ci-test:private-pull@sha256:c74f1b1166784193ea6c8f9440263b9be6cae07dfe35e32a5df7a31358ac2060 diff --git a/tests/tasks/gitpull/private_repo.cue b/tests/tasks/gitpull/private_repo.cue index b9b84dc1..1bedb4c5 100644 --- a/tests/tasks/gitpull/private_repo.cue +++ b/tests/tasks/gitpull/private_repo.cue @@ -21,7 +21,7 @@ engine.#Plan & { input: inputs.secrets.sops.contents #function: { input: _ - output: yaml.Unmarshal(input).TestPAT + output: yaml.Unmarshal(input) } } @@ -30,7 +30,7 @@ engine.#Plan & { ref: "main" auth: { username: "dagger-test" - password: repoPassword.output + password: repoPassword.output.TestPAT.contents } } From 63e092c3b1b6cad2afe360e66bbe45ce7e230906 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Tue, 11 Jan 2022 13:37:03 -0700 Subject: [PATCH 4/5] commented out replace colons Signed-off-by: Richard Jones --- plan/task/transformsecret.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plan/task/transformsecret.go b/plan/task/transformsecret.go index 516f802b..262999de 100644 --- a/plan/task/transformsecret.go +++ b/plan/task/transformsecret.go @@ -40,8 +40,8 @@ func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context diffs := dmp.DiffMain(inputSecretPlaintext, err.Error(), false) for _, diff := range diffs { if diff.Type == diffmatchpatch.DiffEqual { - diffText := strings.ReplaceAll(diff.Text, ":", "") - errStr = strings.ReplaceAll(errStr, diffText, "") + // diffText := strings.ReplaceAll(diff.Text, ":", "") // colons are tricky. Yaml keys end with them but if a secret contained one that got replaced, the secret wouldn't get redacted + errStr = strings.ReplaceAll(errStr, diff.Text, "") } } From d7731fdc195bd27ba3bd939f8d930d5c0def69d1 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 13 Jan 2022 15:21:30 -0700 Subject: [PATCH 5/5] redacting with stars Signed-off-by: Richard Jones --- plan/task/transformsecret.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plan/task/transformsecret.go b/plan/task/transformsecret.go index 262999de..c956e2e8 100644 --- a/plan/task/transformsecret.go +++ b/plan/task/transformsecret.go @@ -41,7 +41,7 @@ func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context for _, diff := range diffs { if diff.Type == diffmatchpatch.DiffEqual { // diffText := strings.ReplaceAll(diff.Text, ":", "") // colons are tricky. Yaml keys end with them but if a secret contained one that got replaced, the secret wouldn't get redacted - errStr = strings.ReplaceAll(errStr, diff.Text, "") + errStr = strings.ReplaceAll(errStr, diff.Text, "***") } }