From 54e8a6262d2ce646d6b3f7641751a288026254f2 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 14 May 2021 17:53:23 -0700 Subject: [PATCH 01/15] implemented @dagger(input) attributes to detect inputs Signed-off-by: Sam Alba --- pkg/cuetils/scan.go | 56 +++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/pkg/cuetils/scan.go b/pkg/cuetils/scan.go index 499944bd..0e327554 100644 --- a/pkg/cuetils/scan.go +++ b/pkg/cuetils/scan.go @@ -14,40 +14,15 @@ func ScanForInputs(value cue.Value) ([]cue.Value, error) { err error ) - // walk before function, bool return is if the walk should recurse again + // walk before function, bool return true if the walk should recurse again before := func(v cue.Value) (bool, error) { - // explicit phase - // look for #up - label, _ := v.Label() - if label == "#up" { - return false, nil - } - - // look to exclude any @dagger(computed) - attrs := v.Attributes(cue.ValueAttr) - for _, attr := range attrs { - name := attr.Name() - // match `@dagger(...)` - if name == "dagger" { - // loop over args (CSV content in attribute) - for i := 0; i < attr.NumArgs(); i++ { - key, _ := attr.Arg(i) - - // we found an explicit computed value - if key == "computed" { - return false, nil - } - } - } - } - // inference phase switch v.IncompleteKind() { case cue.StructKind: return true, nil case cue.ListKind: - if !v.IsConcrete() { + if !v.IsConcrete() && isInput(v) { vals = append(vals, v) return false, nil } @@ -55,6 +30,11 @@ func ScanForInputs(value cue.Value) ([]cue.Value, error) { default: + if !isInput(v) { + // Not an input + return false, nil + } + // a leaf with default? _, has := v.Default() if has { @@ -81,6 +61,28 @@ func ScanForInputs(value cue.Value) ([]cue.Value, error) { return vals, nil } +func isInput(v cue.Value) bool { + attrs := v.Attributes(cue.ValueAttr) + + for _, attr := range attrs { + name := attr.Name() + // match `@dagger(...)` + if name == "dagger" { + // loop over args (CSV content in attribute) + for i := 0; i < attr.NumArgs(); i++ { + key, _ := attr.Arg(i) + + // we found an explicit input value + if key == "input" { + return true + } + } + } + } + + return false +} + // walkValue is a custome walk function so that we recurse into more types than CUE's buildin walk // specificially, we need to customize the options to val.Fields when val is a struct func walkValue(val cue.Value, before func(cue.Value) (bool, error), after func(cue.Value) error) error { From a369c7a1ba9ec54a5d7a333a601ed6de0c7a1823 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 14 May 2021 17:53:57 -0700 Subject: [PATCH 02/15] examples: simple-s3: converted to inputs attributes for testing Signed-off-by: Sam Alba --- examples/simple-s3/main.cue | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/simple-s3/main.cue b/examples/simple-s3/main.cue index f8785da5..06e3e8f3 100644 --- a/examples/simple-s3/main.cue +++ b/examples/simple-s3/main.cue @@ -8,13 +8,15 @@ import ( // AWS Config for credentials and default region awsConfig: aws.#Config & { - region: *"us-east-1" | string + region: *"us-east-1" | string @dagger(input) + accessKey: dagger.#Secret @dagger(input) + secretKey: dagger.#Secret @dagger(input) } // Name of the S3 bucket to use -bucket: *"dagger-io-examples" | string +bucket: *"dagger-io-examples" | string @dagger(input) -source: dagger.#Artifact +source: dagger.#Artifact @dagger(input) url: "\(deploy.url)index.html" deploy: s3.#Put & { From 85cc9abb9378b195ab9ebca37b355b4b237e0f3e Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 18 May 2021 21:37:34 -0700 Subject: [PATCH 03/15] stdlib: adapted lib to new input spec Signed-off-by: Sam Alba --- stdlib/aws/aws.cue | 6 +++--- stdlib/aws/s3/s3.cue | 14 +++++++------- stdlib/dagger/dagger.cue | 3 +++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/stdlib/aws/aws.cue b/stdlib/aws/aws.cue index f0f65d07..9a3be0ff 100644 --- a/stdlib/aws/aws.cue +++ b/stdlib/aws/aws.cue @@ -9,11 +9,11 @@ import ( // Base AWS Config #Config: { // AWS region - region: string + region: string @dagger(input) // AWS access key - accessKey: dagger.#Secret + accessKey: dagger.#Secret @dagger(input) // AWS secret key - secretKey: dagger.#Secret + secretKey: dagger.#Secret @dagger(input) } // Re-usable aws-cli component diff --git a/stdlib/aws/s3/s3.cue b/stdlib/aws/s3/s3.cue index c3c1cf57..306f2b9e 100644 --- a/stdlib/aws/s3/s3.cue +++ b/stdlib/aws/s3/s3.cue @@ -9,25 +9,25 @@ import ( #Put: { // AWS Config - config: aws.#Config + config: aws.#Config @dagger(input) // Source Artifact to upload to S3 - source?: dagger.#Artifact + source?: dagger.#Artifact @dagger(input) // Source inlined as a string to upload to S3 - sourceInline?: string + sourceInline?: string @dagger(input) // Target S3 URL (eg. s3:////) - target: string + target: string @dagger(input) // Object content type - contentType: string | *"" + contentType: string | *"" @dagger(input) // URL of the uploaded S3 object - url: out + url: out @dagger(output) // Always write the object to S3 - always?: bool + always?: bool @dagger(input) out: string aws.#Script & { diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index 39128169..ff67e740 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -7,7 +7,9 @@ import ( // An artifact such as source code checkout, container image, binary archive... // May be passed as user input, or computed by a buildkit pipeline +// FIXME (perf). See https://github.com/dagger/dagger/issues/445 #Artifact: { + @dagger(artifact) #up: [...op.#Op] _ ... @@ -17,5 +19,6 @@ import ( // FIXME: currently aliased as a string to mark secrets // this requires proper support. #Secret: { + @dagger(secret) string | bytes } From ef8995b9233bb29a7b71a364f668696280b22cf7 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 18 May 2021 21:38:21 -0700 Subject: [PATCH 04/15] cmd/input: removed unused option Signed-off-by: Sam Alba --- cmd/dagger/cmd/input/list.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 3af7abfb..4fb95b50 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -10,7 +10,6 @@ import ( "dagger.io/go/cmd/dagger/logger" "dagger.io/go/dagger" - "cuelang.org/go/cue/ast" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -69,26 +68,7 @@ var listCmd = &cobra.Command{ fmt.Fprintln(w, "Path\tType") for _, val := range inputs { - // check for references - // this is here because it has issues - // so we wrap it in a flag to control its usage while debugging - _, vals := val.Expr() - if !viper.GetBool("keep-references") { - foundRef := false - for _, ve := range vals { - s := ve.Source() - switch s.(type) { - case *ast.Ident: - foundRef = true - } - } - if foundRef { - continue - } - } - fmt.Fprintf(w, "%s\t%v\n", val.Path(), val) - } // ensure we flush the output buf w.Flush() @@ -104,8 +84,6 @@ var listCmd = &cobra.Command{ } func init() { - listCmd.Flags().BoolP("keep-references", "R", false, "Try to eliminate references") - if err := viper.BindPFlags(listCmd.Flags()); err != nil { panic(err) } From f291994cfff140c4ff638f04e8390ce5037842be Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 19 May 2021 14:50:02 -0700 Subject: [PATCH 05/15] dagger/compiler: added value.HasAttr Signed-off-by: Sam Alba --- dagger/compiler/value.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/dagger/compiler/value.go b/dagger/compiler/value.go index 9b6e0ee5..89a590bc 100644 --- a/dagger/compiler/value.go +++ b/dagger/compiler/value.go @@ -229,3 +229,32 @@ func (v *Value) IsEmptyStruct() bool { func (v *Value) Cue() cue.Value { return v.val } + +// Returns true if value has a dagger attribute (eg. artifact, secret, input) +func (v *Value) HasAttr(filter ...string) bool { + attrs := v.val.Attributes(cue.FieldAttr) + + for _, attr := range attrs { + name := attr.Name() + // match `@dagger(...)` + if name == "dagger" { + // did not provide filter, match any @dagger attr + if len(filter) == 0 { + return true + } + + // loop over args (CSV content in attribute) + for i := 0; i < attr.NumArgs(); i++ { + key, _ := attr.Arg(i) + // one or several values where provided, filter + for _, val := range filter { + if key == val { + return true + } + } + } + } + } + + return false +} From c6e407067e343eb3b6af7961db5799c15e0f807b Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 19 May 2021 14:50:40 -0700 Subject: [PATCH 06/15] implemented input scanner Signed-off-by: Sam Alba --- dagger/environment.go | 5 ++- dagger/inputs_scan.go | 82 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 dagger/inputs_scan.go diff --git a/dagger/environment.go b/dagger/environment.go index 93e4cf7a..8e690c4c 100644 --- a/dagger/environment.go +++ b/dagger/environment.go @@ -10,7 +10,6 @@ import ( "cuelang.org/go/cue" cueflow "cuelang.org/go/tools/flow" "dagger.io/go/dagger/compiler" - "dagger.io/go/pkg/cuetils" "dagger.io/go/stdlib" "github.com/opentracing/opentracing-go" @@ -297,8 +296,8 @@ func newPipelineRunner(computed *compiler.Value, s Solver) cueflow.RunnerFunc { }) } -func (e *Environment) ScanInputs() ([]cue.Value, error) { - vals, err := cuetils.ScanForInputs(e.plan.Cue()) +func (e *Environment) ScanInputs() ([]*compiler.Value, error) { + vals, err := ScanInputs(e.plan) if err != nil { return nil, err } diff --git a/dagger/inputs_scan.go b/dagger/inputs_scan.go new file mode 100644 index 00000000..a7f7ca43 --- /dev/null +++ b/dagger/inputs_scan.go @@ -0,0 +1,82 @@ +package dagger + +import ( + "fmt" + + "cuelang.org/go/cue" + "dagger.io/go/dagger/compiler" +) + +// func isReference(val cue.Value) bool { +// _, ref := val.ReferencePath() +// isRef := len(ref.Selectors()) > 0 + +// if isRef { +// return true +// } + +// _, vals := val.Expr() +// for _, v := range vals { +// // walk recursively +// if v.Path().String() == val.Path().String() { +// // avoid loop by checking the same value +// continue +// } +// return isReference(v) +// } + +// return isRef +// } + +// walk recursively to find references +// func isReference(val cue.Value) bool { +// _, vals := val.Expr() +// for _, v := range vals { +// // walk recursively +// if v.Path().String() == val.Path().String() { +// // avoid loop by checking the same value +// continue +// } +// return isReference(v) +// } + +// _, ref := val.ReferencePath() +// return len(ref.Selectors()) > 0 +// } + +func isReference(val cue.Value) bool { + checkRef := func(vv cue.Value) bool { + _, ref := vv.ReferencePath() + return len(ref.Selectors()) > 0 + } + + _, vals := val.Expr() + for _, v := range vals { + if checkRef(v) { + return true + } + } + + return checkRef(val) +} + +func ScanInputs(value *compiler.Value) ([]*compiler.Value, error) { + vals := []*compiler.Value{} + + value.Walk( + func(val *compiler.Value) bool { + // if isReference(val.Cue()) { + // fmt.Println("### isReference ->", val.Cue().Path()) + // return false + // } + + if val.HasAttr("input") && !isReference(val.Cue()) { + fmt.Printf("#### FOUND: %s\n", val.Path()) + vals = append(vals, val) + } + return true + }, nil, + ) + + return vals, nil +} From 4edc2ad38e9d6cc3c0e12891897072b1d041cb6f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 19 May 2021 14:50:58 -0700 Subject: [PATCH 07/15] stdlib: aws/s3: removed unused input attr Signed-off-by: Sam Alba --- stdlib/aws/s3/s3.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/aws/s3/s3.cue b/stdlib/aws/s3/s3.cue index 306f2b9e..3c380979 100644 --- a/stdlib/aws/s3/s3.cue +++ b/stdlib/aws/s3/s3.cue @@ -9,7 +9,7 @@ import ( #Put: { // AWS Config - config: aws.#Config @dagger(input) + config: aws.#Config // Source Artifact to upload to S3 source?: dagger.#Artifact @dagger(input) From df564dd877051b5bb43c061b69f42cf7f41c8d6d Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 21 May 2021 16:01:13 -0700 Subject: [PATCH 08/15] added helpers Dereference, Default, Concrete options to compiler.Value Signed-off-by: Sam Alba --- dagger/compiler/value.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/dagger/compiler/value.go b/dagger/compiler/value.go index 89a590bc..ded6e810 100644 --- a/dagger/compiler/value.go +++ b/dagger/compiler/value.go @@ -145,8 +145,10 @@ func (v *Value) List() ([]*Value, error) { } // Recursive concreteness check. -func (v *Value) IsConcreteR() error { - return v.val.Validate(cue.Concrete(true)) +func (v *Value) IsConcreteR(opts ...cue.Option) error { + o := []cue.Option{cue.Concrete(true)} + o = append(o, opts...) + return v.val.Validate(o...) } func (v *Value) Walk(before func(*Value) bool, after func(*Value)) { @@ -232,7 +234,7 @@ func (v *Value) Cue() cue.Value { // Returns true if value has a dagger attribute (eg. artifact, secret, input) func (v *Value) HasAttr(filter ...string) bool { - attrs := v.val.Attributes(cue.FieldAttr) + attrs := v.val.Attributes(cue.ValueAttr) for _, attr := range attrs { name := attr.Name() @@ -258,3 +260,13 @@ func (v *Value) HasAttr(filter ...string) bool { return false } + +func (v *Value) Dereference() *Value { + dVal := cue.Dereference(v.val) + return v.cc.Wrap(dVal) +} + +func (v *Value) Default() (*Value, bool) { + val, hasDef := v.val.Default() + return v.cc.Wrap(val), hasDef +} From 40376bb214d2640d6489b0d21a261af653473749 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 21 May 2021 16:02:38 -0700 Subject: [PATCH 09/15] re-implemented ScanInputs Signed-off-by: Sam Alba --- dagger/environment.go | 9 ++--- dagger/inputs_scan.go | 83 +++++++++++++------------------------------ 2 files changed, 27 insertions(+), 65 deletions(-) diff --git a/dagger/environment.go b/dagger/environment.go index 8e690c4c..c0177676 100644 --- a/dagger/environment.go +++ b/dagger/environment.go @@ -296,11 +296,6 @@ func newPipelineRunner(computed *compiler.Value, s Solver) cueflow.RunnerFunc { }) } -func (e *Environment) ScanInputs() ([]*compiler.Value, error) { - vals, err := ScanInputs(e.plan) - if err != nil { - return nil, err - } - - return vals, nil +func (e *Environment) ScanInputs(ctx context.Context) []*compiler.Value { + return ScanInputs(ctx, e.plan) } diff --git a/dagger/inputs_scan.go b/dagger/inputs_scan.go index a7f7ca43..50edfa7f 100644 --- a/dagger/inputs_scan.go +++ b/dagger/inputs_scan.go @@ -1,82 +1,49 @@ package dagger import ( - "fmt" + "context" - "cuelang.org/go/cue" "dagger.io/go/dagger/compiler" + "github.com/rs/zerolog/log" ) -// func isReference(val cue.Value) bool { -// _, ref := val.ReferencePath() -// isRef := len(ref.Selectors()) > 0 +func isReference(val *compiler.Value) bool { + _, ref := val.ReferencePath() -// if isRef { -// return true -// } - -// _, vals := val.Expr() -// for _, v := range vals { -// // walk recursively -// if v.Path().String() == val.Path().String() { -// // avoid loop by checking the same value -// continue -// } -// return isReference(v) -// } - -// return isRef -// } - -// walk recursively to find references -// func isReference(val cue.Value) bool { -// _, vals := val.Expr() -// for _, v := range vals { -// // walk recursively -// if v.Path().String() == val.Path().String() { -// // avoid loop by checking the same value -// continue -// } -// return isReference(v) -// } - -// _, ref := val.ReferencePath() -// return len(ref.Selectors()) > 0 -// } - -func isReference(val cue.Value) bool { - checkRef := func(vv cue.Value) bool { - _, ref := vv.ReferencePath() - return len(ref.Selectors()) > 0 + if ref.String() == "" || val.Path().String() == ref.String() { + return false } - _, vals := val.Expr() - for _, v := range vals { - if checkRef(v) { - return true + for _, s := range ref.Selectors() { + if s.IsDefinition() { + return false } } - return checkRef(val) + return true } -func ScanInputs(value *compiler.Value) ([]*compiler.Value, error) { - vals := []*compiler.Value{} +func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value { + lg := log.Ctx(ctx) + inputs := []*compiler.Value{} value.Walk( func(val *compiler.Value) bool { - // if isReference(val.Cue()) { - // fmt.Println("### isReference ->", val.Cue().Path()) - // return false - // } - - if val.HasAttr("input") && !isReference(val.Cue()) { - fmt.Printf("#### FOUND: %s\n", val.Path()) - vals = append(vals, val) + if isReference(val) { + lg.Debug().Str("value.Path", val.Path().String()).Msg("found reference, stop walk") + return false } + + if !val.HasAttr("input") { + return true + } + + lg.Debug().Str("value.Path", val.Path().String()).Msg("found input") + inputs = append(inputs, val) + return true }, nil, ) - return vals, nil + return inputs } From c86f75f730a7b0ebf272796164162ce43b3bf512 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 21 May 2021 16:03:22 -0700 Subject: [PATCH 10/15] cmd/input: implemented UI for list cmd Signed-off-by: Sam Alba --- cmd/dagger/cmd/input/list.go | 63 ++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 4fb95b50..7d7b60b4 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -9,6 +9,7 @@ import ( "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" "dagger.io/go/dagger" + "dagger.io/go/dagger/compiler" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -36,17 +37,6 @@ var listCmd = &cobra.Command{ environment := common.GetCurrentEnvironmentState(ctx, store) - // print any persisted inputs - if len(environment.Inputs) > 0 { - fmt.Println("Saved Inputs:") - for _, input := range environment.Inputs { - // Todo, how to pull apart an input to print relevant information - fmt.Printf("%s: %v\n", input.Key, input.Value) - } - // add some space - fmt.Println() - } - lg = lg.With(). Str("environmentName", environment.Name). Str("environmentId", environment.ID). @@ -58,21 +48,32 @@ var listCmd = &cobra.Command{ } _, err = c.Do(ctx, environment, func(lCtx context.Context, lDeploy *dagger.Environment, lSolver dagger.Solver) error { - inputs, err := lDeploy.ScanInputs() - if err != nil { - return err - } + inputs := lDeploy.ScanInputs(ctx) - fmt.Println("Plan Inputs:") w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) - fmt.Fprintln(w, "Path\tType") + fmt.Fprintln(w, "Input\tType\tValue\tSet by user\tSet in plan") - for _, val := range inputs { - fmt.Fprintf(w, "%s\t%v\n", val.Path(), val) + for _, inp := range inputs { + isConcrete := (inp.IsConcreteR() == nil) + _, hasDefault := inp.Default() + valStr := "-" + if isConcrete { + valStr, _ = inp.Cue().String() + } + if hasDefault { + valStr = fmt.Sprintf("%s (default)", valStr) + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%t\t%t\n", + inp.Path(), + getType(inp), + valStr, + isUserSet(environment, inp), + isConcrete, + ) } - // ensure we flush the output buf - w.Flush() + w.Flush() return nil }) @@ -83,6 +84,26 @@ var listCmd = &cobra.Command{ }, } +func isUserSet(env *dagger.EnvironmentState, val *compiler.Value) bool { + for _, i := range env.Inputs { + if val.Path().String() == i.Key { + return true + } + } + + return false +} + +func getType(val *compiler.Value) string { + if val.HasAttr("artifact") { + return "dagger.#Artifact" + } + if val.HasAttr("secret") { + return "dagger.#Secret" + } + return val.Cue().IncompleteKind().String() +} + func init() { if err := viper.BindPFlags(listCmd.Flags()); err != nil { panic(err) From 223e3f66814f2ac48b4ee4f1d69642e0d130fb81 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 21 May 2021 16:06:21 -0700 Subject: [PATCH 11/15] cue fmt Signed-off-by: Sam Alba --- examples/simple-s3/main.cue | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/simple-s3/main.cue b/examples/simple-s3/main.cue index 06e3e8f3..122a11d8 100644 --- a/examples/simple-s3/main.cue +++ b/examples/simple-s3/main.cue @@ -9,8 +9,6 @@ import ( // AWS Config for credentials and default region awsConfig: aws.#Config & { region: *"us-east-1" | string @dagger(input) - accessKey: dagger.#Secret @dagger(input) - secretKey: dagger.#Secret @dagger(input) } // Name of the S3 bucket to use From 57a168e6b51c0166d230ed1ff8bccf76af3a815f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 21 May 2021 16:10:07 -0700 Subject: [PATCH 12/15] pkg: removed unused code Signed-off-by: Sam Alba --- pkg/cuetils/scan.go | 138 -------------------------------------------- 1 file changed, 138 deletions(-) delete mode 100644 pkg/cuetils/scan.go diff --git a/pkg/cuetils/scan.go b/pkg/cuetils/scan.go deleted file mode 100644 index 0e327554..00000000 --- a/pkg/cuetils/scan.go +++ /dev/null @@ -1,138 +0,0 @@ -package cuetils - -import ( - "cuelang.org/go/cue" -) - -// ScanForInputs walks a Value looking for potential inputs -// - non-concrete values or values with defaults -// - exclude @dagger(computed) and #up -// - exclude values which have references -func ScanForInputs(value cue.Value) ([]cue.Value, error) { - var ( - vals []cue.Value - err error - ) - - // walk before function, bool return true if the walk should recurse again - before := func(v cue.Value) (bool, error) { - // inference phase - switch v.IncompleteKind() { - case cue.StructKind: - return true, nil - - case cue.ListKind: - if !v.IsConcrete() && isInput(v) { - vals = append(vals, v) - return false, nil - } - return true, nil - - default: - - if !isInput(v) { - // Not an input - return false, nil - } - - // a leaf with default? - _, has := v.Default() - if has { - vals = append(vals, v) - // recurse here? - return false, nil - } - - // is this leaf not concrete? (should cause an error) - if v.Validate(cue.Concrete(true), cue.Optional(true)) != nil { - vals = append(vals, v) - } - - return false, nil - } - } - - // walk - err = walkValue(value, before, nil) - if err != nil { - return nil, err - } - - return vals, nil -} - -func isInput(v cue.Value) bool { - attrs := v.Attributes(cue.ValueAttr) - - for _, attr := range attrs { - name := attr.Name() - // match `@dagger(...)` - if name == "dagger" { - // loop over args (CSV content in attribute) - for i := 0; i < attr.NumArgs(); i++ { - key, _ := attr.Arg(i) - - // we found an explicit input value - if key == "input" { - return true - } - } - } - } - - return false -} - -// walkValue is a custome walk function so that we recurse into more types than CUE's buildin walk -// specificially, we need to customize the options to val.Fields when val is a struct -func walkValue(val cue.Value, before func(cue.Value) (bool, error), after func(cue.Value) error) error { - if before != nil { - recurse, err := before(val) - if err != nil { - return err - } - - // should we recurse into fields - if recurse { - switch val.IncompleteKind() { - case cue.StructKind: - // provide custom args to ensure we walk nested defs - // and that optionals are included - iter, err := val.Fields( - cue.Definitions(true), - cue.Optional(true), - ) - if err != nil { - return err - } - for iter.Next() { - err := walkValue(iter.Value(), before, after) - if err != nil { - return err - } - } - - case cue.ListKind: - iter, err := val.List() - if err != nil { - return err - } - for iter.Next() { - err := walkValue(iter.Value(), before, after) - if err != nil { - return err - } - } - } - } - } - - if after != nil { - err := after(val) - if err != nil { - return err - } - } - - return nil -} From 7484df45cd690a6ff5c234ef943d0f0faf9ff48b Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 21 May 2021 16:56:30 -0700 Subject: [PATCH 13/15] clean up comments Signed-off-by: Sam Alba --- dagger/inputs_scan.go | 2 ++ stdlib/dagger/dagger.cue | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dagger/inputs_scan.go b/dagger/inputs_scan.go index 50edfa7f..a76f54a1 100644 --- a/dagger/inputs_scan.go +++ b/dagger/inputs_scan.go @@ -11,11 +11,13 @@ func isReference(val *compiler.Value) bool { _, ref := val.ReferencePath() if ref.String() == "" || val.Path().String() == ref.String() { + // not a reference return false } for _, s := range ref.Selectors() { if s.IsDefinition() { + // if we reference to a definition, we skip the check return false } } diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index ff67e740..3f9bcdd2 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -7,7 +7,6 @@ import ( // An artifact such as source code checkout, container image, binary archive... // May be passed as user input, or computed by a buildkit pipeline -// FIXME (perf). See https://github.com/dagger/dagger/issues/445 #Artifact: { @dagger(artifact) #up: [...op.#Op] From 6b73de7ea315ea41377383773d4bbcaa56c34eaf Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 21 May 2021 17:38:33 -0700 Subject: [PATCH 14/15] detect reference on cue conjunction Signed-off-by: Sam Alba --- dagger/inputs_scan.go | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/dagger/inputs_scan.go b/dagger/inputs_scan.go index a76f54a1..2b04becc 100644 --- a/dagger/inputs_scan.go +++ b/dagger/inputs_scan.go @@ -3,26 +3,43 @@ package dagger import ( "context" + "cuelang.org/go/cue" "dagger.io/go/dagger/compiler" "github.com/rs/zerolog/log" ) -func isReference(val *compiler.Value) bool { - _, ref := val.ReferencePath() +func isReference(val cue.Value) bool { + isRef := func(v cue.Value) bool { + _, ref := v.ReferencePath() - if ref.String() == "" || val.Path().String() == ref.String() { - // not a reference - return false + if ref.String() == "" || v.Path().String() == ref.String() { + // not a reference + return false + } + + for _, s := range ref.Selectors() { + if s.IsDefinition() { + // if we reference to a definition, we skip the check + return false + } + } + + return true } - for _, s := range ref.Selectors() { - if s.IsDefinition() { - // if we reference to a definition, we skip the check - return false + op, vals := val.Expr() + if op == cue.NoOp { + return isRef(val) + } + + for _, v := range vals { + // if the expr has an op (& or |, etc...), check the expr values, recursively + if isReference(v) { + return true } } - return true + return isRef(val) } func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value { @@ -31,7 +48,7 @@ func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value { value.Walk( func(val *compiler.Value) bool { - if isReference(val) { + if isReference(val.Cue()) { lg.Debug().Str("value.Path", val.Path().String()).Msg("found reference, stop walk") return false } From a8837aaaedb0032a1de3ebe8d30e8246881f7f5e Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 21 May 2021 17:39:17 -0700 Subject: [PATCH 15/15] cmd/inputs: added --all option to list Signed-off-by: Sam Alba --- cmd/dagger/cmd/input/list.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 7d7b60b4..ab752235 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -51,7 +51,7 @@ var listCmd = &cobra.Command{ inputs := lDeploy.ScanInputs(ctx) w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) - fmt.Fprintln(w, "Input\tType\tValue\tSet by user\tSet in plan") + fmt.Fprintln(w, "Input\tType\tValue\tSet by user") for _, inp := range inputs { isConcrete := (inp.IsConcreteR() == nil) @@ -64,12 +64,18 @@ var listCmd = &cobra.Command{ valStr = fmt.Sprintf("%s (default)", valStr) } - fmt.Fprintf(w, "%s\t%s\t%s\t%t\t%t\n", + if !viper.GetBool("all") { + // skip input that is not overridable + if !hasDefault && isConcrete { + continue + } + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%t\n", inp.Path(), getType(inp), valStr, isUserSet(environment, inp), - isConcrete, ) } @@ -105,6 +111,8 @@ func getType(val *compiler.Value) string { } func init() { + listCmd.Flags().BoolP("all", "a", false, "List all inputs (include non-overridable)") + if err := viper.BindPFlags(listCmd.Flags()); err != nil { panic(err) }