From 95e3819164afb8245121f1f9a29ff67b061b5a8d Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 26 Apr 2021 10:21:26 -0700 Subject: [PATCH 1/3] bump cue to v0.4.0-alpha.2 Signed-off-by: Andrea Luzzardi --- go.mod | 2 +- go.sum | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 30c0a073..9cfea836 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module dagger.io/go go 1.16 require ( - cuelang.org/go v0.3.2 + cuelang.org/go v0.4.0-alpha.2 github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db github.com/containerd/console v1.0.2 diff --git a/go.sum b/go.sum index 33126a20..0e01d7bc 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrL contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= -cuelang.org/go v0.3.2 h1:/Am5yFDwqnaEi+g942OPM1M4/qtfVSm49wtkQbeh5Z4= -cuelang.org/go v0.3.2/go.mod h1:jvMO35Q4D2D3m2ujAmKESICaYkjMbu5+D+2zIGuWTpQ= +cuelang.org/go v0.4.0-alpha.2 h1:lifRw7DYPNlCcuHltBxZgvqDEDtIiIX0RjW9cgSITKQ= +cuelang.org/go v0.4.0-alpha.2/go.mod h1:IDCcM/+MZjfpt6IypYE3uWkJVGRWHWqwvi2zwTKBtxI= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/age v1.0.0-beta7 h1:RZiSK+N3KL2UwT82xiCavjYw8jJHzWMEUYePAukTpk0= @@ -416,6 +416,7 @@ github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -862,6 +863,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc h1:gSVONBi2HWMFXCa9jFdYvYk7IwW/mTLxWOF7rXS4LO0= +github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc/go.mod h1:KbKfKPy2I6ecOIGA9apfheFv14+P3RSmmQvshofQyMY= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/quasilyte/go-ruleguard v0.1.2-0.20200318202121-b00d7a75d3d8/go.mod h1:CGFX09Ci3pq9QZdj86B+VGIdNj4VyCo2iPOGS9esB/k= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= From 82b08b7d8543ff96df131f4d8779768d6fdf32fb Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 30 Apr 2021 18:05:37 -0700 Subject: [PATCH 2/3] refactor for cue 0.4 cue deprecated the use of instances in favor of values. Refactored the code to accomodate the new changes. Signed-off-by: Andrea Luzzardi --- dagger/compiler/build.go | 9 +++- dagger/compiler/compiler.go | 91 ++++++++++++-------------------- dagger/compiler/compiler_test.go | 6 +-- dagger/compiler/json.go | 33 ------------ dagger/compiler/value.go | 49 ++++++++--------- dagger/deployment.go | 35 ++++++------ dagger/pipeline.go | 8 +-- 7 files changed, 89 insertions(+), 142 deletions(-) diff --git a/dagger/compiler/build.go b/dagger/compiler/build.go index eec050ad..f20da154 100644 --- a/dagger/compiler/build.go +++ b/dagger/compiler/build.go @@ -13,6 +13,8 @@ import ( // Build a cue configuration tree from the files in fs. func Build(sources map[string]fs.FS, args ...string) (*Value, error) { + c := DefaultCompiler + buildConfig := &cueload.Config{ // The CUE overlay needs to be prefixed by a non-conflicting path with the // local filesystem, otherwise Cue will merge the Overlay with whatever Cue @@ -55,9 +57,12 @@ func Build(sources map[string]fs.FS, args ...string) (*Value, error) { if len(instances) != 1 { return nil, errors.New("only one package is supported at a time") } - inst, err := Cue().Build(instances[0]) + v, err := c.Context.BuildInstances(instances) if err != nil { return nil, errors.New(cueerrors.Details(err, &cueerrors.Config{})) } - return Wrap(inst.Value(), inst), nil + if len(v) != 1 { + return nil, errors.New("internal: wrong number of values") + } + return Wrap(v[0]), nil } diff --git a/dagger/compiler/compiler.go b/dagger/compiler/compiler.go index f750136c..0f3c5784 100644 --- a/dagger/compiler/compiler.go +++ b/dagger/compiler/compiler.go @@ -2,10 +2,10 @@ package compiler import ( "errors" - "fmt" "sync" "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" cueerrors "cuelang.org/go/cue/errors" cuejson "cuelang.org/go/encoding/json" cueyaml "cuelang.org/go/encoding/yaml" @@ -13,10 +13,10 @@ import ( var ( // DefaultCompiler is the default Compiler and is used by Compile - DefaultCompiler = &Compiler{} + DefaultCompiler = New() ) -func Compile(name string, src interface{}) (*Value, error) { +func Compile(name string, src string) (*Value, error) { return DefaultCompiler.Compile(name, src) } @@ -25,16 +25,8 @@ func NewValue() *Value { } // FIXME can be refactored away now? -func Wrap(v cue.Value, inst *cue.Instance) *Value { - return DefaultCompiler.Wrap(v, inst) -} - -func InstanceMerge(src ...*Value) (*Value, error) { - return DefaultCompiler.InstanceMerge(src...) -} - -func Cue() *cue.Runtime { - return DefaultCompiler.Cue() +func Wrap(v cue.Value) *Value { + return DefaultCompiler.Wrap(v) } func Err(err error) error { @@ -54,7 +46,13 @@ func DecodeYAML(path string, data []byte) (*Value, error) { // Use this instead of cue.Runtime type Compiler struct { l sync.RWMutex - cue.Runtime + *cue.Context +} + +func New() *Compiler { + return &Compiler{ + Context: cuecontext.New(), + } } func (c *Compiler) lock() { @@ -73,10 +71,6 @@ func (c *Compiler) runlock() { c.l.RUnlock() } -func (c *Compiler) Cue() *cue.Runtime { - return &(c.Runtime) -} - // Compile an empty value func (c *Compiler) NewValue() *Value { empty, err := c.Compile("", "_") @@ -86,64 +80,47 @@ func (c *Compiler) NewValue() *Value { return empty } -func (c *Compiler) Compile(name string, src interface{}) (*Value, error) { +func (c *Compiler) Compile(name string, src string) (*Value, error) { c.lock() defer c.unlock() - inst, err := c.Cue().Compile(name, src) - if err != nil { + v := c.Context.CompileString(src, cue.Filename(name)) + if v.Err() != nil { // FIXME: cleaner way to unwrap cue error details? - return nil, Err(err) + return nil, Err(v.Err()) } - return c.Wrap(inst.Value(), inst), nil -} - -// InstanceMerge merges multiple values and mirrors the value in the cue.Instance. -// FIXME: AVOID THIS AT ALL COST -// Special case: we must return an instance with the same -// contents as v, for the purposes of cueflow. -func (c *Compiler) InstanceMerge(src ...*Value) (*Value, error) { - var ( - v = c.NewValue() - inst = v.CueInst() - err error - ) - - c.lock() - defer c.unlock() - - for _, s := range src { - inst, err = inst.Fill(s.val) - if err != nil { - return nil, fmt.Errorf("merge failed: %w", err) - } - if err := inst.Value().Err(); err != nil { - return nil, err - } - } - - v = c.Wrap(inst.Value(), inst) - return v, nil + return c.Wrap(v), nil } func (c *Compiler) DecodeJSON(path string, data []byte) (*Value, error) { - inst, err := cuejson.Decode(c.Cue(), path, data) + expr, err := cuejson.Extract(path, data) if err != nil { return nil, Err(err) } - return c.Wrap(inst.Value(), inst), nil + v := c.Context.BuildExpr(expr, cue.Filename(path)) + if err := v.Err(); err != nil { + return nil, Err(err) + } + return c.Wrap(v), nil } func (c *Compiler) DecodeYAML(path string, data []byte) (*Value, error) { - inst, err := cueyaml.Decode(c.Cue(), path, data) + f, err := cueyaml.Extract(path, data) if err != nil { return nil, Err(err) } - return c.Wrap(inst.Value(), inst), nil + v := c.Context.BuildFile(f, cue.Filename(path)) + if err := v.Err(); err != nil { + return nil, Err(err) + } + return c.Wrap(v), nil } -func (c *Compiler) Wrap(v cue.Value, inst *cue.Instance) *Value { - return wrapValue(v, inst, c) +func (c *Compiler) Wrap(v cue.Value) *Value { + return &Value{ + val: v, + cc: c, + } } func (c *Compiler) Err(err error) error { diff --git a/dagger/compiler/compiler_test.go b/dagger/compiler/compiler_test.go index 7b21d947..344d5720 100644 --- a/dagger/compiler/compiler_test.go +++ b/dagger/compiler/compiler_test.go @@ -8,7 +8,7 @@ import ( // Test that a non-existing field is detected correctly func TestFieldNotExist(t *testing.T) { - c := &Compiler{} + c := New() root, err := c.Compile("test.cue", `foo: "bar"`) require.NoError(t, err) require.True(t, root.Lookup("foo").Exists()) @@ -17,7 +17,7 @@ func TestFieldNotExist(t *testing.T) { // Test that a non-existing definition is detected correctly func TestDefNotExist(t *testing.T) { - c := &Compiler{} + c := New() root, err := c.Compile("test.cue", `foo: #bla: "bar"`) require.NoError(t, err) require.True(t, root.Lookup("foo.#bla").Exists()) @@ -25,7 +25,7 @@ func TestDefNotExist(t *testing.T) { } func TestJSON(t *testing.T) { - c := &Compiler{} + c := New() v, err := c.Compile("", `foo: hello: "world"`) require.NoError(t, err) require.Equal(t, `{"foo":{"hello":"world"}}`, string(v.JSON())) diff --git a/dagger/compiler/json.go b/dagger/compiler/json.go index fd2f59d7..e138f7af 100644 --- a/dagger/compiler/json.go +++ b/dagger/compiler/json.go @@ -5,8 +5,6 @@ import ( "encoding/json" "fmt" - "cuelang.org/go/cue" - cuejson "cuelang.org/go/encoding/json" "github.com/KromDaniel/jonson" ) @@ -100,37 +98,6 @@ func (s JSON) Set(valueJSON []byte, path ...string) (JSON, error) { return root.ToJSON() } -func (s JSON) Merge(layers ...JSON) (JSON, error) { - r := new(cue.Runtime) - var resultInst *cue.Instance - for i, l := range append([]JSON{s}, layers...) { - if l == nil { - continue - } - filename := fmt.Sprintf("%d", i) - inst, err := cuejson.Decode(r, filename, []byte(l)) - if err != nil { - return nil, err - } - if resultInst == nil { - resultInst = inst - } else { - resultInst, err = resultInst.Fill(inst.Value()) - if err != nil { - return nil, err - } - if resultInst.Err != nil { - return nil, resultInst.Err - } - } - } - b, err := resultInst.Value().MarshalJSON() - if err != nil { - return nil, err - } - return JSON(b), nil -} - func (s JSON) String() string { if s == nil { return "{}" diff --git a/dagger/compiler/value.go b/dagger/compiler/value.go index 964cce8d..3f20f50e 100644 --- a/dagger/compiler/value.go +++ b/dagger/compiler/value.go @@ -2,6 +2,7 @@ package compiler import ( "sort" + "strconv" "cuelang.org/go/cue" cueformat "cuelang.org/go/cue/format" @@ -10,25 +11,8 @@ import ( // Value is a wrapper around cue.Value. // Use instead of cue.Value and cue.Instance type Value struct { - val cue.Value - cc *Compiler - inst *cue.Instance -} - -func (v *Value) CueInst() *cue.Instance { - return v.inst -} - -func (v *Value) Wrap(v2 cue.Value) *Value { - return wrapValue(v2, v.inst, v.cc) -} - -func wrapValue(v cue.Value, inst *cue.Instance, cc *Compiler) *Value { - return &Value{ - val: v, - cc: cc, - inst: inst, - } + val cue.Value + cc *Compiler } // FillPath fills the value in-place @@ -51,7 +35,7 @@ func (v *Value) LookupPath(p cue.Path) *Value { v.cc.rlock() defer v.cc.runlock() - return v.Wrap(v.val.LookupPath(p)) + return v.cc.Wrap(v.val.LookupPath(p)) } // Lookup is a helper function to lookup by path parts. @@ -71,8 +55,17 @@ func (v *Value) Kind() cue.Kind { // Field represents a struct field type Field struct { - Label string - Value *Value + Selector cue.Selector + Value *Value +} + +// Label returns the unquoted selector +func (f Field) Label() string { + l := f.Selector.String() + if unquoted, err := strconv.Unquote(l); err == nil { + return unquoted + } + return l } // Proxy function to the underlying cue.Value @@ -86,13 +79,13 @@ func (v *Value) Fields() ([]Field, error) { fields := []Field{} for it.Next() { fields = append(fields, Field{ - Label: it.Label(), - Value: v.Wrap(it.Value()), + Selector: it.Selector(), + Value: v.cc.Wrap(it.Value()), }) } sort.SliceStable(fields, func(i, j int) bool { - return fields[i].Label < fields[j].Label + return fields[i].Selector.String() < fields[j].Selector.String() }) return fields, nil @@ -140,7 +133,7 @@ func (v *Value) List() ([]*Value, error) { return nil, err } for it.Next() { - l = append(l, v.Wrap(it.Value())) + l = append(l, v.cc.Wrap(it.Value())) } return l, nil @@ -159,12 +152,12 @@ func (v *Value) Walk(before func(*Value) bool, after func(*Value)) { ) if before != nil { llBefore = func(child cue.Value) bool { - return before(v.Wrap(child)) + return before(v.cc.Wrap(child)) } } if after != nil { llAfter = func(child cue.Value) { - after(v.Wrap(child)) + after(v.cc.Wrap(child)) } } v.val.Walk(llBefore, llAfter) diff --git a/dagger/deployment.go b/dagger/deployment.go index 258c48fd..bbbb97c2 100644 --- a/dagger/deployment.go +++ b/dagger/deployment.go @@ -142,17 +142,20 @@ func (d *Environment) LocalDirs() map[string]string { } // 1. Scan the environment state // FIXME: use a common `flow` instance to avoid rescanning the tree. - src, err := compiler.InstanceMerge(d.plan, d.input) - if err != nil { - panic(err) + src := compiler.NewValue() + if err := src.FillPath(cue.MakePath(), d.plan); err != nil { + return nil + } + if err := src.FillPath(cue.MakePath(), d.input); err != nil { + return nil } flow := cueflow.New( &cueflow.Config{}, - src.CueInst(), - newTaskFunc(src.CueInst(), noOpRunner), + src.Cue(), + newTaskFunc(noOpRunner), ) for _, t := range flow.Tasks() { - v := compiler.Wrap(t.Value(), src.CueInst()) + v := compiler.Wrap(t.Value()) localdirs(v.Lookup("#up")) } @@ -173,17 +176,19 @@ func (d *Environment) Up(ctx context.Context, s Solver) error { // Reset the computed values d.computed = compiler.NewValue() - // Cueflow cue instance - src, err := compiler.InstanceMerge(d.plan, d.input) - if err != nil { + src := compiler.NewValue() + if err := src.FillPath(cue.MakePath(), d.plan); err != nil { + return err + } + if err := src.FillPath(cue.MakePath(), d.input); err != nil { return err } // Orchestrate execution with cueflow flow := cueflow.New( &cueflow.Config{}, - src.CueInst(), - newTaskFunc(src.CueInst(), newPipelineRunner(src.CueInst(), d.computed, s)), + src.Cue(), + newTaskFunc(newPipelineRunner(d.computed, s)), ) if err := flow.Run(ctx); err != nil { return err @@ -200,9 +205,9 @@ func (d *Environment) Down(ctx context.Context, _ *DownOpts) error { type QueryOpts struct{} -func newTaskFunc(inst *cue.Instance, runner cueflow.RunnerFunc) cueflow.TaskFunc { +func newTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc { return func(flowVal cue.Value) (cueflow.Runner, error) { - v := compiler.Wrap(flowVal, inst) + v := compiler.Wrap(flowVal) if !isComponent(v) { // No compute script return nil, nil @@ -215,7 +220,7 @@ func noOpRunner(t *cueflow.Task) error { return nil } -func newPipelineRunner(inst *cue.Instance, computed *compiler.Value, s Solver) cueflow.RunnerFunc { +func newPipelineRunner(computed *compiler.Value, s Solver) cueflow.RunnerFunc { return cueflow.RunnerFunc(func(t *cueflow.Task) error { ctx := t.Context() lg := log. @@ -239,7 +244,7 @@ func newPipelineRunner(inst *cue.Instance, computed *compiler.Value, s Solver) c Str("dependency", dep.Path().String()). Msg("dependency detected") } - v := compiler.Wrap(t.Value(), inst) + v := compiler.Wrap(t.Value()) p := NewPipeline(t.Path().String(), s) err := p.Do(ctx, v) if err != nil { diff --git a/dagger/pipeline.go b/dagger/pipeline.go index 12e178ef..80112524 100644 --- a/dagger/pipeline.go +++ b/dagger/pipeline.go @@ -382,7 +382,7 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value, st llb.State) ( if err != nil { return st, err } - opts = append(opts, llb.AddEnv(env.Label, v)) + opts = append(opts, llb.AddEnv(env.Label(), v)) } } @@ -420,7 +420,7 @@ func (p *Pipeline) mountAll(ctx context.Context, mounts *compiler.Value) ([]llb. return nil, err } for _, mnt := range fields { - o, err := p.mount(ctx, mnt.Label, mnt.Value) + o, err := p.mount(ctx, mnt.Label(), mnt.Value) if err != nil { return nil, err } @@ -825,7 +825,7 @@ func dockerBuildOpts(op *compiler.Value) (map[string]string, error) { if err != nil { return nil, err } - opts["build-arg:"+buildArg.Label] = v + opts["build-arg:"+buildArg.Label()] = v } } @@ -839,7 +839,7 @@ func dockerBuildOpts(op *compiler.Value) (map[string]string, error) { if err != nil { return nil, err } - opts["label:"+label.Label] = s + opts["label:"+label.Label()] = s } } From c32cabf488da541c62f0f27f8a3c0bc8c14cf510 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 30 Apr 2021 18:13:34 -0700 Subject: [PATCH 3/3] list: remove call to Reference as it is deprecated Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/input/list.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 0cb81a48..3af7abfb 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -66,7 +66,7 @@ var listCmd = &cobra.Command{ fmt.Println("Plan Inputs:") w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) - fmt.Fprintln(w, "Path\tFrom\tType") + fmt.Fprintln(w, "Path\tType") for _, val := range inputs { // check for references @@ -87,15 +87,7 @@ var listCmd = &cobra.Command{ } } - // Construct output as a tab-table - // get path / pkg import (if available) - inst, _ := val.Reference() - pkg := "(plan)" - if inst != nil { - pkg = inst.ImportPath - } - - fmt.Fprintf(w, "%s\t%s\t%v\n", val.Path(), pkg, val) + fmt.Fprintf(w, "%s\t%v\n", val.Path(), val) } // ensure we flush the output buf