diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index 4c0aae01..5be2c947 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -3,7 +3,9 @@ package common import ( "context" - "dagger.io/go/dagger" + "dagger.io/go/dagger/client" + "dagger.io/go/dagger/environment" + "dagger.io/go/dagger/solver" "dagger.io/go/dagger/state" "github.com/rs/zerolog/log" "github.com/spf13/viper" @@ -79,14 +81,14 @@ func CurrentEnvironmentState(ctx context.Context, workspace *state.Workspace) *s } // Re-compute an environment (equivalent to `dagger up`). -func EnvironmentUp(ctx context.Context, state *state.State, noCache bool) *dagger.Environment { +func EnvironmentUp(ctx context.Context, state *state.State, noCache bool) *environment.Environment { lg := log.Ctx(ctx) - c, err := dagger.NewClient(ctx, "", noCache) + c, err := client.New(ctx, "", noCache) if err != nil { lg.Fatal().Err(err).Msg("unable to create client") } - result, err := c.Do(ctx, state, func(ctx context.Context, environment *dagger.Environment, s dagger.Solver) error { + result, err := c.Do(ctx, state, func(ctx context.Context, environment *environment.Environment, s solver.Solver) error { log.Ctx(ctx).Debug().Msg("bringing environment up") return environment.Up(ctx, s) }) diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index bc1faeb8..313f68e9 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -8,8 +8,10 @@ import ( "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/client" "dagger.io/go/dagger/compiler" + "dagger.io/go/dagger/environment" + "dagger.io/go/dagger/solver" "dagger.io/go/dagger/state" "github.com/spf13/cobra" @@ -32,19 +34,19 @@ var listCmd = &cobra.Command{ ctx := lg.WithContext(cmd.Context()) workspace := common.CurrentWorkspace(ctx) - environment := common.CurrentEnvironmentState(ctx, workspace) + st := common.CurrentEnvironmentState(ctx, workspace) lg = lg.With(). - Str("environment", environment.Name). + Str("environment", st.Name). Logger() - c, err := dagger.NewClient(ctx, "", false) + c, err := client.New(ctx, "", false) if err != nil { lg.Fatal().Err(err).Msg("unable to create client") } - _, err = c.Do(ctx, environment, func(lCtx context.Context, lDeploy *dagger.Environment, lSolver dagger.Solver) error { - inputs := lDeploy.ScanInputs(ctx) + _, err = c.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + inputs := env.ScanInputs(ctx) w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) fmt.Fprintln(w, "Input\tType\tValue\tSet by user") @@ -71,7 +73,7 @@ var listCmd = &cobra.Command{ inp.Path(), getType(inp), valStr, - isUserSet(environment, inp), + isUserSet(st, inp), ) } diff --git a/cmd/dagger/cmd/query.go b/cmd/dagger/cmd/query.go index 2c689d5e..2149fc8a 100644 --- a/cmd/dagger/cmd/query.go +++ b/cmd/dagger/cmd/query.go @@ -6,7 +6,7 @@ import ( "cuelang.org/go/cue" "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/client" "dagger.io/go/dagger/compiler" "github.com/spf13/cobra" @@ -42,7 +42,7 @@ var queryCmd = &cobra.Command{ cuePath = cue.ParsePath(args[0]) } - c, err := dagger.NewClient(ctx, "", false) + c, err := client.New(ctx, "", false) if err != nil { lg.Fatal().Err(err).Msg("unable to create client") } diff --git a/dagger/client.go b/dagger/client/client.go similarity index 79% rename from dagger/client.go rename to dagger/client/client.go index 3fcd8ff3..b732b483 100644 --- a/dagger/client.go +++ b/dagger/client/client.go @@ -1,7 +1,8 @@ -package dagger +package client import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -26,6 +27,8 @@ import ( "dagger.io/go/pkg/progressui" "dagger.io/go/dagger/compiler" + "dagger.io/go/dagger/environment" + "dagger.io/go/dagger/solver" "dagger.io/go/dagger/state" ) @@ -35,7 +38,7 @@ type Client struct { noCache bool } -func NewClient(ctx context.Context, host string, noCache bool) (*Client, error) { +func New(ctx context.Context, host string, noCache bool) (*Client, error) { if host == "" { host = os.Getenv("BUILDKIT_HOST") } @@ -61,14 +64,14 @@ func NewClient(ctx context.Context, host string, noCache bool) (*Client, error) }, nil } -type ClientDoFunc func(context.Context, *Environment, Solver) error +type ClientDoFunc func(context.Context, *environment.Environment, solver.Solver) error // FIXME: return completed *Route, instead of *compiler.Value -func (c *Client) Do(ctx context.Context, state *state.State, fn ClientDoFunc) (*Environment, error) { +func (c *Client) Do(ctx context.Context, state *state.State, fn ClientDoFunc) (*environment.Environment, error) { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) - environment, err := NewEnvironment(state) + environment, err := environment.New(state) if err != nil { return nil, err } @@ -90,11 +93,11 @@ func (c *Client) Do(ctx context.Context, state *state.State, fn ClientDoFunc) (* return environment, eg.Wait() } -func (c *Client) buildfn(ctx context.Context, environment *Environment, fn ClientDoFunc, ch chan *bk.SolveStatus) error { +func (c *Client) buildfn(ctx context.Context, env *environment.Environment, fn ClientDoFunc, ch chan *bk.SolveStatus) error { lg := log.Ctx(ctx) // Scan local dirs to grant access - localdirs := environment.LocalDirs() + localdirs := env.LocalDirs() for label, dir := range localdirs { abs, err := filepath.Abs(dir) if err != nil { @@ -104,7 +107,7 @@ func (c *Client) buildfn(ctx context.Context, environment *Environment, fn Clien } // buildkit auth provider (registry) - auth := newRegistryAuthProvider() + auth := solver.NewRegistryAuthProvider() // Setup solve options opts := bk.SolveOpt{ @@ -119,16 +122,22 @@ func (c *Client) buildfn(ctx context.Context, environment *Environment, fn Clien Msg("spawning buildkit job") resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) { - s := NewSolver(c.c, gw, ch, auth, c.noCache) + s := solver.New(solver.Opts{ + Control: c.c, + Gateway: gw, + Events: ch, + Auth: auth, + NoCache: c.noCache, + }) lg.Debug().Msg("loading configuration") - if err := environment.LoadPlan(ctx, s); err != nil { + if err := env.LoadPlan(ctx, s); err != nil { return nil, err } // Compute output overlay if fn != nil { - if err := fn(ctx, environment, s); err != nil { + if err := fn(ctx, env, s); err != nil { return nil, compiler.Err(err) } } @@ -139,7 +148,7 @@ func (c *Client) buildfn(ctx context.Context, environment *Environment, fn Clien span, _ := opentracing.StartSpanFromContext(ctx, "Environment.Export") defer span.Finish() - computed := environment.Computed().JSON().PrettyString() + computed := env.Computed().JSON().PrettyString() st := llb. Scratch(). File( @@ -234,3 +243,22 @@ func (c *Client) logSolveStatus(ctx context.Context, ch chan *bk.SolveStatus) er }, ) } + +// A helper to remove noise from buildkit error messages. +// FIXME: Obviously a cleaner solution would be nice. +func bkCleanError(err error) error { + noise := []string{ + "executor failed running ", + "buildkit-runc did not terminate successfully", + "rpc error: code = Unknown desc = ", + "failed to solve: ", + } + + msg := err.Error() + + for _, s := range noise { + msg = strings.ReplaceAll(msg, s, "") + } + + return errors.New(msg) +} diff --git a/dagger/environment.go b/dagger/environment/environment.go similarity index 94% rename from dagger/environment.go rename to dagger/environment/environment.go index b12673bf..1007569a 100644 --- a/dagger/environment.go +++ b/dagger/environment/environment.go @@ -1,4 +1,4 @@ -package dagger +package environment import ( "context" @@ -10,6 +10,7 @@ import ( "cuelang.org/go/cue" cueflow "cuelang.org/go/tools/flow" "dagger.io/go/dagger/compiler" + "dagger.io/go/dagger/solver" "dagger.io/go/dagger/state" "dagger.io/go/stdlib" @@ -32,7 +33,7 @@ type Environment struct { computed *compiler.Value } -func NewEnvironment(st *state.State) (*Environment, error) { +func New(st *state.State) (*Environment, error) { e := &Environment{ state: st, @@ -81,7 +82,7 @@ func (e *Environment) Computed() *compiler.Value { } // LoadPlan loads the plan -func (e *Environment) LoadPlan(ctx context.Context, s Solver) error { +func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error { span, ctx := opentracing.StartSpanFromContext(ctx, "environment.LoadPlan") defer span.Finish() @@ -165,7 +166,7 @@ func (e *Environment) LocalDirs() map[string]string { } // Up missing values in environment configuration, and write them to state. -func (e *Environment) Up(ctx context.Context, s Solver) error { +func (e *Environment) Up(ctx context.Context, s solver.Solver) error { span, ctx := opentracing.StartSpanFromContext(ctx, "environment.Up") defer span.Finish() @@ -216,7 +217,7 @@ func noOpRunner(t *cueflow.Task) error { return nil } -func newPipelineRunner(computed *compiler.Value, s Solver) cueflow.RunnerFunc { +func newPipelineRunner(computed *compiler.Value, s solver.Solver) cueflow.RunnerFunc { return cueflow.RunnerFunc(func(t *cueflow.Task) error { ctx := t.Context() lg := log. @@ -294,5 +295,5 @@ func newPipelineRunner(computed *compiler.Value, s Solver) cueflow.RunnerFunc { } func (e *Environment) ScanInputs(ctx context.Context) []*compiler.Value { - return ScanInputs(ctx, e.plan) + return scanInputs(ctx, e.plan) } diff --git a/dagger/environment_test.go b/dagger/environment/environment_test.go similarity index 89% rename from dagger/environment_test.go rename to dagger/environment/environment_test.go index 3da6ba6c..49af60de 100644 --- a/dagger/environment_test.go +++ b/dagger/environment/environment_test.go @@ -1,4 +1,4 @@ -package dagger +package environment import ( "testing" @@ -14,7 +14,7 @@ func TestLocalDirs(t *testing.T) { } require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{}))) - environment, err := NewEnvironment(st) + environment, err := New(st) require.NoError(t, err) localdirs := environment.LocalDirs() diff --git a/dagger/inputs_scan.go b/dagger/environment/inputs_scan.go similarity index 93% rename from dagger/inputs_scan.go rename to dagger/environment/inputs_scan.go index 2b04becc..38d173b6 100644 --- a/dagger/inputs_scan.go +++ b/dagger/environment/inputs_scan.go @@ -1,4 +1,4 @@ -package dagger +package environment import ( "context" @@ -42,7 +42,7 @@ func isReference(val cue.Value) bool { return isRef(val) } -func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value { +func scanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value { lg := log.Ctx(ctx) inputs := []*compiler.Value{} diff --git a/dagger/pipeline.go b/dagger/environment/pipeline.go similarity index 99% rename from dagger/pipeline.go rename to dagger/environment/pipeline.go index 04b7d436..8657429c 100644 --- a/dagger/pipeline.go +++ b/dagger/environment/pipeline.go @@ -1,4 +1,4 @@ -package dagger +package environment import ( "bytes" @@ -26,6 +26,7 @@ import ( "gopkg.in/yaml.v3" "dagger.io/go/dagger/compiler" + "dagger.io/go/dagger/solver" ) const ( @@ -36,14 +37,14 @@ const ( type Pipeline struct { code *compiler.Value name string - s Solver + s solver.Solver state llb.State result bkgw.Reference image dockerfile2llb.Image computed *compiler.Value } -func NewPipeline(code *compiler.Value, s Solver) *Pipeline { +func NewPipeline(code *compiler.Value, s solver.Solver) *Pipeline { return &Pipeline{ code: code, name: code.Path().String(), @@ -70,7 +71,7 @@ func (p *Pipeline) Result() (llb.State, error) { } func (p *Pipeline) FS() fs.FS { - return NewBuildkitFS(p.result) + return solver.NewBuildkitFS(p.result) } func (p *Pipeline) ImageConfig() dockerfile2llb.Image { @@ -641,7 +642,7 @@ func (p *Pipeline) DockerLogin(ctx context.Context, op *compiler.Value, st llb.S return st, err } - p.s.auth.AddCredentials(target, username, secret) + p.s.AddCredentials(target, username, secret) log. Ctx(ctx). Debug(). @@ -862,7 +863,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S return st, err } - if p.s.noCache { + if p.s.NoCache() { opts["no-cache"] = "" } diff --git a/dagger/fs.go b/dagger/solver/fs.go similarity index 99% rename from dagger/fs.go rename to dagger/solver/fs.go index ecae9401..dd208256 100644 --- a/dagger/fs.go +++ b/dagger/solver/fs.go @@ -1,4 +1,4 @@ -package dagger +package solver import ( "context" diff --git a/dagger/registryauth.go b/dagger/solver/registryauth.go similarity index 76% rename from dagger/registryauth.go rename to dagger/solver/registryauth.go index 5acd618a..af17d38e 100644 --- a/dagger/registryauth.go +++ b/dagger/solver/registryauth.go @@ -1,4 +1,4 @@ -package dagger +package solver import ( "context" @@ -12,20 +12,20 @@ import ( "google.golang.org/grpc/status" ) -// registryAuthProvider is a buildkit provider for registry authentication +// RegistryAuthProvider is a buildkit provider for registry authentication // Adapted from: https://github.com/moby/buildkit/blob/master/session/auth/authprovider/authprovider.go -type registryAuthProvider struct { +type RegistryAuthProvider struct { credentials map[string]*bkauth.CredentialsResponse m sync.RWMutex } -func newRegistryAuthProvider() *registryAuthProvider { - return ®istryAuthProvider{ +func NewRegistryAuthProvider() *RegistryAuthProvider { + return &RegistryAuthProvider{ credentials: map[string]*bkauth.CredentialsResponse{}, } } -func (a *registryAuthProvider) AddCredentials(target, username, secret string) { +func (a *RegistryAuthProvider) AddCredentials(target, username, secret string) { a.m.Lock() defer a.m.Unlock() @@ -35,11 +35,11 @@ func (a *registryAuthProvider) AddCredentials(target, username, secret string) { } } -func (a *registryAuthProvider) Register(server *grpc.Server) { +func (a *RegistryAuthProvider) Register(server *grpc.Server) { bkauth.RegisterAuthServer(server, a) } -func (a *registryAuthProvider) Credentials(ctx context.Context, req *bkauth.CredentialsRequest) (*bkauth.CredentialsResponse, error) { +func (a *RegistryAuthProvider) Credentials(ctx context.Context, req *bkauth.CredentialsRequest) (*bkauth.CredentialsResponse, error) { reqURL, err := parseAuthHost(req.Host) if err != nil { return nil, err @@ -73,14 +73,14 @@ func parseAuthHost(host string) (*url.URL, error) { return url.Parse(host) } -func (a *registryAuthProvider) FetchToken(ctx context.Context, req *bkauth.FetchTokenRequest) (rr *bkauth.FetchTokenResponse, err error) { +func (a *RegistryAuthProvider) FetchToken(ctx context.Context, req *bkauth.FetchTokenRequest) (rr *bkauth.FetchTokenResponse, err error) { return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented") } -func (a *registryAuthProvider) GetTokenAuthority(ctx context.Context, req *bkauth.GetTokenAuthorityRequest) (*bkauth.GetTokenAuthorityResponse, error) { +func (a *RegistryAuthProvider) GetTokenAuthority(ctx context.Context, req *bkauth.GetTokenAuthorityRequest) (*bkauth.GetTokenAuthorityResponse, error) { return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented") } -func (a *registryAuthProvider) VerifyTokenAuthority(ctx context.Context, req *bkauth.VerifyTokenAuthorityRequest) (*bkauth.VerifyTokenAuthorityResponse, error) { +func (a *RegistryAuthProvider) VerifyTokenAuthority(ctx context.Context, req *bkauth.VerifyTokenAuthorityRequest) (*bkauth.VerifyTokenAuthorityResponse, error) { return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented") } diff --git a/dagger/solver.go b/dagger/solver/solver.go similarity index 78% rename from dagger/solver.go rename to dagger/solver/solver.go index e057155b..4a33ac62 100644 --- a/dagger/solver.go +++ b/dagger/solver/solver.go @@ -1,11 +1,9 @@ -package dagger +package solver import ( "context" "encoding/json" - "errors" "fmt" - "strings" bk "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" @@ -19,20 +17,20 @@ import ( ) type Solver struct { - events chan *bk.SolveStatus - control *bk.Client - gw bkgw.Client - auth *registryAuthProvider - noCache bool + opts Opts } -func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, auth *registryAuthProvider, noCache bool) Solver { +type Opts struct { + Control *bk.Client + Gateway bkgw.Client + Events chan *bk.SolveStatus + Auth *RegistryAuthProvider + NoCache bool +} + +func New(opts Opts) Solver { return Solver{ - events: events, - control: control, - gw: gw, - auth: auth, - noCache: noCache, + opts: opts, } } @@ -55,6 +53,14 @@ func invalidateCache(def *llb.Definition) error { return nil } +func (s Solver) NoCache() bool { + return s.opts.NoCache +} + +func (s Solver) AddCredentials(target, username, secret string) { + s.opts.Auth.AddCredentials(target, username, secret) +} + func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, error) { // FIXME: do not hardcode the platform def, err := st.Marshal(ctx, llb.LinuxAmd64) @@ -62,7 +68,7 @@ func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, er return nil, err } - if s.noCache { + if s.opts.NoCache { if err := invalidateCache(def); err != nil { return nil, err } @@ -72,7 +78,7 @@ func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, er } func (s Solver) SessionID() string { - return s.gw.BuildOpts().SessionID + return s.opts.Gateway.BuildOpts().SessionID } func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, error) { @@ -81,7 +87,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res // Load image metadata and convert to to LLB. // Inspired by https://github.com/moby/buildkit/blob/master/frontend/dockerfile/dockerfile2llb/convert.go // FIXME: this needs to handle platform - _, meta, err := s.gw.ResolveImageConfig(ctx, ref, opts) + _, meta, err := s.opts.Gateway.ResolveImageConfig(ctx, ref, opts) if err != nil { return image, err } @@ -94,12 +100,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res // Solve will block until the state is solved and returns a Reference. func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.Result, error) { - // call solve - res, err := s.gw.Solve(ctx, req) - if err != nil { - return nil, bkCleanError(err) - } - return res, nil + return s.opts.Gateway.Solve(ctx, req) } // Solve will block until the state is solved and returns a Reference. @@ -149,7 +150,7 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im opts := bk.SolveOpt{ Exports: []bk.ExportEntry{output}, - Session: []session.Attachable{s.auth}, + Session: []session.Attachable{s.opts.Auth}, } ch := make(chan *bk.SolveStatus) @@ -158,11 +159,11 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im // purposes. go func() { for event := range ch { - s.events <- event + s.opts.Events <- event } }() - return s.control.Build(ctx, opts, "", func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) { + return s.opts.Control.Build(ctx, opts, "", func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) { res, err := c.Solve(ctx, bkgw.SolveRequest{ Definition: def, }) @@ -203,22 +204,3 @@ func dumpLLB(def *bkpb.Definition) ([]byte, error) { } return json.Marshal(ops) } - -// A helper to remove noise from buildkit error messages. -// FIXME: Obviously a cleaner solution would be nice. -func bkCleanError(err error) error { - noise := []string{ - "executor failed running ", - "buildkit-runc did not terminate successfully", - "rpc error: code = Unknown desc = ", - "failed to solve: ", - } - - msg := err.Error() - - for _, s := range noise { - msg = strings.ReplaceAll(msg, s, "") - } - - return errors.New(msg) -}