From c35eca99e137d9ab17dbc6d6957b97e8ff85ea86 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 11 Mar 2021 16:41:19 -0800 Subject: [PATCH] push container support Signed-off-by: Andrea Luzzardi --- dagger/client.go | 4 ++-- dagger/fs.go | 20 ++++++++++++++++ dagger/pipeline.go | 26 ++++++++++++++++++++ dagger/solver.go | 51 +++++++++++++++++++++++++++++++++++----- go.sum | 1 + stdlib/dagger/dagger.cue | 5 ++++ 6 files changed, 99 insertions(+), 8 deletions(-) diff --git a/dagger/client.go b/dagger/client.go index ac0f8089..8e5c59a4 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -127,8 +127,8 @@ func (c *Client) buildfn(ctx context.Context, env *Env, ch chan *bk.SolveStatus, Interface("attrs", opts.FrontendAttrs). Msg("spawning buildkit job") - resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) { - s := NewSolver(c) + resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) { + s := NewSolver(c.c, gw, ch) lg.Debug().Msg("loading configuration") if err := env.Update(ctx, s); err != nil { diff --git a/dagger/fs.go b/dagger/fs.go index e158bf20..44312917 100644 --- a/dagger/fs.go +++ b/dagger/fs.go @@ -7,6 +7,7 @@ import ( "path" "strings" + bk "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" bkgw "github.com/moby/buildkit/frontend/gateway/client" bkpb "github.com/moby/buildkit/solver/pb" @@ -190,6 +191,25 @@ func (fs FS) Result(ctx context.Context) (*bkgw.Result, error) { return res, nil } +func (fs FS) Export(ctx context.Context, output bk.ExportEntry) (*bk.SolveResponse, error) { + // Lazy solve + if err := (&fs).solve(ctx); err != nil { + return nil, err + } + // NOTE: llb.Scratch is represented by a `nil` reference. If solve result is + // Scratch, then `fs.output` is `nil`. + if fs.output == nil { + return nil, os.ErrNotExist + } + + st, err := fs.output.ToState() + if err != nil { + return nil, err + } + + return fs.s.Export(ctx, st, output) +} + // A helper to remove noise from buildkit error messages. // FIXME: Obviously a cleaner solution would be nice. func bkCleanError(err error) error { diff --git a/dagger/pipeline.go b/dagger/pipeline.go index 7999044c..04ec3960 100644 --- a/dagger/pipeline.go +++ b/dagger/pipeline.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/docker/distribution/reference" + bk "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" dockerfilebuilder "github.com/moby/buildkit/frontend/dockerfile/builder" bkgw "github.com/moby/buildkit/frontend/gateway/client" @@ -158,6 +159,8 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value) error { return p.Export(ctx, op) case "fetch-container": return p.FetchContainer(ctx, op) + case "push-container": + return p.PushContainer(ctx, op) case "fetch-git": return p.FetchGit(ctx, op) case "local": @@ -541,6 +544,29 @@ func parseKeyValue(env string) (string, string) { return parts[0], v } +func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value) error { + rawRef, err := op.Get("ref").String() + if err != nil { + return err + } + + ref, err := reference.ParseNormalizedNamed(rawRef) + if err != nil { + return fmt.Errorf("failed to parse ref %s: %w", rawRef, err) + } + // Add the default tag "latest" to a reference if it only has a repo name. + ref = reference.TagNameOnly(ref) + + _, err = p.fs.Export(ctx, bk.ExportEntry{ + Type: bk.ExporterImage, + Attrs: map[string]string{ + "name": ref.String(), + "push": "true", + }, + }) + return err +} + func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value) error { remote, err := op.Get("remote").String() if err != nil { diff --git a/dagger/solver.go b/dagger/solver.go index 36799e83..c72628ef 100644 --- a/dagger/solver.go +++ b/dagger/solver.go @@ -5,9 +5,12 @@ import ( "encoding/json" "fmt" + bk "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" bkgw "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/auth/authprovider" bkpb "github.com/moby/buildkit/solver/pb" "github.com/opencontainers/go-digest" "github.com/rs/zerolog/log" @@ -16,12 +19,16 @@ import ( // Polyfill for buildkit gateway client // Use instead of bkgw.Client type Solver struct { - c bkgw.Client + events chan *bk.SolveStatus + control *bk.Client + gw bkgw.Client } -func NewSolver(c bkgw.Client) Solver { +func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus) Solver { return Solver{ - c: c, + events: events, + control: control, + gw: gw, } } @@ -37,7 +44,7 @@ func (s Solver) Scratch() FS { } func (s Solver) SessionID() string { - return s.c.BuildOpts().SessionID + return s.gw.BuildOpts().SessionID } func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, error) { @@ -46,7 +53,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res // Load image metadata and convert to to LLB. // Inspired by https://github.com/moby/buildkit/blob/master/frontend/dockerfile/dockerfile2llb/convert.go // FIXME: this needs to handle platform - _, meta, err := s.c.ResolveImageConfig(ctx, ref, opts) + _, meta, err := s.gw.ResolveImageConfig(ctx, ref, opts) if err != nil { return image, err } @@ -60,7 +67,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res // Solve will block until the state is solved and returns a Reference. func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (bkgw.Reference, error) { // call solve - res, err := s.c.Solve(ctx, req) + res, err := s.gw.Solve(ctx, req) if err != nil { return nil, bkCleanError(err) } @@ -98,6 +105,38 @@ func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) }) } +// Export will export `st` to `output` +// FIXME: this is currently impleneted as a hack, starting a new Build session +// within buildkit from the Control API. Ideally the Gateway API should allow to +// Export directly. +func (s Solver) Export(ctx context.Context, st llb.State, output bk.ExportEntry) (*bk.SolveResponse, error) { + def, err := st.Marshal(ctx, llb.LinuxAmd64) + if err != nil { + return nil, err + } + + opts := bk.SolveOpt{ + Exports: []bk.ExportEntry{output}, + Session: []session.Attachable{ + authprovider.NewDockerAuthProvider(log.Ctx(ctx)), + }, + } + + ch := make(chan *bk.SolveStatus) + + go func() { + for event := range ch { + s.events <- event + } + }() + + return s.control.Build(ctx, opts, "", func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) { + return c.Solve(ctx, bkgw.SolveRequest{ + Definition: def.ToPB(), + }) + }, ch) +} + type llbOp struct { Op bkpb.Op Digest digest.Digest diff --git a/go.sum b/go.sum index 01aaec10..054be3c4 100644 --- a/go.sum +++ b/go.sum @@ -268,6 +268,7 @@ github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r github.com/docker/docker v17.12.0-ce-rc1.0.20200730172259-9f28837c1d93+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible h1:J2OhsbfqoBRRT048iD/tqXBvEQWQATQ8vew6LqQmDSU= github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index 6900d094..ca4725d4 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -52,6 +52,11 @@ package dagger ref: string } +#PushContainer: { + do: "push-container" + ref: string +} + #FetchGit: { do: "fetch-git" remote: string