embedded stdlib

This PR embeds the stdlib into the dagger binary itself for convenience.
Long term we will want to source the stdlib directly from git.

- Updated go to 1.16 to use the new built-in embedding functionality
- The `stdlib` go package now contains an embed of the stdlib

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

dagger/build.go

@@ -2,6 +2,7 @@ package dagger
 
 import (
 	"context"
+	"fmt"
 	"path"
 	"path/filepath"
 
@@ -11,24 +12,31 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"dagger.cloud/go/dagger/cc"
+	"dagger.cloud/go/stdlib"
 )
 
 // Build a cue configuration tree from the files in fs.
 func CueBuild(ctx context.Context, fs FS, args ...string) (*cc.Value, error) {
-	lg := log.Ctx(ctx)
-
-	// The CUE overlay needs to be prefixed by a non-conflicting path with the
-	// local filesystem, otherwise Cue will merge the Overlay with whatever Cue
-	// files it finds locally.
-	const overlayPrefix = "/config"
+	var (
+		err error
+		lg  = log.Ctx(ctx)
+	)
 
 	buildConfig := &cueload.Config{
-		Dir:     overlayPrefix,
-		Overlay: map[string]cueload.Source{},
+		// The CUE overlay needs to be prefixed by a non-conflicting path with the
+		// local filesystem, otherwise Cue will merge the Overlay with whatever Cue
+		// files it finds locally.
+		Dir: "/config",
 	}
-	buildArgs := args
 
-	err := fs.Walk(ctx, func(p string, f Stat) error {
+	// Start by creating an overlay with the stdlib
+	buildConfig.Overlay, err = stdlib.Overlay(buildConfig.Dir)
+	if err != nil {
+		return nil, err
+	}
+
+	// Add the config files on top of the overlay
+	err = fs.Walk(ctx, func(p string, f Stat) error {
 		lg.Debug().Str("path", p).Msg("Compiler.Build: processing")
 		if f.IsDir() {
 			return nil
@@ -38,16 +46,16 @@ func CueBuild(ctx context.Context, fs FS, args ...string) (*cc.Value, error) {
 		}
 		contents, err := fs.ReadFile(ctx, p)
 		if err != nil {
-			return errors.Wrap(err, p)
+			return fmt.Errorf("%s: %w", p, err)
 		}
-		overlayPath := path.Join(overlayPrefix, p)
+		overlayPath := path.Join(buildConfig.Dir, p)
 		buildConfig.Overlay[overlayPath] = cueload.FromBytes(contents)
 		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
-	instances := cueload.Instances(buildArgs, buildConfig)
+	instances := cueload.Instances(args, buildConfig)
 	if len(instances) != 1 {
 		return nil, errors.New("only one package is supported at a time")
 	}

dagger/env.go

@@ -2,7 +2,6 @@ package dagger
 
 import (
 	"context"
-	"os"
 
 	"cuelang.org/go/cue"
 	cueflow "cuelang.org/go/tools/flow"
@@ -91,36 +90,9 @@ func (env *Env) SetInput(i *cc.Value) error {
 	)
 }
 
-func stdlibLoader() (*cc.Value, error) {
-	if dev := os.Getenv("DAGGER_DEV_STDLIB"); dev != "" {
-		v, err := cc.Compile("stdlib.cue", `
-			do: "local"
-			dir: string
-			include: ["cue.mod/pkg"]
-		`)
-		if err != nil {
-			return nil, err
-		}
-		return v.MergeTarget(dev, "dir")
-	}
-	return cc.Compile("stdlib.cue", `
-		do: "fetch-git"
-		remote: "https://github.com/blocklayerhq/dagger-stdlib"
-		ref: "0625677b5aec1162621ad18fbd7b90dc9d7d54e5"
-	`)
-}
-
 // Update the base configuration
 func (env *Env) Update(ctx context.Context, s Solver) error {
 	p := NewPipeline(s, nil)
-	// always inject stdlib in cue.mod/pkg
-	stdlib, err := stdlibLoader()
-	if err != nil {
-		return err
-	}
-	if err := p.Do(ctx, stdlib); err != nil {
-		return err
-	}
 	// execute updater script
 	if err := p.Do(ctx, env.updater); err != nil {
 		return err
@@ -189,10 +161,6 @@ func (env *Env) LocalDirs() map[string]string {
 	)
 	// 2. Scan the environment updater
 	localdirs(env.Updater())
-	// 3. In dev mode, always include dev stdlib directory
-	if dev := os.Getenv("DAGGER_DEV_STDLIB"); dev != "" {
-		dirs[dev] = dev
-	}
 	return dirs
 }
 
@@ -337,14 +305,7 @@ func (env *Env) Compute(ctx context.Context, s Solver) error {
 			}
 			v := cc.Wrap(t.Value(), flowInst)
 			p := NewPipeline(s, NewFillable(t))
-			err := p.Do(ctx, v)
-			if err == ErrAbortExecution {
-				// Pipeline was partially executed
-				// FIXME: tell user which inputs are missing (by inspecting references)
-				lg.Warn().Msg("pipeline was partially executed because of missing inputs")
-				return nil
-			}
-			return err
+			return p.Do(ctx, v)
 		}), nil
 	}
 	// Orchestrate execution with cueflow

dagger/pipeline.go

@@ -13,10 +13,6 @@ import (
 	"dagger.cloud/go/dagger/cc"
 )
 
-var (
-	ErrAbortExecution = errors.New("execution stopped")
-)
-
 // An execution pipeline
 type Pipeline struct {
 	s   Solver
@@ -114,11 +110,11 @@ func (p *Pipeline) Do(ctx context.Context, code ...*cc.Value) error {
 		if err := op.IsConcreteR(); err != nil {
 			log.
 				Ctx(ctx).
-				Debug().
+				Warn().
 				Str("original_cue_error", err.Error()).
 				Int("op", idx).
-				Msg("script is missing inputs and has not been fully executed")
-			return ErrAbortExecution
+				Msg("pipeline was partially executed because of missing inputs")
+			return nil
 		}
 		if err := p.doOp(ctx, op); err != nil {
 			return err