From b77d1a1e3e31d996dc25de4b0a18f542e59bfd42 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 7 May 2021 14:45:15 -0700 Subject: [PATCH 01/16] gitflow ux Signed-off-by: Andrea Luzzardi --- .dagger/env/dev-stdlib/adhoc.cue | 44 --- .dagger/env/dev/main.cue | 69 ----- .dagger/env/hello-world | 1 - .dagger/env/sandbox/sandbox.cue | 21 -- cmd/dagger/cmd/common/common.go | 72 +++-- cmd/dagger/cmd/compute.go | 22 +- cmd/dagger/cmd/init.go | 66 +++++ cmd/dagger/cmd/input/container.go | 4 +- cmd/dagger/cmd/input/dir.go | 4 +- cmd/dagger/cmd/input/git.go | 4 +- cmd/dagger/cmd/input/json.go | 4 +- cmd/dagger/cmd/input/list.go | 10 +- cmd/dagger/cmd/input/root.go | 16 +- cmd/dagger/cmd/input/text.go | 4 +- cmd/dagger/cmd/input/unset.go | 15 +- cmd/dagger/cmd/input/yaml.go | 4 +- cmd/dagger/cmd/list.go | 73 +++-- cmd/dagger/cmd/new.go | 148 ----------- cmd/dagger/cmd/plan/dir.go | 33 --- cmd/dagger/cmd/plan/file.go | 31 --- cmd/dagger/cmd/plan/git.go | 43 --- cmd/dagger/cmd/plan/package.go | 31 --- cmd/dagger/cmd/plan/root.go | 42 --- cmd/dagger/cmd/query.go | 10 +- cmd/dagger/cmd/root.go | 5 +- cmd/dagger/cmd/up.go | 15 +- dagger/client.go | 3 +- dagger/compiler/json.go | 2 +- dagger/environment.go | 20 +- dagger/{input_test.go => environment_test.go} | 9 +- dagger/{ => state}/input.go | 63 ++--- dagger/{ => state}/state.go | 32 +-- dagger/state/store.go | 205 ++++++++++++++ dagger/state/store_test.go | 39 +++ dagger/store.go | 250 ------------------ dagger/store_test.go | 123 --------- go.mod | 1 - 37 files changed, 495 insertions(+), 1043 deletions(-) delete mode 100644 .dagger/env/dev-stdlib/adhoc.cue delete mode 100644 .dagger/env/dev/main.cue delete mode 120000 .dagger/env/hello-world delete mode 100644 .dagger/env/sandbox/sandbox.cue create mode 100644 cmd/dagger/cmd/init.go delete mode 100644 cmd/dagger/cmd/new.go delete mode 100644 cmd/dagger/cmd/plan/dir.go delete mode 100644 cmd/dagger/cmd/plan/file.go delete mode 100644 cmd/dagger/cmd/plan/git.go delete mode 100644 cmd/dagger/cmd/plan/package.go delete mode 100644 cmd/dagger/cmd/plan/root.go rename dagger/{input_test.go => environment_test.go} (61%) rename dagger/{ => state}/input.go (77%) rename dagger/{ => state}/state.go (58%) create mode 100644 dagger/state/store.go create mode 100644 dagger/state/store_test.go delete mode 100644 dagger/store.go delete mode 100644 dagger/store_test.go diff --git a/.dagger/env/dev-stdlib/adhoc.cue b/.dagger/env/dev-stdlib/adhoc.cue deleted file mode 100644 index 41f5a493..00000000 --- a/.dagger/env/dev-stdlib/adhoc.cue +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "dagger.io/dagger/op" -) - -// Reproduce inline issue. -// See https://github.com/dagger/dagger/issues/395 -test: adhoc: repro395: { - good: { - // This field is correctly computed because its intermediary pipeline is not inlined. - hello: sayHello.message - - // Intermediary pipeline cannot be inlined: it must be visible in a field - sayHello: { - message: { - string - #up: [ - op.#FetchContainer & { ref: "alpine" }, - op.#Exec & { - args: ["sh", "-c", "echo hello > /message"] - }, - op.#Export & { source: "/message", format: "string" }, - ] - } - } - } - bad: { - // This field is NOT correctly computed because its intermediary pipeline is inlined. - hello: { - message: { - string - #up: [ - op.#FetchContainer & { ref: "alpine" }, - op.#Exec & { - args: ["sh", "-c", "echo hello > /message"] - }, - op.#Export & { source: "/message", format: "string" }, - ] - } - }.message - - } -} diff --git a/.dagger/env/dev/main.cue b/.dagger/env/dev/main.cue deleted file mode 100644 index 8d7392fe..00000000 --- a/.dagger/env/dev/main.cue +++ /dev/null @@ -1,69 +0,0 @@ -// A dagger workflow to develop dagger -package main - -import ( - "dagger.io/dagger" - "dagger.io/os" - "dagger.io/alpine" - "dagger.io/docker" - "dagger.io/go" -) - -// Dagger source code -source: dagger.#Artifact - - -test: { - // Go unit tests - unit: { - logs: (os.#File & { - from: build.ctr - path: "/test.log" - read: format: "string" - }).read.data - } - - // Full suite of bats integration tests - integration: { - // FIXME - } -} - -// Build the dagger binaries -build: { - ctr: go.#Container & { - "source": source - setup: [ - "apk add --no-cache file", - ] - command: """ - go test -v ./... > /test.log - go build -o /binaries/ ./cmd/... > /build.log - """ - } - - binaries: docker.#Container & { - image: ctr - outputDir: "/binaries" - } - - logs: (os.#File & { - from: ctr - path: "/build.log" - read: format: "string" - }).read.data -} - - -// Execute `dagger help` -usage: docker.#Container & { - image: alpine.#Image - - command: "dagger help" - - volume: binaries: { - from: build.binaries - dest: "/usr/local/dagger/bin/" - } - shell: search: "/usr/local/dagger/bin": true -} diff --git a/.dagger/env/hello-world b/.dagger/env/hello-world deleted file mode 120000 index a69ee1e0..00000000 --- a/.dagger/env/hello-world +++ /dev/null @@ -1 +0,0 @@ -../../examples/hello-world \ No newline at end of file diff --git a/.dagger/env/sandbox/sandbox.cue b/.dagger/env/sandbox/sandbox.cue deleted file mode 100644 index 42e7e293..00000000 --- a/.dagger/env/sandbox/sandbox.cue +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "dagger.io/docker" - "dagger.io/os" -) - -let ctr = docker.#Container & { - command: "echo 'hello world!' > /etc/motd" -} - -motd: (os.#File & { - from: ctr - path: "/etc/motd" - read: format: "string" -}).read.data - -etc: (os.#Dir & { - from: ctr - path: "/etc" -}).read.tree diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index 2b10a382..b928cffa 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -2,65 +2,79 @@ package common import ( "context" + "errors" "os" "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/rs/zerolog/log" "github.com/spf13/viper" ) -func GetCurrentEnvironmentState(ctx context.Context, store *dagger.Store) *dagger.EnvironmentState { +func GetCurrentEnvironmentState(ctx context.Context) *state.State { lg := log.Ctx(ctx) - environmentName := viper.GetString("environment") - if environmentName != "" { - st, err := store.LookupEnvironmentByName(ctx, environmentName) + // 1) If no environment name has been given, look for the current environment + environment := viper.GetString("environment") + if environment == "" { + st, err := state.Current(ctx) if err != nil { lg. Fatal(). Err(err). - Str("environmentName", environmentName). - Msg("failed to lookup environment by name") + Msg("failed to load environment") } return st } - wd, err := os.Getwd() - if err != nil { - lg.Fatal().Err(err).Msg("cannot get current working directory") + // 2) Check if it's an environment path (can be opened directly) + st, err := state.Open(ctx, environment) + if err == nil { + return st } - st, err := store.LookupEnvironmentByPath(ctx, wd) - if err != nil { + if !errors.Is(err, os.ErrNotExist) { lg. Fatal(). Err(err). - Str("environmentPath", wd). - Msg("failed to lookup environment by path") + Str("environmentPath", environment). + Msg("failed to load environment") } - if len(st) == 0 { - lg. - Fatal(). - Err(err). - Str("environmentPath", wd). - Msg("no environments match the current directory") - } - if len(st) > 1 { - environments := []string{} - for _, s := range st { - environments = append(environments, s.Name) + + // At this point, it must be an environment name + workspace := viper.GetString("workspace") + if workspace == "" { + workspace, err = state.CurrentWorkspace(ctx) + if err != nil { + lg. + Fatal(). + Err(err). + Msg("failed to determine current workspace") } + } + + environments, err := state.List(ctx, workspace) + if err != nil { lg. Fatal(). Err(err). - Str("environmentPath", wd). - Strs("environments", environments). - Msg("multiple environments match the current directory, select one with `--environment`") + Msg("failed to list environments") } - return st[0] + for _, e := range environments { + if e.Name == environment { + return e + } + } + + lg. + Fatal(). + Str("environment", environment). + Msg("environment not found") + + return nil } // Re-compute an environment (equivalent to `dagger up`). -func EnvironmentUp(ctx context.Context, state *dagger.EnvironmentState, noCache bool) *dagger.Environment { +func EnvironmentUp(ctx context.Context, state *state.State, noCache bool) *dagger.Environment { lg := log.Ctx(ctx) c, err := dagger.NewClient(ctx, "", noCache) diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index 1f15e4cb..c49e235e 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -10,12 +10,11 @@ import ( "cuelang.org/go/cue" "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" "dagger.io/go/dagger/compiler" + "dagger.io/go/dagger/state" "go.mozilla.org/sops/v3" "go.mozilla.org/sops/v3/decrypt" - "github.com/google/uuid" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -36,16 +35,15 @@ var computeCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - st := &dagger.EnvironmentState{ - ID: uuid.New().String(), - Name: "FIXME", - PlanSource: dagger.DirInput(args[0], []string{"*.cue", "cue.mod"}), + st := &state.State{ + Name: "FIXME", + Path: args[0], } for _, input := range viper.GetStringSlice("input-string") { parts := strings.SplitN(input, "=", 2) k, v := parts[0], parts[1] - err := st.SetInput(k, dagger.TextInput(v)) + err := st.SetInput(k, state.TextInput(v)) if err != nil { lg. Fatal(). @@ -58,7 +56,7 @@ var computeCmd = &cobra.Command{ for _, input := range viper.GetStringSlice("input-dir") { parts := strings.SplitN(input, "=", 2) k, v := parts[0], parts[1] - err := st.SetInput(k, dagger.DirInput(v, []string{})) + err := st.SetInput(k, state.DirInput(v, []string{})) if err != nil { lg. Fatal(). @@ -71,7 +69,7 @@ var computeCmd = &cobra.Command{ for _, input := range viper.GetStringSlice("input-git") { parts := strings.SplitN(input, "=", 2) k, v := parts[0], parts[1] - err := st.SetInput(k, dagger.GitInput(v, "", "")) + err := st.SetInput(k, state.GitInput(v, "", "")) if err != nil { lg. Fatal(). @@ -102,7 +100,7 @@ var computeCmd = &cobra.Command{ lg.Fatal().Msg("invalid json") } - err = st.SetInput("", dagger.JSONInput(string(content))) + err = st.SetInput("", state.JSONInput(string(content))) if err != nil { lg.Fatal().Err(err).Msg("failed to add input") } @@ -125,7 +123,7 @@ var computeCmd = &cobra.Command{ content = plaintext } - err = st.SetInput("", dagger.YAMLInput(string(content))) + err = st.SetInput("", state.YAMLInput(string(content))) if err != nil { lg.Fatal().Err(err).Msg("failed to add input") } @@ -143,7 +141,7 @@ var computeCmd = &cobra.Command{ } if len(content) > 0 { - err = st.SetInput(k, dagger.FileInput(v)) + err = st.SetInput(k, state.FileInput(v)) if err != nil { lg.Fatal().Err(err).Msg("failed to set input string") } diff --git a/cmd/dagger/cmd/init.go b/cmd/dagger/cmd/init.go new file mode 100644 index 00000000..1bc16633 --- /dev/null +++ b/cmd/dagger/cmd/init.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "os" + "path/filepath" + + "dagger.io/go/cmd/dagger/logger" + "dagger.io/go/dagger/state" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var initCmd = &cobra.Command{ + Use: "init", + Args: cobra.MaximumNArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + // Fix Viper bug for duplicate flags: + // https://github.com/spf13/viper/issues/233 + if err := viper.BindPFlags(cmd.Flags()); err != nil { + panic(err) + } + }, + Run: func(cmd *cobra.Command, args []string) { + lg := logger.New() + ctx := lg.WithContext(cmd.Context()) + + dir := viper.GetString("environment") + if dir == "" { + cwd, err := os.Getwd() + if err != nil { + lg. + Fatal(). + Err(err). + Msg("failed to get current working dir") + } + dir = cwd + } + + var name string + if len(args) > 0 { + name = args[0] + } else { + name = getNewEnvironmentName(dir) + } + + _, err := state.Init(ctx, dir, name) + if err != nil { + lg.Fatal().Err(err).Msg("failed to initialize") + } + }, +} + +func getNewEnvironmentName(dir string) string { + dirName := filepath.Base(dir) + if dirName == "/" { + return "root" + } + + return dirName +} + +func init() { + if err := viper.BindPFlags(initCmd.Flags()); err != nil { + panic(err) + } +} diff --git a/cmd/dagger/cmd/input/container.go b/cmd/dagger/cmd/input/container.go index 7b4a5be6..6ced900f 100644 --- a/cmd/dagger/cmd/input/container.go +++ b/cmd/dagger/cmd/input/container.go @@ -2,7 +2,7 @@ package input import ( "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -22,7 +22,7 @@ var containerCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - updateEnvironmentInput(ctx, args[0], dagger.DockerInput(args[1])) + updateEnvironmentInput(ctx, args[0], state.DockerInput(args[1])) }, } diff --git a/cmd/dagger/cmd/input/dir.go b/cmd/dagger/cmd/input/dir.go index 0dd05073..f18e44f1 100644 --- a/cmd/dagger/cmd/input/dir.go +++ b/cmd/dagger/cmd/input/dir.go @@ -2,7 +2,7 @@ package input import ( "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -22,7 +22,7 @@ var dirCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - updateEnvironmentInput(ctx, args[0], dagger.DirInput(args[1], []string{})) + updateEnvironmentInput(ctx, args[0], state.DirInput(args[1], []string{})) }, } diff --git a/cmd/dagger/cmd/input/git.go b/cmd/dagger/cmd/input/git.go index 384df2bf..f718b395 100644 --- a/cmd/dagger/cmd/input/git.go +++ b/cmd/dagger/cmd/input/git.go @@ -2,7 +2,7 @@ package input import ( "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -32,7 +32,7 @@ var gitCmd = &cobra.Command{ subDir = args[3] } - updateEnvironmentInput(ctx, args[0], dagger.GitInput(args[1], ref, subDir)) + updateEnvironmentInput(ctx, args[0], state.GitInput(args[1], ref, subDir)) }, } diff --git a/cmd/dagger/cmd/input/json.go b/cmd/dagger/cmd/input/json.go index 557a4ab8..508328bf 100644 --- a/cmd/dagger/cmd/input/json.go +++ b/cmd/dagger/cmd/input/json.go @@ -2,7 +2,7 @@ package input import ( "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -25,7 +25,7 @@ var jsonCmd = &cobra.Command{ updateEnvironmentInput( ctx, args[0], - dagger.JSONInput(readInput(ctx, args[1])), + state.JSONInput(readInput(ctx, args[1])), ) }, } diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index ab752235..1ce2b636 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -30,16 +30,10 @@ var listCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - store, err := dagger.DefaultStore() - if err != nil { - lg.Fatal().Err(err).Msg("failed to load store") - } - - environment := common.GetCurrentEnvironmentState(ctx, store) + environment := common.GetCurrentEnvironmentState(ctx) lg = lg.With(). - Str("environmentName", environment.Name). - Str("environmentId", environment.ID). + Str("environment", environment.Name). Logger() c, err := dagger.NewClient(ctx, "", false) diff --git a/cmd/dagger/cmd/input/root.go b/cmd/dagger/cmd/input/root.go index d4d1100e..f035fda8 100644 --- a/cmd/dagger/cmd/input/root.go +++ b/cmd/dagger/cmd/input/root.go @@ -6,7 +6,7 @@ import ( "os" "dagger.io/go/cmd/dagger/cmd/common" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -32,21 +32,15 @@ func init() { ) } -func updateEnvironmentInput(ctx context.Context, target string, input dagger.Input) { +func updateEnvironmentInput(ctx context.Context, target string, input state.Input) { lg := log.Ctx(ctx) - store, err := dagger.DefaultStore() - if err != nil { - lg.Fatal().Err(err).Msg("failed to load store") - } - - st := common.GetCurrentEnvironmentState(ctx, store) + st := common.GetCurrentEnvironmentState(ctx) st.SetInput(target, input) - if err := store.UpdateEnvironment(ctx, st, nil); err != nil { - lg.Fatal().Err(err).Str("environmentId", st.ID).Str("environmentName", st.Name).Msg("cannot update environment") + if err := state.Save(ctx, st); err != nil { + lg.Fatal().Err(err).Str("environment", st.Name).Msg("cannot update environment") } - lg.Info().Str("environmentId", st.ID).Str("environmentName", st.Name).Msg("updated environment") } func readInput(ctx context.Context, source string) string { diff --git a/cmd/dagger/cmd/input/text.go b/cmd/dagger/cmd/input/text.go index 6a2e018e..19f32520 100644 --- a/cmd/dagger/cmd/input/text.go +++ b/cmd/dagger/cmd/input/text.go @@ -2,7 +2,7 @@ package input import ( "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -25,7 +25,7 @@ var textCmd = &cobra.Command{ updateEnvironmentInput( ctx, args[0], - dagger.TextInput(readInput(ctx, args[1])), + state.TextInput(readInput(ctx, args[1])), ) }, } diff --git a/cmd/dagger/cmd/input/unset.go b/cmd/dagger/cmd/input/unset.go index 71a1f78c..248aae7a 100644 --- a/cmd/dagger/cmd/input/unset.go +++ b/cmd/dagger/cmd/input/unset.go @@ -3,7 +3,7 @@ package input import ( "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -23,17 +23,12 @@ var unsetCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - store, err := dagger.DefaultStore() - if err != nil { - lg.Fatal().Err(err).Msg("failed to load store") - } - - st := common.GetCurrentEnvironmentState(ctx, store) + st := common.GetCurrentEnvironmentState(ctx) st.RemoveInputs(args[0]) - if err := store.UpdateEnvironment(ctx, st, nil); err != nil { - lg.Fatal().Err(err).Str("environmentId", st.ID).Str("environmentName", st.Name).Msg("cannot update environment") + if err := state.Save(ctx, st); err != nil { + lg.Fatal().Err(err).Str("environment", st.Name).Msg("cannot update environment") } - lg.Info().Str("environmentId", st.ID).Str("environmentName", st.Name).Msg("updated environment") + lg.Info().Str("environment", st.Name).Msg("updated environment") }, } diff --git a/cmd/dagger/cmd/input/yaml.go b/cmd/dagger/cmd/input/yaml.go index d4216b2f..6afceecd 100644 --- a/cmd/dagger/cmd/input/yaml.go +++ b/cmd/dagger/cmd/input/yaml.go @@ -2,7 +2,7 @@ package input import ( "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -25,7 +25,7 @@ var yamlCmd = &cobra.Command{ updateEnvironmentInput( ctx, args[0], - dagger.YAMLInput(readInput(ctx, args[1])), + state.YAMLInput(readInput(ctx, args[1])), ) }, } diff --git a/cmd/dagger/cmd/list.go b/cmd/dagger/cmd/list.go index 233e879b..fd6b30a1 100644 --- a/cmd/dagger/cmd/list.go +++ b/cmd/dagger/cmd/list.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "fmt" "os" "os/user" @@ -10,7 +11,7 @@ import ( "text/tabwriter" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -30,12 +31,22 @@ var listCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { lg := logger.New() ctx := lg.WithContext(cmd.Context()) - store, err := dagger.DefaultStore() - if err != nil { - lg.Fatal().Err(err).Msg("failed to load store") + + var ( + workspace = viper.GetString("workspace") + err error + ) + if workspace == "" { + workspace, err = state.CurrentWorkspace(ctx) + if err != nil { + lg. + Fatal(). + Err(err). + Msg("failed to determine current workspace") + } } - environments, err := store.ListEnvironments(ctx) + environments, err := state.List(ctx, workspace) if err != nil { lg. Fatal(). @@ -43,45 +54,32 @@ var listCmd = &cobra.Command{ Msg("cannot list environments") } - environmentID := getCurrentEnvironmentID(ctx, store) + environmentPath := getCurrentEnvironmentPath(ctx) w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.TabIndent) - for _, r := range environments { - line := fmt.Sprintf("%s\t%s\t", r.Name, formatPlanSource(r.PlanSource)) - if r.ID == environmentID { + defer w.Flush() + for _, e := range environments { + line := fmt.Sprintf("%s\t%s\t", e.Name, formatPath(e.Path)) + if e.Path == environmentPath { line = fmt.Sprintf("%s- active environment", line) } fmt.Fprintln(w, line) } - w.Flush() }, } -func init() { - if err := viper.BindPFlags(listCmd.Flags()); err != nil { - panic(err) - } -} - -func getCurrentEnvironmentID(ctx context.Context, store *dagger.Store) string { +func getCurrentEnvironmentPath(ctx context.Context) string { lg := log.Ctx(ctx) - wd, err := os.Getwd() + st, err := state.Current(ctx) if err != nil { - lg.Warn().Err(err).Msg("cannot get current working directory") - return "" + // Ignore error if not initialized + if errors.Is(err, state.ErrNotInit) { + return "" + } + lg.Fatal().Err(err).Msg("failed to load current environment") } - st, err := store.LookupEnvironmentByPath(ctx, wd) - if err != nil { - // Ignore error - return "" - } - - if len(st) == 1 { - return st[0].ID - } - - return "" + return st.Path } func formatPath(p string) string { @@ -99,15 +97,8 @@ func formatPath(p string) string { return p } -func formatPlanSource(i dagger.Input) string { - switch i.Type { - case dagger.InputTypeDir: - return formatPath(i.Dir.Path) - case dagger.InputTypeGit: - return i.Git.Remote - case dagger.InputTypeDocker: - return i.Docker.Ref +func init() { + if err := viper.BindPFlags(listCmd.Flags()); err != nil { + panic(err) } - - return "no plan" } diff --git a/cmd/dagger/cmd/new.go b/cmd/dagger/cmd/new.go deleted file mode 100644 index 7d71adae..00000000 --- a/cmd/dagger/cmd/new.go +++ /dev/null @@ -1,148 +0,0 @@ -package cmd - -import ( - "context" - "net/url" - "os" - "path/filepath" - - "dagger.io/go/cmd/dagger/cmd/common" - "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" - - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var newCmd = &cobra.Command{ - Use: "new", - Short: "Create a new environment", - Args: cobra.MaximumNArgs(1), - PreRun: func(cmd *cobra.Command, args []string) { - // Fix Viper bug for duplicate flags: - // https://github.com/spf13/viper/issues/233 - if err := viper.BindPFlags(cmd.Flags()); err != nil { - panic(err) - } - }, - Run: func(cmd *cobra.Command, args []string) { - lg := logger.New() - ctx := lg.WithContext(cmd.Context()) - store, err := dagger.DefaultStore() - if err != nil { - lg.Fatal().Err(err).Msg("failed to load store") - } - - if viper.GetString("environment") != "" { - lg. - Fatal(). - Msg("cannot use option -d,--environment for this command") - } - - name := "" - if len(args) > 0 { - name = args[0] - } else { - name = getNewEnvironmentName(ctx) - } - - st := &dagger.EnvironmentState{ - Name: name, - PlanSource: getPlanSource(ctx), - } - - err = store.CreateEnvironment(ctx, st) - if err != nil { - lg.Fatal().Err(err).Msg("failed to create environment") - } - lg. - Info(). - Str("environmentId", st.ID). - Str("environmentName", st.Name). - Msg("environment created") - - if viper.GetBool("up") { - common.EnvironmentUp(ctx, st, false) - } - }, -} - -func getNewEnvironmentName(ctx context.Context) string { - lg := log.Ctx(ctx) - - workDir, err := os.Getwd() - if err != nil { - lg. - Fatal(). - Err(err). - Msg("failed to get current working dir") - } - - currentDir := filepath.Base(workDir) - if currentDir == "/" { - return "root" - } - - return currentDir -} - -func getPlanSource(ctx context.Context) dagger.Input { - lg := log.Ctx(ctx) - - src := dagger.Input{} - checkFirstSet := func() { - if src.Type != dagger.InputTypeEmpty { - lg.Fatal().Msg("only one of those options can be set: --plan-dir, --plan-git, --plan-package, --plan-file") - } - } - - planDir := viper.GetString("plan-dir") - planGit := viper.GetString("plan-git") - - if planDir != "" { - checkFirstSet() - - src = dagger.DirInput(planDir, []string{"*.cue", "cue.mod"}) - } - - if planGit != "" { - checkFirstSet() - - u, err := url.Parse(planGit) - if err != nil { - lg.Fatal().Err(err).Str("url", planGit).Msg("cannot get current working directory") - } - ref := u.Fragment // eg. #main - u.Fragment = "" - remote := u.String() - - src = dagger.GitInput(remote, ref, "") - } - - if src.Type == dagger.InputTypeEmpty { - var err error - wd, err := os.Getwd() - if err != nil { - lg.Fatal().Err(err).Msg("cannot get current working directory") - } - return dagger.DirInput(wd, []string{"*.cue", "cue.mod"}) - } - - return src -} - -func init() { - newCmd.Flags().BoolP("up", "u", false, "Bring the environment online") - - newCmd.Flags().String("plan-dir", "", "Load plan from a local directory") - newCmd.Flags().String("plan-git", "", "Load plan from a git repository") - newCmd.Flags().String("plan-package", "", "Load plan from a cue package") - newCmd.Flags().String("plan-file", "", "Load plan from a cue or json file") - - newCmd.Flags().String("setup", "auto", "Specify whether to prompt user for initial setup (no|yes|auto)") - - if err := viper.BindPFlags(newCmd.Flags()); err != nil { - panic(err) - } -} diff --git a/cmd/dagger/cmd/plan/dir.go b/cmd/dagger/cmd/plan/dir.go deleted file mode 100644 index 10ccf853..00000000 --- a/cmd/dagger/cmd/plan/dir.go +++ /dev/null @@ -1,33 +0,0 @@ -package plan - -import ( - "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var dirCmd = &cobra.Command{ - Use: "dir PATH", - Short: "Load plan from a local directory", - Args: cobra.ExactArgs(1), - PreRun: func(cmd *cobra.Command, args []string) { - // Fix Viper bug for duplicate flags: - // https://github.com/spf13/viper/issues/233 - if err := viper.BindPFlags(cmd.Flags()); err != nil { - panic(err) - } - }, - Run: func(cmd *cobra.Command, args []string) { - lg := logger.New() - ctx := lg.WithContext(cmd.Context()) - - updateEnvironmentPlan(ctx, dagger.DirInput(args[0], []string{"*.cue", "cue.mod"})) - }, -} - -func init() { - if err := viper.BindPFlags(dirCmd.Flags()); err != nil { - panic(err) - } -} diff --git a/cmd/dagger/cmd/plan/file.go b/cmd/dagger/cmd/plan/file.go deleted file mode 100644 index 39f12a1b..00000000 --- a/cmd/dagger/cmd/plan/file.go +++ /dev/null @@ -1,31 +0,0 @@ -package plan - -import ( - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var fileCmd = &cobra.Command{ - Use: "file PATH|-", - Short: "Load plan from a cue file", - Args: cobra.ExactArgs(1), - PreRun: func(cmd *cobra.Command, args []string) { - // Fix Viper bug for duplicate flags: - // https://github.com/spf13/viper/issues/233 - if err := viper.BindPFlags(cmd.Flags()); err != nil { - panic(err) - } - }, - Run: func(cmd *cobra.Command, args []string) { - // lg := logger.New() - // ctx := lg.WithContext(cmd.Context()) - - panic("not implemented") - }, -} - -func init() { - if err := viper.BindPFlags(fileCmd.Flags()); err != nil { - panic(err) - } -} diff --git a/cmd/dagger/cmd/plan/git.go b/cmd/dagger/cmd/plan/git.go deleted file mode 100644 index 35153ba3..00000000 --- a/cmd/dagger/cmd/plan/git.go +++ /dev/null @@ -1,43 +0,0 @@ -package plan - -import ( - "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var gitCmd = &cobra.Command{ - Use: "git REMOTE [REF] [SUBDIR]", - Short: "Load plan from a git package", - Args: cobra.RangeArgs(1, 3), - PreRun: func(cmd *cobra.Command, args []string) { - // Fix Viper bug for duplicate flags: - // https://github.com/spf13/viper/issues/233 - if err := viper.BindPFlags(cmd.Flags()); err != nil { - panic(err) - } - }, - Run: func(cmd *cobra.Command, args []string) { - lg := logger.New() - ctx := lg.WithContext(cmd.Context()) - - ref := "HEAD" - if len(args) > 1 { - ref = args[1] - } - - subDir := "" - if len(args) > 2 { - subDir = args[2] - } - - updateEnvironmentPlan(ctx, dagger.GitInput(args[0], ref, subDir)) - }, -} - -func init() { - if err := viper.BindPFlags(gitCmd.Flags()); err != nil { - panic(err) - } -} diff --git a/cmd/dagger/cmd/plan/package.go b/cmd/dagger/cmd/plan/package.go deleted file mode 100644 index 215225cb..00000000 --- a/cmd/dagger/cmd/plan/package.go +++ /dev/null @@ -1,31 +0,0 @@ -package plan - -import ( - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var packageCmd = &cobra.Command{ - Use: "package PKG", - Short: "Load plan from a cue package", - Args: cobra.ExactArgs(1), - PreRun: func(cmd *cobra.Command, args []string) { - // Fix Viper bug for duplicate flags: - // https://github.com/spf13/viper/issues/233 - if err := viper.BindPFlags(cmd.Flags()); err != nil { - panic(err) - } - }, - Run: func(cmd *cobra.Command, args []string) { - // lg := logger.New() - // ctx := lg.WithContext(cmd.Context()) - - panic("not implemented") - }, -} - -func init() { - if err := viper.BindPFlags(packageCmd.Flags()); err != nil { - panic(err) - } -} diff --git a/cmd/dagger/cmd/plan/root.go b/cmd/dagger/cmd/plan/root.go deleted file mode 100644 index 0bf82327..00000000 --- a/cmd/dagger/cmd/plan/root.go +++ /dev/null @@ -1,42 +0,0 @@ -package plan - -import ( - "context" - - "dagger.io/go/cmd/dagger/cmd/common" - "dagger.io/go/dagger" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -// Cmd exposes the top-level command -var Cmd = &cobra.Command{ - Use: "plan", - Short: "Manage an environment's plan", -} - -func init() { - Cmd.AddCommand( - packageCmd, - dirCmd, - gitCmd, - fileCmd, - ) -} - -func updateEnvironmentPlan(ctx context.Context, planSource dagger.Input) { - lg := log.Ctx(ctx) - - store, err := dagger.DefaultStore() - if err != nil { - lg.Fatal().Err(err).Msg("failed to load store") - } - - st := common.GetCurrentEnvironmentState(ctx, store) - st.PlanSource = planSource - - if err := store.UpdateEnvironment(ctx, st, nil); err != nil { - lg.Fatal().Err(err).Str("environmentId", st.ID).Str("environmentName", st.Name).Msg("cannot update environment") - } - lg.Info().Str("environmentId", st.ID).Str("environmentName", st.Name).Msg("updated environment") -} diff --git a/cmd/dagger/cmd/query.go b/cmd/dagger/cmd/query.go index f64933b3..62981608 100644 --- a/cmd/dagger/cmd/query.go +++ b/cmd/dagger/cmd/query.go @@ -30,16 +30,10 @@ var queryCmd = &cobra.Command{ cueOpts := parseQueryFlags() - store, err := dagger.DefaultStore() - if err != nil { - lg.Fatal().Err(err).Msg("failed to load store") - } - - state := common.GetCurrentEnvironmentState(ctx, store) + state := common.GetCurrentEnvironmentState(ctx) lg = lg.With(). - Str("environmentName", state.Name). - Str("environmentId", state.ID). + Str("environment", state.Name). Logger() cuePath := cue.MakePath() diff --git a/cmd/dagger/cmd/root.go b/cmd/dagger/cmd/root.go index 2c5da999..09868d79 100644 --- a/cmd/dagger/cmd/root.go +++ b/cmd/dagger/cmd/root.go @@ -6,7 +6,6 @@ import ( "dagger.io/go/cmd/dagger/cmd/input" "dagger.io/go/cmd/dagger/cmd/output" - "dagger.io/go/cmd/dagger/cmd/plan" "dagger.io/go/cmd/dagger/logger" "github.com/moby/buildkit/util/appcontext" "github.com/opentracing/opentracing-go" @@ -24,6 +23,7 @@ func init() { rootCmd.PersistentFlags().String("log-format", "", "Log format (json, pretty). Defaults to json if the terminal is not a tty") rootCmd.PersistentFlags().StringP("log-level", "l", "info", "Log level") rootCmd.PersistentFlags().StringP("environment", "e", "", "Select an environment") + rootCmd.PersistentFlags().StringP("workspace", "w", "", "Specify a workspace (defaults to current git repository)") rootCmd.PersistentPreRun = func(*cobra.Command, []string) { go checkVersion() @@ -33,8 +33,8 @@ func init() { } rootCmd.AddCommand( + initCmd, computeCmd, - newCmd, listCmd, queryCmd, upCmd, @@ -43,7 +43,6 @@ func init() { historyCmd, loginCmd, logoutCmd, - plan.Cmd, input.Cmd, output.Cmd, versionCmd, diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 2daed27e..808313e6 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -3,7 +3,7 @@ package cmd import ( "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -23,15 +23,12 @@ var upCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { lg := logger.New() ctx := lg.WithContext(cmd.Context()) - store, err := dagger.DefaultStore() - if err != nil { - lg.Fatal().Err(err).Msg("failed to load store") - } - state := common.GetCurrentEnvironmentState(ctx, store) - result := common.EnvironmentUp(ctx, state, viper.GetBool("no-cache")) - state.Computed = result.Computed().JSON().String() - if err := store.UpdateEnvironment(ctx, state, nil); err != nil { + st := common.GetCurrentEnvironmentState(ctx) + result := common.EnvironmentUp(ctx, st, viper.GetBool("no-cache")) + + st.Computed = result.Computed().JSON().PrettyString() + if err := state.Save(ctx, st); err != nil { lg.Fatal().Err(err).Msg("failed to update environment") } }, diff --git a/dagger/client.go b/dagger/client.go index 1ea41978..3fcd8ff3 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -26,6 +26,7 @@ import ( "dagger.io/go/pkg/progressui" "dagger.io/go/dagger/compiler" + "dagger.io/go/dagger/state" ) // A dagger client @@ -63,7 +64,7 @@ func NewClient(ctx context.Context, host string, noCache bool) (*Client, error) type ClientDoFunc func(context.Context, *Environment, Solver) error // FIXME: return completed *Route, instead of *compiler.Value -func (c *Client) Do(ctx context.Context, state *EnvironmentState, fn ClientDoFunc) (*Environment, error) { +func (c *Client) Do(ctx context.Context, state *state.State, fn ClientDoFunc) (*Environment, error) { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) diff --git a/dagger/compiler/json.go b/dagger/compiler/json.go index e138f7af..4f67da92 100644 --- a/dagger/compiler/json.go +++ b/dagger/compiler/json.go @@ -112,5 +112,5 @@ func (s JSON) PrettyString() string { if err := json.Indent(b, []byte(raw), "", " "); err != nil { return raw } - return b.String() + return fmt.Sprintf("%s\n", b.String()) } diff --git a/dagger/environment.go b/dagger/environment.go index c0177676..b4048cd4 100644 --- a/dagger/environment.go +++ b/dagger/environment.go @@ -10,6 +10,8 @@ import ( "cuelang.org/go/cue" cueflow "cuelang.org/go/tools/flow" "dagger.io/go/dagger/compiler" + "dagger.io/go/dagger/state" + "dagger.io/go/pkg/cuetils" "dagger.io/go/stdlib" "github.com/opentracing/opentracing-go" @@ -19,7 +21,7 @@ import ( ) type Environment struct { - state *EnvironmentState + state *state.State // Layer 1: plan configuration plan *compiler.Value @@ -31,7 +33,7 @@ type Environment struct { computed *compiler.Value } -func NewEnvironment(st *EnvironmentState) (*Environment, error) { +func NewEnvironment(st *state.State) (*Environment, error) { e := &Environment{ state: st, @@ -42,7 +44,7 @@ func NewEnvironment(st *EnvironmentState) (*Environment, error) { // Prepare inputs for _, input := range st.Inputs { - v, err := input.Value.Compile() + v, err := input.Value.Compile(st) if err != nil { return nil, err } @@ -59,16 +61,12 @@ func NewEnvironment(st *EnvironmentState) (*Environment, error) { return e, nil } -func (e *Environment) ID() string { - return e.state.ID -} - func (e *Environment) Name() string { return e.state.Name } -func (e *Environment) PlanSource() Input { - return e.state.PlanSource +func (e *Environment) PlanSource() state.Input { + return e.state.PlanSource() } func (e *Environment) Plan() *compiler.Value { @@ -88,7 +86,7 @@ func (e *Environment) LoadPlan(ctx context.Context, s Solver) error { span, ctx := opentracing.StartSpanFromContext(ctx, "environment.LoadPlan") defer span.Finish() - planSource, err := e.state.PlanSource.Compile() + planSource, err := e.state.PlanSource().Compile(e.state) if err != nil { return err } @@ -159,7 +157,7 @@ func (e *Environment) LocalDirs() map[string]string { } // 2. Scan the plan - plan, err := e.state.PlanSource.Compile() + plan, err := e.state.PlanSource().Compile(e.state) if err != nil { panic(err) } diff --git a/dagger/input_test.go b/dagger/environment_test.go similarity index 61% rename from dagger/input_test.go rename to dagger/environment_test.go index 8fe60262..cb200b8d 100644 --- a/dagger/input_test.go +++ b/dagger/environment_test.go @@ -3,14 +3,15 @@ package dagger import ( "testing" + "dagger.io/go/dagger/state" "github.com/stretchr/testify/require" ) -func TestInputDir(t *testing.T) { - st := &EnvironmentState{ - PlanSource: DirInput("/tmp/source", []string{}), +func TestLocalDirs(t *testing.T) { + st := &state.State{ + Path: "/tmp/source", } - require.NoError(t, st.SetInput("www.source", DirInput("/", []string{}))) + require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{}))) environment, err := NewEnvironment(st) require.NoError(t, err) diff --git a/dagger/input.go b/dagger/state/input.go similarity index 77% rename from dagger/input.go rename to dagger/state/input.go index 563cb017..c1ad5676 100644 --- a/dagger/input.go +++ b/dagger/state/input.go @@ -1,9 +1,10 @@ -package dagger +package state import ( "encoding/json" "fmt" "io/ioutil" + "path" "path/filepath" "cuelang.org/go/cue" @@ -37,33 +38,33 @@ const ( ) type Input struct { - Type InputType `json:"type,omitempty"` + Type InputType `yaml:"type,omitempty"` - Dir *dirInput `json:"dir,omitempty"` - Git *gitInput `json:"git,omitempty"` - Docker *dockerInput `json:"docker,omitempty"` - Text *textInput `json:"text,omitempty"` - JSON *jsonInput `json:"json,omitempty"` - YAML *yamlInput `json:"yaml,omitempty"` - File *fileInput `json:"file,omitempty"` + Dir *dirInput `yaml:"dir,omitempty"` + Git *gitInput `yaml:"git,omitempty"` + Docker *dockerInput `yaml:"docker,omitempty"` + Text *textInput `yaml:"text,omitempty"` + JSON *jsonInput `yaml:"json,omitempty"` + YAML *yamlInput `yaml:"yaml,omitempty"` + File *fileInput `yaml:"file,omitempty"` } -func (i Input) Compile() (*compiler.Value, error) { +func (i Input) Compile(state *State) (*compiler.Value, error) { switch i.Type { case InputTypeDir: - return i.Dir.Compile() + return i.Dir.Compile(state) case InputTypeGit: - return i.Git.Compile() + return i.Git.Compile(state) case InputTypeDocker: - return i.Docker.Compile() + return i.Docker.Compile(state) case InputTypeText: - return i.Text.Compile() + return i.Text.Compile(state) case InputTypeJSON: - return i.JSON.Compile() + return i.JSON.Compile(state) case InputTypeYAML: - return i.YAML.Compile() + return i.YAML.Compile(state) case InputTypeFile: - return i.File.Compile() + return i.File.Compile(state) case "": return nil, fmt.Errorf("input has not been set") default: @@ -73,12 +74,6 @@ func (i Input) Compile() (*compiler.Value, error) { // An input artifact loaded from a local directory func DirInput(path string, include []string) Input { - // resolve absolute path - path, err := filepath.Abs(path) - if err != nil { - panic(err) - } - return Input{ Type: InputTypeDir, Dir: &dirInput{ @@ -93,7 +88,7 @@ type dirInput struct { Include []string `json:"include,omitempty"` } -func (dir dirInput) Compile() (*compiler.Value, error) { +func (dir dirInput) Compile(state *State) (*compiler.Value, error) { // FIXME: serialize an intermediate struct, instead of generating cue source // json.Marshal([]string{}) returns []byte("null"), which wreaks havoc @@ -106,9 +101,15 @@ func (dir dirInput) Compile() (*compiler.Value, error) { return nil, err } } + + p := dir.Path + if !filepath.IsAbs(p) { + p = filepath.Clean(path.Join(state.Path, p)) + } + llb := fmt.Sprintf( `#up: [{do:"local",dir:"%s", include:%s}]`, - dir.Path, + p, includeLLB, ) return compiler.Compile("", llb) @@ -132,7 +133,7 @@ func GitInput(remote, ref, dir string) Input { } } -func (git gitInput) Compile() (*compiler.Value, error) { +func (git gitInput) Compile(_ *State) (*compiler.Value, error) { ref := "HEAD" if git.Ref != "" { ref = git.Ref @@ -159,7 +160,7 @@ type dockerInput struct { Ref string `json:"ref,omitempty"` } -func (i dockerInput) Compile() (*compiler.Value, error) { +func (i dockerInput) Compile(_ *State) (*compiler.Value, error) { panic("NOT IMPLEMENTED") } @@ -177,7 +178,7 @@ type textInput struct { Data string `json:"data,omitempty"` } -func (i textInput) Compile() (*compiler.Value, error) { +func (i textInput) Compile(_ *State) (*compiler.Value, error) { return compiler.Compile("", fmt.Sprintf("%q", i.Data)) } @@ -196,7 +197,7 @@ type jsonInput struct { Data string `json:"data,omitempty"` } -func (i jsonInput) Compile() (*compiler.Value, error) { +func (i jsonInput) Compile(_ *State) (*compiler.Value, error) { return compiler.DecodeJSON("", []byte(i.Data)) } @@ -215,7 +216,7 @@ type yamlInput struct { Data string `json:"data,omitempty"` } -func (i yamlInput) Compile() (*compiler.Value, error) { +func (i yamlInput) Compile(_ *State) (*compiler.Value, error) { return compiler.DecodeYAML("", []byte(i.Data)) } @@ -232,7 +233,7 @@ type fileInput struct { Path string `json:"data,omitempty"` } -func (i fileInput) Compile() (*compiler.Value, error) { +func (i fileInput) Compile(_ *State) (*compiler.Value, error) { data, err := ioutil.ReadFile(i.Path) if err != nil { return nil, err diff --git a/dagger/state.go b/dagger/state/state.go similarity index 58% rename from dagger/state.go rename to dagger/state/state.go index 6276eaa9..619824c5 100644 --- a/dagger/state.go +++ b/dagger/state/state.go @@ -1,32 +1,34 @@ -package dagger +package state // Contents of an environment serialized to a file -type EnvironmentState struct { - // Globally unique environment ID - ID string `json:"id,omitempty"` +type State struct { + // State path + Path string `yaml:"-"` // Human-friendly environment name. // A environment may have more than one name. // FIXME: store multiple names? - Name string `json:"name,omitempty"` - - // Cue module containing the environment plan - // The input's top-level artifact is used as a module directory. - PlanSource Input `json:"plan,omitempty"` + Name string `yaml:"name,omitempty"` // User Inputs - Inputs []inputKV `json:"inputs,omitempty"` + Inputs []inputKV `yaml:"inputs,omitempty"` // Computed values - Computed string `json:"output,omitempty"` + Computed string `yaml:"-"` } type inputKV struct { - Key string `json:"key,omitempty"` - Value Input `json:"value,omitempty"` + Key string `yaml:"key,omitempty"` + Value Input `yaml:"value,omitempty"` } -func (s *EnvironmentState) SetInput(key string, value Input) error { +// Cue module containing the environment plan +// The input's top-level artifact is used as a module directory. +func (s *State) PlanSource() Input { + return DirInput(s.Path, []string{"*.cue", "cue.mod"}) +} + +func (s *State) SetInput(key string, value Input) error { for i, inp := range s.Inputs { if inp.Key != key { continue @@ -42,7 +44,7 @@ func (s *EnvironmentState) SetInput(key string, value Input) error { // Remove all inputs at the given key, including sub-keys. // For example RemoveInputs("foo.bar") will remove all inputs // at foo.bar, foo.bar.baz, etc. -func (s *EnvironmentState) RemoveInputs(key string) error { +func (s *State) RemoveInputs(key string) error { newInputs := make([]inputKV, 0, len(s.Inputs)) for _, i := range s.Inputs { if i.Key == key { diff --git a/dagger/state/store.go b/dagger/state/store.go new file mode 100644 index 00000000..966d4494 --- /dev/null +++ b/dagger/state/store.go @@ -0,0 +1,205 @@ +package state + +import ( + "context" + "errors" + "os" + "path" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +var ( + ErrNotInit = errors.New("not initialized") + ErrAlreadyInit = errors.New("already initialized") + ErrNoCurrentWorkspace = errors.New("not in a git directory") +) + +const ( + daggerDir = ".dagger" + stateDir = "state" + manifestFile = "values.yaml" + computedFile = "computed.json" +) + +func Init(ctx context.Context, dir, name string) (*State, error) { + root := path.Join(dir, daggerDir) + err := os.Mkdir(root, 0755) + if err != nil { + if errors.Is(err, os.ErrExist) { + return nil, ErrAlreadyInit + } + return nil, err + } + + err = os.WriteFile( + path.Join(root, ".gitignore"), + []byte("# dagger state\nstate/**\n"), + 0600, + ) + if err != nil { + return nil, err + } + + st := &State{ + Path: dir, + Name: name, + } + + return st, Save(ctx, st) +} + +func Current(ctx context.Context) (*State, error) { + current, err := os.Getwd() + if err != nil { + return nil, err + } + + // Walk every parent directory to find .dagger + for { + _, err := os.Stat(path.Join(current, daggerDir)) + if err == nil { + return Open(ctx, current) + } + parent := filepath.Dir(current) + if parent == current { + break + } + current = parent + } + + return nil, ErrNotInit +} + +func Open(ctx context.Context, dir string) (*State, error) { + _, err := os.Stat(path.Join(dir, daggerDir)) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, ErrNotInit + } + return nil, err + } + + root, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + data, err := os.ReadFile(path.Join(root, daggerDir, manifestFile)) + if err != nil { + return nil, err + } + var st State + if err := yaml.Unmarshal(data, &st); err != nil { + return nil, err + } + st.Path = root + + computed, err := os.ReadFile(path.Join(root, daggerDir, stateDir, computedFile)) + if err == nil { + st.Computed = string(computed) + } + + return &st, nil +} + +func Save(ctx context.Context, st *State) error { + data, err := yaml.Marshal(st) + if err != nil { + return err + } + + if err := os.WriteFile(path.Join(st.Path, daggerDir, manifestFile), data, 0600); err != nil { + return err + } + + if st.Computed != "" { + state := path.Join(st.Path, daggerDir, stateDir) + if err := os.MkdirAll(state, 0755); err != nil { + return err + } + err := os.WriteFile( + path.Join(state, "computed.json"), + []byte(st.Computed), + 0600) + if err != nil { + return err + } + } + + return nil +} + +func CurrentWorkspace(ctx context.Context) (string, error) { + current, err := os.Getwd() + if err != nil { + return "", err + } + + // Walk every parent directory to find .dagger + for { + _, err := os.Stat(path.Join(current, ".git")) + if err == nil { + return current, nil + } + parent := filepath.Dir(current) + if parent == current { + break + } + current = parent + } + + return "", ErrNoCurrentWorkspace +} + +func List(ctx context.Context, workspace string) ([]*State, error) { + var ( + environments = []*State{} + err error + ) + + workspace, err = filepath.Abs(workspace) + if err != nil { + return nil, err + } + + err = filepath.WalkDir(workspace, func(p string, info os.DirEntry, err error) error { + // Ignore errors while we walk + if err != nil { + return nil + } + + // Skip regular files + if !info.IsDir() { + return nil + } + + // Skip non-dagger directories + if info.Name() != daggerDir { + // Caveat: limit traversal to a depth of 10 (arbitrary) + relPath := strings.TrimPrefix(p, workspace) + if strings.Count(relPath, string(os.PathSeparator)) > 10 { + return filepath.SkipDir + } + + // Otherwise, continue traversing + return nil + } + + st, err := Open(ctx, filepath.Dir(p)) + if err != nil { + return err + } + environments = append(environments, st) + + return nil + }) + + if err != nil { + return nil, err + } + + return environments, nil +} diff --git a/dagger/state/store_test.go b/dagger/state/store_test.go new file mode 100644 index 00000000..d5ec0afd --- /dev/null +++ b/dagger/state/store_test.go @@ -0,0 +1,39 @@ +package state + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStore(t *testing.T) { + ctx := context.TODO() + + // Init + root, err := os.MkdirTemp(os.TempDir(), "dagger-*") + require.NoError(t, err) + st, err := Init(ctx, root, "test") + require.Equal(t, "test", st.Name) + require.Equal(t, root, st.Path) + require.NoError(t, err) + + // Open + _, err = Open(ctx, "/tmp/not/exist") + require.Error(t, err) + require.ErrorIs(t, ErrNotInit, err) + + st, err = Open(ctx, root) + require.NoError(t, err) + require.Equal(t, "test", st.Name) + require.Equal(t, root, st.Path) + + // Save + computed := `{"hello": "world"}` + st.Computed = computed + require.NoError(t, Save(ctx, st)) + st, err = Open(ctx, root) + require.NoError(t, err) + require.Equal(t, computed, st.Computed) +} diff --git a/dagger/store.go b/dagger/store.go deleted file mode 100644 index d203cb70..00000000 --- a/dagger/store.go +++ /dev/null @@ -1,250 +0,0 @@ -package dagger - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path" - "sync" - - "github.com/google/uuid" -) - -var ( - ErrEnvironmentExist = errors.New("environment already exists") - ErrEnvironmentNotExist = errors.New("environment doesn't exist") -) - -const ( - defaultStoreRoot = "$HOME/.dagger/store" -) - -type Store struct { - root string - - l sync.RWMutex - - // ID -> Environment - environments map[string]*EnvironmentState - - // Name -> Environment - environmentsByName map[string]*EnvironmentState - - // Path -> (ID->Environment) - environmentsByPath map[string]map[string]*EnvironmentState - - // ID -> (Path->{}) - pathsByEnvironmentID map[string]map[string]struct{} -} - -func NewStore(root string) (*Store, error) { - store := &Store{ - root: root, - environments: make(map[string]*EnvironmentState), - environmentsByName: make(map[string]*EnvironmentState), - environmentsByPath: make(map[string]map[string]*EnvironmentState), - pathsByEnvironmentID: make(map[string]map[string]struct{}), - } - return store, store.loadAll() -} - -func DefaultStore() (*Store, error) { - if root := os.Getenv("DAGGER_STORE"); root != "" { - return NewStore(root) - } - - return NewStore(os.ExpandEnv(defaultStoreRoot)) -} - -func (s *Store) environmentPath(name string) string { - // FIXME: rename to environment.json ? - return path.Join(s.root, name, "deployment.json") -} - -func (s *Store) loadAll() error { - files, err := os.ReadDir(s.root) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil - } - return err - } - - for _, f := range files { - if !f.IsDir() { - continue - } - if err := s.loadEnvironment(f.Name()); err != nil { - return err - } - } - - return nil -} - -func (s *Store) loadEnvironment(name string) error { - data, err := os.ReadFile(s.environmentPath(name)) - if err != nil { - return err - } - var st EnvironmentState - if err := json.Unmarshal(data, &st); err != nil { - return err - } - s.indexEnvironment(&st) - return nil -} - -func (s *Store) syncEnvironment(r *EnvironmentState) error { - p := s.environmentPath(r.Name) - - if err := os.MkdirAll(path.Dir(p), 0755); err != nil { - return err - } - - data, err := json.MarshalIndent(r, "", " ") - if err != nil { - return err - } - - if err := os.WriteFile(p, data, 0600); err != nil { - return err - } - - s.reindexEnvironment(r) - - return nil -} - -func (s *Store) indexEnvironment(r *EnvironmentState) { - s.environments[r.ID] = r - s.environmentsByName[r.Name] = r - - mapPath := func(i Input) { - if i.Type != InputTypeDir { - return - } - if s.environmentsByPath[i.Dir.Path] == nil { - s.environmentsByPath[i.Dir.Path] = make(map[string]*EnvironmentState) - } - s.environmentsByPath[i.Dir.Path][r.ID] = r - - if s.pathsByEnvironmentID[r.ID] == nil { - s.pathsByEnvironmentID[r.ID] = make(map[string]struct{}) - } - s.pathsByEnvironmentID[r.ID][i.Dir.Path] = struct{}{} - } - - mapPath(r.PlanSource) - for _, i := range r.Inputs { - mapPath(i.Value) - } -} - -func (s *Store) deindexEnvironment(id string) { - r, ok := s.environments[id] - if !ok { - return - } - delete(s.environments, r.ID) - delete(s.environmentsByName, r.Name) - - for p := range s.pathsByEnvironmentID[r.ID] { - delete(s.environmentsByPath[p], r.ID) - } - delete(s.pathsByEnvironmentID, r.ID) -} - -func (s *Store) reindexEnvironment(r *EnvironmentState) { - s.deindexEnvironment(r.ID) - s.indexEnvironment(r) -} - -func (s *Store) CreateEnvironment(ctx context.Context, st *EnvironmentState) error { - s.l.Lock() - defer s.l.Unlock() - - if _, ok := s.environmentsByName[st.Name]; ok { - return fmt.Errorf("%s: %w", st.Name, ErrEnvironmentExist) - } - - st.ID = uuid.New().String() - return s.syncEnvironment(st) -} - -type UpdateOpts struct{} - -func (s *Store) UpdateEnvironment(ctx context.Context, r *EnvironmentState, o *UpdateOpts) error { - s.l.Lock() - defer s.l.Unlock() - - return s.syncEnvironment(r) -} - -type DeleteOpts struct{} - -func (s *Store) DeleteEnvironment(ctx context.Context, r *EnvironmentState, o *DeleteOpts) error { - s.l.Lock() - defer s.l.Unlock() - - if err := os.Remove(s.environmentPath(r.Name)); err != nil { - return err - } - s.deindexEnvironment(r.ID) - return nil -} - -func (s *Store) LookupEnvironmentByID(ctx context.Context, id string) (*EnvironmentState, error) { - s.l.RLock() - defer s.l.RUnlock() - - st, ok := s.environments[id] - if !ok { - return nil, fmt.Errorf("%s: %w", id, ErrEnvironmentNotExist) - } - return st, nil -} - -func (s *Store) LookupEnvironmentByName(ctx context.Context, name string) (*EnvironmentState, error) { - s.l.RLock() - defer s.l.RUnlock() - - st, ok := s.environmentsByName[name] - if !ok { - return nil, fmt.Errorf("%s: %w", name, ErrEnvironmentNotExist) - } - return st, nil -} - -func (s *Store) LookupEnvironmentByPath(ctx context.Context, path string) ([]*EnvironmentState, error) { - s.l.RLock() - defer s.l.RUnlock() - - res := []*EnvironmentState{} - - environments, ok := s.environmentsByPath[path] - if !ok { - return res, nil - } - - for _, d := range environments { - res = append(res, d) - } - - return res, nil -} - -func (s *Store) ListEnvironments(ctx context.Context) ([]*EnvironmentState, error) { - s.l.RLock() - defer s.l.RUnlock() - - environments := make([]*EnvironmentState, 0, len(s.environments)) - - for _, st := range s.environments { - environments = append(environments, st) - } - - return environments, nil -} diff --git a/dagger/store_test.go b/dagger/store_test.go deleted file mode 100644 index 2f9d6332..00000000 --- a/dagger/store_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package dagger - -import ( - "context" - "errors" - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestStoreLoad(t *testing.T) { - ctx := context.TODO() - - root, err := os.MkdirTemp(os.TempDir(), "dagger-*") - require.NoError(t, err) - store, err := NewStore(root) - require.NoError(t, err) - - _, err = store.LookupEnvironmentByName(ctx, "notexist") - require.Error(t, err) - require.True(t, errors.Is(err, ErrEnvironmentNotExist)) - - st := &EnvironmentState{ - Name: "test", - } - require.NoError(t, store.CreateEnvironment(ctx, st)) - - checkEnvironments := func(store *Store) { - r, err := store.LookupEnvironmentByID(ctx, st.ID) - require.NoError(t, err) - require.NotNil(t, r) - require.Equal(t, "test", r.Name) - - r, err = store.LookupEnvironmentByName(ctx, "test") - require.NoError(t, err) - require.NotNil(t, r) - require.Equal(t, "test", r.Name) - - environments, err := store.ListEnvironments(ctx) - require.NoError(t, err) - require.Len(t, environments, 1) - require.Equal(t, "test", environments[0].Name) - } - - checkEnvironments(store) - - // Reload the environments from disk and check again - newStore, err := NewStore(root) - require.NoError(t, err) - checkEnvironments(newStore) -} - -func TestStoreLookupByPath(t *testing.T) { - ctx := context.TODO() - - root, err := os.MkdirTemp(os.TempDir(), "dagger-*") - require.NoError(t, err) - store, err := NewStore(root) - require.NoError(t, err) - - st := &EnvironmentState{ - Name: "test", - } - require.NoError(t, st.SetInput("foo", DirInput("/test/path", []string{}))) - require.NoError(t, store.CreateEnvironment(ctx, st)) - - // Lookup by path - environments, err := store.LookupEnvironmentByPath(ctx, "/test/path") - require.NoError(t, err) - require.Len(t, environments, 1) - require.Equal(t, st.ID, environments[0].ID) - - // Add a new path - require.NoError(t, st.SetInput("bar", DirInput("/test/anotherpath", []string{}))) - require.NoError(t, store.UpdateEnvironment(ctx, st, nil)) - - // Lookup by the previous path - environments, err = store.LookupEnvironmentByPath(ctx, "/test/path") - require.NoError(t, err) - require.Len(t, environments, 1) - require.Equal(t, st.ID, environments[0].ID) - - // Lookup by the new path - environments, err = store.LookupEnvironmentByPath(ctx, "/test/anotherpath") - require.NoError(t, err) - require.Len(t, environments, 1) - require.Equal(t, st.ID, environments[0].ID) - - // Remove a path - require.NoError(t, st.RemoveInputs("foo")) - require.NoError(t, store.UpdateEnvironment(ctx, st, nil)) - - // Lookup by the removed path should fail - environments, err = store.LookupEnvironmentByPath(ctx, "/test/path") - require.NoError(t, err) - require.Len(t, environments, 0) - - // Lookup by the other path should still work - environments, err = store.LookupEnvironmentByPath(ctx, "/test/anotherpath") - require.NoError(t, err) - require.Len(t, environments, 1) - - // Add another environment using the same path - otherSt := &EnvironmentState{ - Name: "test2", - } - require.NoError(t, otherSt.SetInput("foo", DirInput("/test/anotherpath", []string{}))) - require.NoError(t, store.CreateEnvironment(ctx, otherSt)) - - // Lookup by path should return both environments - environments, err = store.LookupEnvironmentByPath(ctx, "/test/anotherpath") - require.NoError(t, err) - require.Len(t, environments, 2) - - // Remove the first environment. Lookup by path should still return the - // second environment. - require.NoError(t, store.DeleteEnvironment(ctx, st, nil)) - environments, err = store.LookupEnvironmentByPath(ctx, "/test/anotherpath") - require.NoError(t, err) - require.Len(t, environments, 1) - require.Equal(t, otherSt.ID, environments[0].ID) -} diff --git a/go.mod b/go.mod index 5cfc1763..3029f551 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/containerd/console v1.0.2 github.com/docker/distribution v2.7.1+incompatible github.com/emicklei/proto v1.9.0 // indirect - github.com/google/uuid v1.2.0 github.com/hashicorp/go-version v1.3.0 github.com/jaguilar/vt100 v0.0.0-20150826170717-2703a27b14ea github.com/mattn/go-colorable v0.1.8 // indirect From 5442d6b261622a380e5d8ce3dfdb3e9d201e17ff Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 12 May 2021 17:12:40 -0700 Subject: [PATCH 02/16] remove support for environment paths Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/common/common.go | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index b928cffa..b7c87cee 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -2,8 +2,6 @@ package common import ( "context" - "errors" - "os" "dagger.io/go/dagger" "dagger.io/go/dagger/state" @@ -14,7 +12,7 @@ import ( func GetCurrentEnvironmentState(ctx context.Context) *state.State { lg := log.Ctx(ctx) - // 1) If no environment name has been given, look for the current environment + // If no environment name has been given, look for the current environment environment := viper.GetString("environment") if environment == "" { st, err := state.Current(ctx) @@ -27,21 +25,9 @@ func GetCurrentEnvironmentState(ctx context.Context) *state.State { return st } - // 2) Check if it's an environment path (can be opened directly) - st, err := state.Open(ctx, environment) - if err == nil { - return st - } - if !errors.Is(err, os.ErrNotExist) { - lg. - Fatal(). - Err(err). - Str("environmentPath", environment). - Msg("failed to load environment") - } - // At this point, it must be an environment name workspace := viper.GetString("workspace") + var err error if workspace == "" { workspace, err = state.CurrentWorkspace(ctx) if err != nil { From 9d416d65f72b8ca6a54005ab7f3ae7b97ea09ef7 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 17 May 2021 11:12:46 -0700 Subject: [PATCH 03/16] secret input type, simplify state format Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/input/list.go | 7 +-- cmd/dagger/cmd/input/secret.go | 37 ++++++++++++-- dagger/environment.go | 9 ++-- dagger/state/input.go | 93 ++++++++++++++-------------------- dagger/state/state.go | 28 ++-------- 5 files changed, 83 insertions(+), 91 deletions(-) diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 1ce2b636..22929be0 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -10,6 +10,7 @@ import ( "dagger.io/go/cmd/dagger/logger" "dagger.io/go/dagger" "dagger.io/go/dagger/compiler" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -84,9 +85,9 @@ var listCmd = &cobra.Command{ }, } -func isUserSet(env *dagger.EnvironmentState, val *compiler.Value) bool { - for _, i := range env.Inputs { - if val.Path().String() == i.Key { +func isUserSet(env *state.State, val *compiler.Value) bool { + for key := range env.Inputs { + if val.Path().String() == key { return true } } diff --git a/cmd/dagger/cmd/input/secret.go b/cmd/dagger/cmd/input/secret.go index 7701d046..24e4f29c 100644 --- a/cmd/dagger/cmd/input/secret.go +++ b/cmd/dagger/cmd/input/secret.go @@ -1,14 +1,20 @@ package input import ( + "fmt" + "syscall" + + "dagger.io/go/cmd/dagger/logger" + "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" + "golang.org/x/term" ) var secretCmd = &cobra.Command{ - Use: "secret TARGET VALUE", + Use: "secret [-f] []", Short: "Add an encrypted input secret", - Args: cobra.ExactArgs(2), + Args: cobra.RangeArgs(1, 2), PreRun: func(cmd *cobra.Command, args []string) { // Fix Viper bug for duplicate flags: // https://github.com/spf13/viper/issues/233 @@ -17,14 +23,35 @@ var secretCmd = &cobra.Command{ } }, Run: func(cmd *cobra.Command, args []string) { - // lg := logger.New() - // ctx := lg.WithContext(cmd.Context()) + lg := logger.New() + ctx := lg.WithContext(cmd.Context()) - panic("not implemented") + var secret string + if len(args) == 1 { + // No value specified: prompt terminal + fmt.Print("Secret: ") + data, err := term.ReadPassword(syscall.Stdin) + if err != nil { + lg.Fatal().Err(err).Msg("unable to read secret from terminal") + } + fmt.Println("") + secret = string(data) + } else { + // value specified: read it + secret = readInput(ctx, args[1]) + } + + updateEnvironmentInput( + ctx, + args[0], + state.SecretInput(secret), + ) }, } func init() { + secretCmd.Flags().BoolP("file", "f", false, "Read value from file") + if err := viper.BindPFlags(secretCmd.Flags()); err != nil { panic(err) } diff --git a/dagger/environment.go b/dagger/environment.go index b4048cd4..cddf579f 100644 --- a/dagger/environment.go +++ b/dagger/environment.go @@ -11,7 +11,6 @@ import ( cueflow "cuelang.org/go/tools/flow" "dagger.io/go/dagger/compiler" "dagger.io/go/dagger/state" - "dagger.io/go/pkg/cuetils" "dagger.io/go/stdlib" "github.com/opentracing/opentracing-go" @@ -43,15 +42,15 @@ func NewEnvironment(st *state.State) (*Environment, error) { } // Prepare inputs - for _, input := range st.Inputs { - v, err := input.Value.Compile(st) + for key, input := range st.Inputs { + v, err := input.Compile(st) if err != nil { return nil, err } - if input.Key == "" { + if key == "" { err = e.input.FillPath(cue.MakePath(), v) } else { - err = e.input.FillPath(cue.ParsePath(input.Key), v) + err = e.input.FillPath(cue.ParsePath(key), v) } if err != nil { return nil, err diff --git a/dagger/state/input.go b/dagger/state/input.go index c1ad5676..9c34bdd6 100644 --- a/dagger/state/input.go +++ b/dagger/state/input.go @@ -24,25 +24,12 @@ import ( // Under the hood, an artifact is encoded as a LLB pipeline, and // attached to the cue configuration as a // -type InputType string - -const ( - InputTypeDir InputType = "dir" - InputTypeGit InputType = "git" - InputTypeDocker InputType = "docker" - InputTypeText InputType = "text" - InputTypeJSON InputType = "json" - InputTypeYAML InputType = "yaml" - InputTypeFile InputType = "file" - InputTypeEmpty InputType = "" -) type Input struct { - Type InputType `yaml:"type,omitempty"` - Dir *dirInput `yaml:"dir,omitempty"` Git *gitInput `yaml:"git,omitempty"` Docker *dockerInput `yaml:"docker,omitempty"` + Secret *secretInput `yaml:"secret,omitempty"` Text *textInput `yaml:"text,omitempty"` JSON *jsonInput `yaml:"json,omitempty"` YAML *yamlInput `yaml:"yaml,omitempty"` @@ -50,32 +37,31 @@ type Input struct { } func (i Input) Compile(state *State) (*compiler.Value, error) { - switch i.Type { - case InputTypeDir: + switch { + case i.Dir != nil: return i.Dir.Compile(state) - case InputTypeGit: + case i.Git != nil: return i.Git.Compile(state) - case InputTypeDocker: + case i.Docker != nil: return i.Docker.Compile(state) - case InputTypeText: + case i.Text != nil: return i.Text.Compile(state) - case InputTypeJSON: + case i.Secret != nil: + return i.Secret.Compile(state) + case i.JSON != nil: return i.JSON.Compile(state) - case InputTypeYAML: + case i.YAML != nil: return i.YAML.Compile(state) - case InputTypeFile: + case i.File != nil: return i.File.Compile(state) - case "": - return nil, fmt.Errorf("input has not been set") default: - return nil, fmt.Errorf("unsupported input type: %s", i.Type) + return nil, fmt.Errorf("input has not been set") } } // An input artifact loaded from a local directory func DirInput(path string, include []string) Input { return Input{ - Type: InputTypeDir, Dir: &dirInput{ Path: path, Include: include, @@ -124,7 +110,6 @@ type gitInput struct { func GitInput(remote, ref, dir string) Input { return Input{ - Type: InputTypeGit, Git: &gitInput{ Remote: remote, Ref: ref, @@ -149,7 +134,6 @@ func (git gitInput) Compile(_ *State) (*compiler.Value, error) { // An input artifact loaded from a docker container func DockerInput(ref string) Input { return Input{ - Type: InputTypeDocker, Docker: &dockerInput{ Ref: ref, }, @@ -166,63 +150,62 @@ func (i dockerInput) Compile(_ *State) (*compiler.Value, error) { // An input value encoded as text func TextInput(data string) Input { + i := textInput(data) return Input{ - Type: InputTypeText, - Text: &textInput{ - Data: data, - }, + Text: &i, } } -type textInput struct { - Data string `json:"data,omitempty"` -} +type textInput string func (i textInput) Compile(_ *State) (*compiler.Value, error) { - return compiler.Compile("", fmt.Sprintf("%q", i.Data)) + return compiler.Compile("", fmt.Sprintf("%q", i)) +} + +// A secret input value +func SecretInput(data string) Input { + i := secretInput(data) + return Input{ + Secret: &i, + } +} + +type secretInput string + +func (i secretInput) Compile(_ *State) (*compiler.Value, error) { + return compiler.Compile("", fmt.Sprintf("%q", i)) } // An input value encoded as JSON func JSONInput(data string) Input { + i := jsonInput(data) return Input{ - Type: InputTypeJSON, - JSON: &jsonInput{ - Data: data, - }, + JSON: &i, } } -type jsonInput struct { - // Marshalled JSON data - Data string `json:"data,omitempty"` -} +type jsonInput string func (i jsonInput) Compile(_ *State) (*compiler.Value, error) { - return compiler.DecodeJSON("", []byte(i.Data)) + return compiler.DecodeJSON("", []byte(i)) } // An input value encoded as YAML func YAMLInput(data string) Input { + i := yamlInput(data) return Input{ - Type: InputTypeYAML, - YAML: &yamlInput{ - Data: data, - }, + YAML: &i, } } -type yamlInput struct { - // Marshalled YAML data - Data string `json:"data,omitempty"` -} +type yamlInput string func (i yamlInput) Compile(_ *State) (*compiler.Value, error) { - return compiler.DecodeYAML("", []byte(i.Data)) + return compiler.DecodeYAML("", []byte(i)) } func FileInput(data string) Input { return Input{ - Type: InputTypeFile, File: &fileInput{ Path: data, }, diff --git a/dagger/state/state.go b/dagger/state/state.go index 619824c5..49480ebd 100644 --- a/dagger/state/state.go +++ b/dagger/state/state.go @@ -11,17 +11,12 @@ type State struct { Name string `yaml:"name,omitempty"` // User Inputs - Inputs []inputKV `yaml:"inputs,omitempty"` + Inputs map[string]Input `yaml:"inputs,omitempty"` // Computed values Computed string `yaml:"-"` } -type inputKV struct { - Key string `yaml:"key,omitempty"` - Value Input `yaml:"value,omitempty"` -} - // Cue module containing the environment plan // The input's top-level artifact is used as a module directory. func (s *State) PlanSource() Input { @@ -29,15 +24,10 @@ func (s *State) PlanSource() Input { } func (s *State) SetInput(key string, value Input) error { - for i, inp := range s.Inputs { - if inp.Key != key { - continue - } - // Remove existing inputs with the same key - s.Inputs = append(s.Inputs[:i], s.Inputs[i+1:]...) + if s.Inputs == nil { + s.Inputs = make(map[string]Input) } - - s.Inputs = append(s.Inputs, inputKV{Key: key, Value: value}) + s.Inputs[key] = value return nil } @@ -45,14 +35,6 @@ func (s *State) SetInput(key string, value Input) error { // For example RemoveInputs("foo.bar") will remove all inputs // at foo.bar, foo.bar.baz, etc. func (s *State) RemoveInputs(key string) error { - newInputs := make([]inputKV, 0, len(s.Inputs)) - for _, i := range s.Inputs { - if i.Key == key { - continue - } - newInputs = append(newInputs, i) - } - s.Inputs = newInputs - + delete(s.Inputs, key) return nil } From 12436c20bc7db63c936dd5a8d1622ca5e1471d6f Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 17 May 2021 12:04:07 -0700 Subject: [PATCH 04/16] remove delete command Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/delete.go | 31 ------------------------------- cmd/dagger/cmd/root.go | 1 - 2 files changed, 32 deletions(-) delete mode 100644 cmd/dagger/cmd/delete.go diff --git a/cmd/dagger/cmd/delete.go b/cmd/dagger/cmd/delete.go deleted file mode 100644 index 37e6b404..00000000 --- a/cmd/dagger/cmd/delete.go +++ /dev/null @@ -1,31 +0,0 @@ -package cmd - -import ( - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var deleteCmd = &cobra.Command{ - Use: "delete", - Short: "Delete an environment after taking it offline (WARNING: may destroy infrastructure)", - Args: cobra.NoArgs, - PreRun: func(cmd *cobra.Command, args []string) { - // Fix Viper bug for duplicate flags: - // https://github.com/spf13/viper/issues/233 - if err := viper.BindPFlags(cmd.Flags()); err != nil { - panic(err) - } - }, - Run: func(cmd *cobra.Command, args []string) { - // lg := logger.New() - // ctx := lg.WithContext(cmd.Context()) - - panic("not implemented") - }, -} - -func init() { - if err := viper.BindPFlags(deleteCmd.Flags()); err != nil { - panic(err) - } -} diff --git a/cmd/dagger/cmd/root.go b/cmd/dagger/cmd/root.go index 09868d79..d3cc7cd8 100644 --- a/cmd/dagger/cmd/root.go +++ b/cmd/dagger/cmd/root.go @@ -39,7 +39,6 @@ func init() { queryCmd, upCmd, downCmd, - deleteCmd, historyCmd, loginCmd, logoutCmd, From f0156f449f52008556be51ed6d53c9afb156f616 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 17 May 2021 17:02:00 -0700 Subject: [PATCH 05/16] state encryption support Signed-off-by: Andrea Luzzardi --- dagger/keychain/encrypt.go | 123 +++++++++++++++++++++++++++++++++++++ dagger/keychain/keys.go | 96 +++++++++++++++++++++++++++++ dagger/state/store.go | 46 +++++++++++--- go.mod | 5 +- go.sum | 13 ++-- 5 files changed, 266 insertions(+), 17 deletions(-) create mode 100644 dagger/keychain/encrypt.go create mode 100644 dagger/keychain/keys.go diff --git a/dagger/keychain/encrypt.go b/dagger/keychain/encrypt.go new file mode 100644 index 00000000..bd78ea01 --- /dev/null +++ b/dagger/keychain/encrypt.go @@ -0,0 +1,123 @@ +package keychain + +import ( + "context" + "fmt" + "os" + + "go.mozilla.org/sops/v3" + "go.mozilla.org/sops/v3/aes" + sopsage "go.mozilla.org/sops/v3/age" + "go.mozilla.org/sops/v3/cmd/sops/common" + "go.mozilla.org/sops/v3/cmd/sops/formats" + sopsdecrypt "go.mozilla.org/sops/v3/decrypt" + sopskeys "go.mozilla.org/sops/v3/keys" + sopsyaml "go.mozilla.org/sops/v3/stores/yaml" + "go.mozilla.org/sops/v3/version" +) + +// setupEnv: hack to inject a SOPS env var for age +func setupEnv() error { + p, err := Path() + if err != nil { + return err + } + return os.Setenv("SOPS_AGE_KEY_FILE", p) +} + +// Encrypt data using SOPS with the AGE backend, using the provided public key +func Encrypt(ctx context.Context, path string, plaintext []byte, key string) ([]byte, error) { + if err := setupEnv(); err != nil { + return nil, err + } + + store := &sopsyaml.Store{} + branches, err := store.LoadPlainFile(plaintext) + if err != nil { + return nil, err + } + + ageKeys, err := sopsage.MasterKeysFromRecipients(key) + if err != nil { + return nil, err + } + ageMasterKeys := make([]sopskeys.MasterKey, 0, len(ageKeys)) + for _, k := range ageKeys { + ageMasterKeys = append(ageMasterKeys, k) + } + var group sops.KeyGroup + group = append(group, ageMasterKeys...) + + tree := sops.Tree{ + Branches: branches, + Metadata: sops.Metadata{ + KeyGroups: []sops.KeyGroup{group}, + EncryptedSuffix: "secret", + Version: version.Version, + }, + FilePath: path, + } + + // Generate a data key + dataKey, errs := tree.GenerateDataKey() + if len(errs) > 0 { + return nil, fmt.Errorf("error encrypting the data key with one or more master keys: %v", errs) + } + + err = common.EncryptTree(common.EncryptTreeOpts{ + DataKey: dataKey, Tree: &tree, Cipher: aes.NewCipher(), + }) + if err != nil { + return nil, err + } + return store.EmitEncryptedFile(tree) +} + +// Reencrypt a file with new content using the same keys +func Reencrypt(_ context.Context, path string, plaintext []byte) ([]byte, error) { + if err := setupEnv(); err != nil { + return nil, err + } + + current, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + // Load the encrypted file + store := &sopsyaml.Store{} + tree, err := store.LoadEncryptedFile(current) + if err != nil { + return nil, err + } + + // Update the file with the new data + newBranches, err := store.LoadPlainFile(plaintext) + if err != nil { + return nil, err + } + tree.Branches = newBranches + + // Re-encrypt the file + key, err := tree.Metadata.GetDataKey() + if err != nil { + return nil, err + } + err = common.EncryptTree(common.EncryptTreeOpts{ + DataKey: key, Tree: &tree, Cipher: aes.NewCipher(), + }) + if err != nil { + return nil, err + } + + return store.EmitEncryptedFile(tree) +} + +// Decrypt data using sops +func Decrypt(_ context.Context, encrypted []byte) ([]byte, error) { + if err := setupEnv(); err != nil { + return nil, err + } + + return sopsdecrypt.DataWithFormat(encrypted, formats.Yaml) +} diff --git a/dagger/keychain/keys.go b/dagger/keychain/keys.go new file mode 100644 index 00000000..06ab78b3 --- /dev/null +++ b/dagger/keychain/keys.go @@ -0,0 +1,96 @@ +package keychain + +import ( + "context" + "errors" + "fmt" + "os" + "os/user" + "path" + "path/filepath" + "time" + + "filippo.io/age" + "github.com/rs/zerolog/log" +) + +func Path() (string, error) { + usr, err := user.Current() + if err != nil { + return "", err + } + + return path.Join(usr.HomeDir, ".dagger", "keys.txt"), nil +} + +func Default(ctx context.Context) (string, error) { + keys, err := List(ctx) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Generate(ctx) + } + return "", err + } + if len(keys) == 0 { + return "", errors.New("no identities found in the keys file") + } + + return keys[0].Recipient().String(), nil +} + +func Generate(ctx context.Context) (string, error) { + keysFile, err := Path() + if err != nil { + return "", err + } + + k, err := age.GenerateX25519Identity() + if err != nil { + return "", fmt.Errorf("internal error: %v", err) + } + + if err := os.MkdirAll(filepath.Dir(keysFile), 0755); err != nil { + return "", err + } + f, err := os.OpenFile(keysFile, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return "", fmt.Errorf("failed to open keys file %q: %v", keysFile, err) + } + defer f.Close() + fmt.Fprintf(f, "# created: %s\n", time.Now().Format(time.RFC3339)) + fmt.Fprintf(f, "# public key: %s\n", k.Recipient()) + fmt.Fprintf(f, "%s\n", k) + + pubkey := k.Recipient().String() + + log.Ctx(ctx).Debug().Str("publicKey", pubkey).Msg("generating keypair") + + return pubkey, nil +} + +func List(ctx context.Context) ([]*age.X25519Identity, error) { + keysFile, err := Path() + if err != nil { + return nil, err + } + + f, err := os.Open(keysFile) + if err != nil { + return nil, fmt.Errorf("failed to open keys file file %q: %w", keysFile, err) + } + ids, err := age.ParseIdentities(f) + if err != nil { + return nil, fmt.Errorf("failed to parse input: %w", err) + } + + keys := make([]*age.X25519Identity, 0, len(ids)) + for _, id := range ids { + key, ok := ids[0].(*age.X25519Identity) + if !ok { + return nil, fmt.Errorf("internal error: unexpected identity type: %T", id) + } + keys = append(keys, key) + } + + return keys, nil +} diff --git a/dagger/state/store.go b/dagger/state/store.go index 966d4494..d219942c 100644 --- a/dagger/state/store.go +++ b/dagger/state/store.go @@ -3,11 +3,13 @@ package state import ( "context" "errors" + "fmt" "os" "path" "path/filepath" "strings" + "dagger.io/go/dagger/keychain" "gopkg.in/yaml.v3" ) @@ -26,13 +28,33 @@ const ( func Init(ctx context.Context, dir, name string) (*State, error) { root := path.Join(dir, daggerDir) - err := os.Mkdir(root, 0755) - if err != nil { + if err := os.Mkdir(root, 0755); err != nil { if errors.Is(err, os.ErrExist) { return nil, ErrAlreadyInit } return nil, err } + manifestPath := path.Join(dir, daggerDir, manifestFile) + + st := &State{ + Path: dir, + Name: name, + } + data, err := yaml.Marshal(st) + if err != nil { + return nil, err + } + key, err := keychain.Default(ctx) + if err != nil { + return nil, err + } + encrypted, err := keychain.Encrypt(ctx, manifestPath, data, key) + if err != nil { + return nil, err + } + if err := os.WriteFile(manifestPath, encrypted, 0600); err != nil { + return nil, err + } err = os.WriteFile( path.Join(root, ".gitignore"), @@ -43,12 +65,7 @@ func Init(ctx context.Context, dir, name string) (*State, error) { return nil, err } - st := &State{ - Path: dir, - Name: name, - } - - return st, Save(ctx, st) + return st, nil } func Current(ctx context.Context) (*State, error) { @@ -91,6 +108,11 @@ func Open(ctx context.Context, dir string) (*State, error) { if err != nil { return nil, err } + data, err = keychain.Decrypt(ctx, data) + if err != nil { + return nil, fmt.Errorf("unable to decrypt state: %w", err) + } + var st State if err := yaml.Unmarshal(data, &st); err != nil { return nil, err @@ -111,7 +133,13 @@ func Save(ctx context.Context, st *State) error { return err } - if err := os.WriteFile(path.Join(st.Path, daggerDir, manifestFile), data, 0600); err != nil { + manifestPath := path.Join(st.Path, daggerDir, manifestFile) + + encrypted, err := keychain.Reencrypt(ctx, manifestPath, data) + if err != nil { + return err + } + if err := os.WriteFile(manifestPath, encrypted, 0600); err != nil { return err } diff --git a/go.mod b/go.mod index 3029f551..ab46d73b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module dagger.io/go go 1.16 require ( - cuelang.org/go v0.4.0-rc.1 + cuelang.org/go v0.4.0-beta.1 + filippo.io/age v1.0.0-rc.1 github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db github.com/containerd/console v1.0.2 @@ -17,7 +18,7 @@ require ( github.com/morikuni/aec v1.0.0 github.com/opencontainers/go-digest v1.0.0 github.com/opentracing/opentracing-go v1.2.0 - github.com/rs/zerolog v1.22.0 + github.com/rs/zerolog v1.21.0 github.com/spf13/cobra v1.1.3 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 56679f4b..a85441b1 100644 --- a/go.sum +++ b/go.sum @@ -44,12 +44,13 @@ contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrL contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= -cuelang.org/go v0.4.0-rc.1 h1:X8fsqVhLCvXFhsWMGbI8rjTal45YOgt+ko+m7rOCySM= -cuelang.org/go v0.4.0-rc.1/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs= +cuelang.org/go v0.4.0-beta.1 h1:/YjeAmymfNdTLSA3jHXNrj8Q+5Zq9by7qNOssqUBM+c= +cuelang.org/go v0.4.0-beta.1/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -filippo.io/age v1.0.0-beta7 h1:RZiSK+N3KL2UwT82xiCavjYw8jJHzWMEUYePAukTpk0= filippo.io/age v1.0.0-beta7/go.mod h1:chAuTrTb0FTTmKtvs6fQTGhYTvH9AigjN1uEUsvLdZ0= +filippo.io/age v1.0.0-rc.1 h1:jQ+dz16Xxx3W/WY+YS0J96nVAAidLHO3kfQe0eOmKgI= +filippo.io/age v1.0.0-rc.1/go.mod h1:Vvd9IlwNo4Au31iqNZeZVnYtGcOf/wT4mtvZQ2ODlSk= filippo.io/edwards25519 v1.0.0-alpha.2/go.mod h1:X+pm78QAUPtFLi1z9PYIlS/bdDnvbCOGKtZ+ACWEf7o= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= @@ -879,8 +880,8 @@ github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.22.0 h1:XrVUjV4K+izZpKXZHlPrYQiDtmdGiCylnT4i43AAWxg= -github.com/rs/zerolog v1.22.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= +github.com/rs/zerolog v1.21.0 h1:Q3vdXlfLNT+OftyBHsU0Y445MD+8m8axjKgf2si0QcM= +github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -899,7 +900,6 @@ github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7A github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A= github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= @@ -1084,6 +1084,7 @@ golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= From 90abadf0deb23c1795631db4c54c4eed86112969 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 18 May 2021 19:15:17 -0700 Subject: [PATCH 06/16] gitflow: multi env support Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/common/common.go | 75 +++++++---- cmd/dagger/cmd/init.go | 23 +--- cmd/dagger/cmd/input/dir.go | 23 +++- cmd/dagger/cmd/input/list.go | 3 +- cmd/dagger/cmd/input/root.go | 5 +- cmd/dagger/cmd/input/unset.go | 6 +- cmd/dagger/cmd/list.go | 41 +----- cmd/dagger/cmd/new.go | 42 ++++++ cmd/dagger/cmd/query.go | 3 +- cmd/dagger/cmd/root.go | 1 + cmd/dagger/cmd/up.go | 6 +- dagger/state/input.go | 6 +- dagger/state/state.go | 7 +- dagger/state/store.go | 232 +++++++++++++++++--------------- 14 files changed, 262 insertions(+), 211 deletions(-) create mode 100644 cmd/dagger/cmd/new.go diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index b7c87cee..4c0aae01 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -9,13 +9,37 @@ import ( "github.com/spf13/viper" ) -func GetCurrentEnvironmentState(ctx context.Context) *state.State { +func CurrentWorkspace(ctx context.Context) *state.Workspace { lg := log.Ctx(ctx) - // If no environment name has been given, look for the current environment - environment := viper.GetString("environment") - if environment == "" { - st, err := state.Current(ctx) + if workspacePath := viper.GetString("workspace"); workspacePath != "" { + workspace, err := state.Open(ctx, workspacePath) + if err != nil { + lg. + Fatal(). + Err(err). + Str("path", workspacePath). + Msg("failed to open workspace") + } + return workspace + } + + workspace, err := state.Current(ctx) + if err != nil { + lg. + Fatal(). + Err(err). + Msg("failed to determine current workspace") + } + return workspace +} + +func CurrentEnvironmentState(ctx context.Context, workspace *state.Workspace) *state.State { + lg := log.Ctx(ctx) + + environmentName := viper.GetString("environment") + if environmentName != "" { + st, err := workspace.Get(ctx, environmentName) if err != nil { lg. Fatal(). @@ -25,38 +49,33 @@ func GetCurrentEnvironmentState(ctx context.Context) *state.State { return st } - // At this point, it must be an environment name - workspace := viper.GetString("workspace") - var err error - if workspace == "" { - workspace, err = state.CurrentWorkspace(ctx) - if err != nil { - lg. - Fatal(). - Err(err). - Msg("failed to determine current workspace") - } - } - - environments, err := state.List(ctx, workspace) + environments, err := workspace.List(ctx) if err != nil { lg. Fatal(). Err(err). Msg("failed to list environments") } - for _, e := range environments { - if e.Name == environment { - return e - } + + if len(environments) == 0 { + lg. + Fatal(). + Msg("no environments") } - lg. - Fatal(). - Str("environment", environment). - Msg("environment not found") + if len(environments) > 1 { + envNames := []string{} + for _, e := range environments { + envNames = append(envNames, e.Name) + } + lg. + Fatal(). + Err(err). + Strs("environments", envNames). + Msg("multiple environments available in the workspace, select one with `--environment`") + } - return nil + return environments[0] } // Re-compute an environment (equivalent to `dagger up`). diff --git a/cmd/dagger/cmd/init.go b/cmd/dagger/cmd/init.go index 1bc16633..c47346f7 100644 --- a/cmd/dagger/cmd/init.go +++ b/cmd/dagger/cmd/init.go @@ -2,7 +2,6 @@ package cmd import ( "os" - "path/filepath" "dagger.io/go/cmd/dagger/logger" "dagger.io/go/dagger/state" @@ -24,7 +23,7 @@ var initCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - dir := viper.GetString("environment") + dir := viper.GetString("workspace") if dir == "" { cwd, err := os.Getwd() if err != nil { @@ -36,29 +35,13 @@ var initCmd = &cobra.Command{ dir = cwd } - var name string - if len(args) > 0 { - name = args[0] - } else { - name = getNewEnvironmentName(dir) - } - - _, err := state.Init(ctx, dir, name) + _, err := state.Init(ctx, dir) if err != nil { - lg.Fatal().Err(err).Msg("failed to initialize") + lg.Fatal().Err(err).Msg("failed to initialize workspace") } }, } -func getNewEnvironmentName(dir string) string { - dirName := filepath.Base(dir) - if dirName == "/" { - return "root" - } - - return dirName -} - func init() { if err := viper.BindPFlags(initCmd.Flags()); err != nil { panic(err) diff --git a/cmd/dagger/cmd/input/dir.go b/cmd/dagger/cmd/input/dir.go index f18e44f1..0e23ccee 100644 --- a/cmd/dagger/cmd/input/dir.go +++ b/cmd/dagger/cmd/input/dir.go @@ -1,6 +1,10 @@ package input import ( + "path/filepath" + "strings" + + "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" "dagger.io/go/dagger/state" "github.com/spf13/cobra" @@ -22,7 +26,24 @@ var dirCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - updateEnvironmentInput(ctx, args[0], state.DirInput(args[1], []string{})) + p, err := filepath.Abs(args[1]) + if err != nil { + lg.Fatal().Err(err).Str("path", args[1]).Msg("unable to resolve path") + } + + workspace := common.CurrentWorkspace(ctx) + if !strings.HasPrefix(p, workspace.Path) { + lg.Fatal().Err(err).Str("path", args[1]).Msg("dir is outside the workspace") + } + p, err = filepath.Rel(workspace.Path, p) + if err != nil { + lg.Fatal().Err(err).Str("path", args[1]).Msg("unable to resolve path") + } + if !strings.HasPrefix(p, ".") { + p = "./" + p + } + + updateEnvironmentInput(ctx, args[0], state.DirInput(p, []string{})) }, } diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 22929be0..bc1faeb8 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -31,7 +31,8 @@ var listCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - environment := common.GetCurrentEnvironmentState(ctx) + workspace := common.CurrentWorkspace(ctx) + environment := common.CurrentEnvironmentState(ctx, workspace) lg = lg.With(). Str("environment", environment.Name). diff --git a/cmd/dagger/cmd/input/root.go b/cmd/dagger/cmd/input/root.go index f035fda8..e9cf2e75 100644 --- a/cmd/dagger/cmd/input/root.go +++ b/cmd/dagger/cmd/input/root.go @@ -35,10 +35,11 @@ func init() { func updateEnvironmentInput(ctx context.Context, target string, input state.Input) { lg := log.Ctx(ctx) - st := common.GetCurrentEnvironmentState(ctx) + workspace := common.CurrentWorkspace(ctx) + st := common.CurrentEnvironmentState(ctx, workspace) st.SetInput(target, input) - if err := state.Save(ctx, st); err != nil { + if err := workspace.Save(ctx, st); err != nil { lg.Fatal().Err(err).Str("environment", st.Name).Msg("cannot update environment") } } diff --git a/cmd/dagger/cmd/input/unset.go b/cmd/dagger/cmd/input/unset.go index 248aae7a..9b643c0b 100644 --- a/cmd/dagger/cmd/input/unset.go +++ b/cmd/dagger/cmd/input/unset.go @@ -3,7 +3,6 @@ package input import ( "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -23,10 +22,11 @@ var unsetCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - st := common.GetCurrentEnvironmentState(ctx) + workspace := common.CurrentWorkspace(ctx) + st := common.CurrentEnvironmentState(ctx, workspace) st.RemoveInputs(args[0]) - if err := state.Save(ctx, st); err != nil { + if err := workspace.Save(ctx, st); err != nil { lg.Fatal().Err(err).Str("environment", st.Name).Msg("cannot update environment") } lg.Info().Str("environment", st.Name).Msg("updated environment") diff --git a/cmd/dagger/cmd/list.go b/cmd/dagger/cmd/list.go index fd6b30a1..4e1fb8d0 100644 --- a/cmd/dagger/cmd/list.go +++ b/cmd/dagger/cmd/list.go @@ -1,8 +1,6 @@ package cmd import ( - "context" - "errors" "fmt" "os" "os/user" @@ -10,9 +8,8 @@ import ( "strings" "text/tabwriter" + "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger/state" - "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -32,21 +29,8 @@ var listCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - var ( - workspace = viper.GetString("workspace") - err error - ) - if workspace == "" { - workspace, err = state.CurrentWorkspace(ctx) - if err != nil { - lg. - Fatal(). - Err(err). - Msg("failed to determine current workspace") - } - } - - environments, err := state.List(ctx, workspace) + workspace := common.CurrentWorkspace(ctx) + environments, err := workspace.List(ctx) if err != nil { lg. Fatal(). @@ -54,34 +38,15 @@ var listCmd = &cobra.Command{ Msg("cannot list environments") } - environmentPath := getCurrentEnvironmentPath(ctx) w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.TabIndent) defer w.Flush() for _, e := range environments { line := fmt.Sprintf("%s\t%s\t", e.Name, formatPath(e.Path)) - if e.Path == environmentPath { - line = fmt.Sprintf("%s- active environment", line) - } fmt.Fprintln(w, line) } }, } -func getCurrentEnvironmentPath(ctx context.Context) string { - lg := log.Ctx(ctx) - - st, err := state.Current(ctx) - if err != nil { - // Ignore error if not initialized - if errors.Is(err, state.ErrNotInit) { - return "" - } - lg.Fatal().Err(err).Msg("failed to load current environment") - } - - return st.Path -} - func formatPath(p string) string { usr, err := user.Current() if err != nil { diff --git a/cmd/dagger/cmd/new.go b/cmd/dagger/cmd/new.go new file mode 100644 index 00000000..c2c019c0 --- /dev/null +++ b/cmd/dagger/cmd/new.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "dagger.io/go/cmd/dagger/cmd/common" + "dagger.io/go/cmd/dagger/logger" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var newCmd = &cobra.Command{ + Use: "new", + Args: cobra.ExactArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + // Fix Viper bug for duplicate flags: + // https://github.com/spf13/viper/issues/233 + if err := viper.BindPFlags(cmd.Flags()); err != nil { + panic(err) + } + }, + Run: func(cmd *cobra.Command, args []string) { + lg := logger.New() + ctx := lg.WithContext(cmd.Context()) + + workspace := common.CurrentWorkspace(ctx) + + if viper.GetString("environment") != "" { + lg. + Fatal(). + Msg("cannot use option -e,--environment for this command") + } + name := args[0] + if _, err := workspace.Create(ctx, name); err != nil { + lg.Fatal().Err(err).Msg("failed to create environment") + } + }, +} + +func init() { + if err := viper.BindPFlags(newCmd.Flags()); err != nil { + panic(err) + } +} diff --git a/cmd/dagger/cmd/query.go b/cmd/dagger/cmd/query.go index 62981608..2c689d5e 100644 --- a/cmd/dagger/cmd/query.go +++ b/cmd/dagger/cmd/query.go @@ -30,7 +30,8 @@ var queryCmd = &cobra.Command{ cueOpts := parseQueryFlags() - state := common.GetCurrentEnvironmentState(ctx) + workspace := common.CurrentWorkspace(ctx) + state := common.CurrentEnvironmentState(ctx, workspace) lg = lg.With(). Str("environment", state.Name). diff --git a/cmd/dagger/cmd/root.go b/cmd/dagger/cmd/root.go index d3cc7cd8..77006d9c 100644 --- a/cmd/dagger/cmd/root.go +++ b/cmd/dagger/cmd/root.go @@ -34,6 +34,7 @@ func init() { rootCmd.AddCommand( initCmd, + newCmd, computeCmd, listCmd, queryCmd, diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 808313e6..440216c5 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -3,7 +3,6 @@ package cmd import ( "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" - "dagger.io/go/dagger/state" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -24,11 +23,12 @@ var upCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - st := common.GetCurrentEnvironmentState(ctx) + workspace := common.CurrentWorkspace(ctx) + st := common.CurrentEnvironmentState(ctx, workspace) result := common.EnvironmentUp(ctx, st, viper.GetBool("no-cache")) st.Computed = result.Computed().JSON().PrettyString() - if err := state.Save(ctx, st); err != nil { + if err := workspace.Save(ctx, st); err != nil { lg.Fatal().Err(err).Msg("failed to update environment") } }, diff --git a/dagger/state/input.go b/dagger/state/input.go index 9c34bdd6..4c06e47e 100644 --- a/dagger/state/input.go +++ b/dagger/state/input.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "path" "path/filepath" + "strings" "cuelang.org/go/cue" @@ -90,7 +91,10 @@ func (dir dirInput) Compile(state *State) (*compiler.Value, error) { p := dir.Path if !filepath.IsAbs(p) { - p = filepath.Clean(path.Join(state.Path, p)) + p = filepath.Clean(path.Join(state.Workspace, dir.Path)) + } + if !strings.HasPrefix(p, state.Workspace) { + return nil, fmt.Errorf("%q is outside the workspace", dir.Path) } llb := fmt.Sprintf( diff --git a/dagger/state/state.go b/dagger/state/state.go index 49480ebd..becf5d1a 100644 --- a/dagger/state/state.go +++ b/dagger/state/state.go @@ -1,10 +1,15 @@ package state +import "path" + // Contents of an environment serialized to a file type State struct { // State path Path string `yaml:"-"` + // Workspace path + Workspace string `yaml:"-"` + // Human-friendly environment name. // A environment may have more than one name. // FIXME: store multiple names? @@ -20,7 +25,7 @@ type State struct { // Cue module containing the environment plan // The input's top-level artifact is used as a module directory. func (s *State) PlanSource() Input { - return DirInput(s.Path, []string{"*.cue", "cue.mod"}) + return DirInput(path.Join(s.Path, planDir), []string{"*.cue", "cue.mod"}) } func (s *State) SetInput(key string, value Input) error { diff --git a/dagger/state/store.go b/dagger/state/store.go index d219942c..b6699e5d 100644 --- a/dagger/state/store.go +++ b/dagger/state/store.go @@ -7,26 +7,32 @@ import ( "os" "path" "path/filepath" - "strings" "dagger.io/go/dagger/keychain" "gopkg.in/yaml.v3" ) var ( - ErrNotInit = errors.New("not initialized") - ErrAlreadyInit = errors.New("already initialized") - ErrNoCurrentWorkspace = errors.New("not in a git directory") + ErrNotInit = errors.New("not initialized") + ErrAlreadyInit = errors.New("already initialized") + ErrNotExist = errors.New("environment doesn't exist") + ErrExist = errors.New("environment already exists") ) const ( daggerDir = ".dagger" + envDir = "env" stateDir = "state" + planDir = "plan" manifestFile = "values.yaml" computedFile = "computed.json" ) -func Init(ctx context.Context, dir, name string) (*State, error) { +type Workspace struct { + Path string +} + +func Init(ctx context.Context, dir string) (*Workspace, error) { root := path.Join(dir, daggerDir) if err := os.Mkdir(root, 0755); err != nil { if errors.Is(err, os.ErrExist) { @@ -34,41 +40,31 @@ func Init(ctx context.Context, dir, name string) (*State, error) { } return nil, err } - manifestPath := path.Join(dir, daggerDir, manifestFile) - - st := &State{ - Path: dir, - Name: name, - } - data, err := yaml.Marshal(st) - if err != nil { - return nil, err - } - key, err := keychain.Default(ctx) - if err != nil { - return nil, err - } - encrypted, err := keychain.Encrypt(ctx, manifestPath, data, key) - if err != nil { - return nil, err - } - if err := os.WriteFile(manifestPath, encrypted, 0600); err != nil { - return nil, err - } - - err = os.WriteFile( - path.Join(root, ".gitignore"), - []byte("# dagger state\nstate/**\n"), - 0600, - ) - if err != nil { - return nil, err - } - - return st, nil + return &Workspace{ + Path: root, + }, nil } -func Current(ctx context.Context) (*State, error) { +func Open(ctx context.Context, dir string) (*Workspace, error) { + _, err := os.Stat(path.Join(dir, daggerDir)) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, ErrNotInit + } + return nil, err + } + + root, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + return &Workspace{ + Path: root, + }, nil +} + +func Current(ctx context.Context) (*Workspace, error) { current, err := os.Getwd() if err != nil { return nil, err @@ -90,36 +86,63 @@ func Current(ctx context.Context) (*State, error) { return nil, ErrNotInit } -func Open(ctx context.Context, dir string) (*State, error) { - _, err := os.Stat(path.Join(dir, daggerDir)) +func (w *Workspace) envPath(name string) string { + return path.Join(w.Path, daggerDir, envDir, name) +} + +func (w *Workspace) List(ctx context.Context) ([]*State, error) { + var ( + environments = []*State{} + err error + ) + + files, err := os.ReadDir(path.Join(w.Path, daggerDir, envDir)) if err != nil { + return nil, err + } + for _, f := range files { + if !f.IsDir() { + continue + } + st, err := w.Get(ctx, f.Name()) + if err != nil { + return nil, err + } + environments = append(environments, st) + } + + return environments, nil +} + +func (w *Workspace) Get(ctx context.Context, name string) (*State, error) { + envPath, err := filepath.Abs(w.envPath(name)) + if err != nil { + return nil, err + } + if _, err := os.Stat(envPath); err != nil { if errors.Is(err, os.ErrNotExist) { - return nil, ErrNotInit + return nil, ErrNotExist } return nil, err } - root, err := filepath.Abs(dir) + manifest, err := os.ReadFile(path.Join(envPath, manifestFile)) if err != nil { return nil, err } - - data, err := os.ReadFile(path.Join(root, daggerDir, manifestFile)) - if err != nil { - return nil, err - } - data, err = keychain.Decrypt(ctx, data) + manifest, err = keychain.Decrypt(ctx, manifest) if err != nil { return nil, fmt.Errorf("unable to decrypt state: %w", err) } var st State - if err := yaml.Unmarshal(data, &st); err != nil { + if err := yaml.Unmarshal(manifest, &st); err != nil { return nil, err } - st.Path = root + st.Path = envPath + st.Workspace = w.Path - computed, err := os.ReadFile(path.Join(root, daggerDir, stateDir, computedFile)) + computed, err := os.ReadFile(path.Join(envPath, stateDir, computedFile)) if err == nil { st.Computed = string(computed) } @@ -127,13 +150,13 @@ func Open(ctx context.Context, dir string) (*State, error) { return &st, nil } -func Save(ctx context.Context, st *State) error { +func (w *Workspace) Save(ctx context.Context, st *State) error { data, err := yaml.Marshal(st) if err != nil { return err } - manifestPath := path.Join(st.Path, daggerDir, manifestFile) + manifestPath := path.Join(st.Path, manifestFile) encrypted, err := keychain.Reencrypt(ctx, manifestPath, data) if err != nil { @@ -144,7 +167,7 @@ func Save(ctx context.Context, st *State) error { } if st.Computed != "" { - state := path.Join(st.Path, daggerDir, stateDir) + state := path.Join(st.Path, stateDir) if err := os.MkdirAll(state, 0755); err != nil { return err } @@ -160,74 +183,59 @@ func Save(ctx context.Context, st *State) error { return nil } -func CurrentWorkspace(ctx context.Context) (string, error) { - current, err := os.Getwd() +func (w *Workspace) Create(ctx context.Context, name string) (*State, error) { + envPath, err := filepath.Abs(w.envPath(name)) if err != nil { - return "", err + return nil, err } - // Walk every parent directory to find .dagger - for { - _, err := os.Stat(path.Join(current, ".git")) - if err == nil { - return current, nil + // Environment directory + if err := os.MkdirAll(envPath, 0755); err != nil { + if errors.Is(err, os.ErrExist) { + return nil, ErrExist } - parent := filepath.Dir(current) - if parent == current { - break - } - current = parent + return nil, err } - return "", ErrNoCurrentWorkspace -} + // Plan directory + if err := os.Mkdir(path.Join(envPath, planDir), 0755); err != nil { + if errors.Is(err, os.ErrExist) { + return nil, ErrExist + } + return nil, err + } -func List(ctx context.Context, workspace string) ([]*State, error) { - var ( - environments = []*State{} - err error + manifestPath := path.Join(envPath, manifestFile) + + st := &State{ + Path: envPath, + Workspace: w.Path, + Name: name, + } + data, err := yaml.Marshal(st) + if err != nil { + return nil, err + } + key, err := keychain.Default(ctx) + if err != nil { + return nil, err + } + encrypted, err := keychain.Encrypt(ctx, manifestPath, data, key) + if err != nil { + return nil, err + } + if err := os.WriteFile(manifestPath, encrypted, 0600); err != nil { + return nil, err + } + + err = os.WriteFile( + path.Join(envPath, ".gitignore"), + []byte("# dagger state\nstate/**\n"), + 0600, ) - - workspace, err = filepath.Abs(workspace) if err != nil { return nil, err } - err = filepath.WalkDir(workspace, func(p string, info os.DirEntry, err error) error { - // Ignore errors while we walk - if err != nil { - return nil - } - - // Skip regular files - if !info.IsDir() { - return nil - } - - // Skip non-dagger directories - if info.Name() != daggerDir { - // Caveat: limit traversal to a depth of 10 (arbitrary) - relPath := strings.TrimPrefix(p, workspace) - if strings.Count(relPath, string(os.PathSeparator)) > 10 { - return filepath.SkipDir - } - - // Otherwise, continue traversing - return nil - } - - st, err := Open(ctx, filepath.Dir(p)) - if err != nil { - return err - } - environments = append(environments, st) - - return nil - }) - - if err != nil { - return nil, err - } - - return environments, nil + return st, nil } From f374f4c5ea2dde2f62a6343a943c3ca7e6e42532 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 21 May 2021 16:16:56 -0700 Subject: [PATCH 07/16] encryption: re-use same IVs so that the ciphertext doesn't change when decrypting and reencrypting Signed-off-by: Andrea Luzzardi --- dagger/keychain/encrypt.go | 48 +++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/dagger/keychain/encrypt.go b/dagger/keychain/encrypt.go index bd78ea01..35e41172 100644 --- a/dagger/keychain/encrypt.go +++ b/dagger/keychain/encrypt.go @@ -4,18 +4,21 @@ import ( "context" "fmt" "os" + "time" "go.mozilla.org/sops/v3" - "go.mozilla.org/sops/v3/aes" + sopsaes "go.mozilla.org/sops/v3/aes" sopsage "go.mozilla.org/sops/v3/age" "go.mozilla.org/sops/v3/cmd/sops/common" - "go.mozilla.org/sops/v3/cmd/sops/formats" - sopsdecrypt "go.mozilla.org/sops/v3/decrypt" sopskeys "go.mozilla.org/sops/v3/keys" sopsyaml "go.mozilla.org/sops/v3/stores/yaml" "go.mozilla.org/sops/v3/version" ) +var ( + cipher = sopsaes.NewCipher() +) + // setupEnv: hack to inject a SOPS env var for age func setupEnv() error { p, err := Path() @@ -65,7 +68,7 @@ func Encrypt(ctx context.Context, path string, plaintext []byte, key string) ([] } err = common.EncryptTree(common.EncryptTreeOpts{ - DataKey: dataKey, Tree: &tree, Cipher: aes.NewCipher(), + DataKey: dataKey, Tree: &tree, Cipher: cipher, }) if err != nil { return nil, err @@ -104,7 +107,7 @@ func Reencrypt(_ context.Context, path string, plaintext []byte) ([]byte, error) return nil, err } err = common.EncryptTree(common.EncryptTreeOpts{ - DataKey: key, Tree: &tree, Cipher: aes.NewCipher(), + DataKey: key, Tree: &tree, Cipher: cipher, }) if err != nil { return nil, err @@ -119,5 +122,38 @@ func Decrypt(_ context.Context, encrypted []byte) ([]byte, error) { return nil, err } - return sopsdecrypt.DataWithFormat(encrypted, formats.Yaml) + store := &sopsyaml.Store{} + + // Load SOPS file and access the data key + tree, err := store.LoadEncryptedFile(encrypted) + if err != nil { + return nil, err + } + key, err := tree.Metadata.GetDataKey() + if err != nil { + return nil, err + } + + // Decrypt the tree + mac, err := tree.Decrypt(key, cipher) + if err != nil { + return nil, err + } + + // Compute the hash of the cleartext tree and compare it with + // the one that was stored in the document. If they match, + // integrity was preserved + originalMac, err := cipher.Decrypt( + tree.Metadata.MessageAuthenticationCode, + key, + tree.Metadata.LastModified.Format(time.RFC3339), + ) + if err != nil { + return nil, err + } + if originalMac != mac { + return nil, fmt.Errorf("failed to verify data integrity. expected mac %q, got %q", originalMac, mac) + } + + return store.EmitPlainFile(tree.Branches) } From 0acd3a256aa81a0c65cba08e67f711e5d71d7e3f Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 21 May 2021 16:18:47 -0700 Subject: [PATCH 08/16] only consider workspaces containing an env directory Signed-off-by: Andrea Luzzardi --- dagger/state/store.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dagger/state/store.go b/dagger/state/store.go index b6699e5d..2308873d 100644 --- a/dagger/state/store.go +++ b/dagger/state/store.go @@ -40,6 +40,9 @@ func Init(ctx context.Context, dir string) (*Workspace, error) { } return nil, err } + if err := os.Mkdir(path.Join(root, envDir), 0755); err != nil { + return nil, err + } return &Workspace{ Path: root, }, nil @@ -72,7 +75,7 @@ func Current(ctx context.Context) (*Workspace, error) { // Walk every parent directory to find .dagger for { - _, err := os.Stat(path.Join(current, daggerDir)) + _, err := os.Stat(path.Join(current, daggerDir, envDir)) if err == nil { return Open(ctx, current) } From 155c90e3e072d1333ab5ca561e60e929614cb3a8 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 21 May 2021 16:19:07 -0700 Subject: [PATCH 09/16] dagger list: skip environment if it can't be decryted Signed-off-by: Andrea Luzzardi --- dagger/state/store.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dagger/state/store.go b/dagger/state/store.go index 2308873d..7e7f1c61 100644 --- a/dagger/state/store.go +++ b/dagger/state/store.go @@ -9,6 +9,7 @@ import ( "path/filepath" "dagger.io/go/dagger/keychain" + "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" ) @@ -109,7 +110,12 @@ func (w *Workspace) List(ctx context.Context) ([]*State, error) { } st, err := w.Get(ctx, f.Name()) if err != nil { - return nil, err + log. + Ctx(ctx). + Err(err). + Str("name", f.Name()). + Msg("failed to load environment") + continue } environments = append(environments, st) } From 322ca54e585dccdfb039a406a2714c9759975100 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 21 May 2021 16:27:19 -0700 Subject: [PATCH 10/16] state: do not overwrite values.yaml if nothing has changed Signed-off-by: Andrea Luzzardi --- dagger/state/store.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/dagger/state/store.go b/dagger/state/store.go index 7e7f1c61..86563483 100644 --- a/dagger/state/store.go +++ b/dagger/state/store.go @@ -1,6 +1,7 @@ package state import ( + "bytes" "context" "errors" "fmt" @@ -167,12 +168,24 @@ func (w *Workspace) Save(ctx context.Context, st *State) error { manifestPath := path.Join(st.Path, manifestFile) - encrypted, err := keychain.Reencrypt(ctx, manifestPath, data) + currentEncrypted, err := os.ReadFile(manifestPath) if err != nil { return err } - if err := os.WriteFile(manifestPath, encrypted, 0600); err != nil { - return err + currentPlain, err := keychain.Decrypt(ctx, currentEncrypted) + if err != nil { + return fmt.Errorf("unable to decrypt state: %w", err) + } + + // Only update the encrypted file if there were changes + if bytes.Compare(data, currentPlain) != 0 { + encrypted, err := keychain.Reencrypt(ctx, manifestPath, data) + if err != nil { + return err + } + if err := os.WriteFile(manifestPath, encrypted, 0600); err != nil { + return err + } } if st.Computed != "" { From bfe9097f6653a186b518d90cc4191fbed08c18b3 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 25 May 2021 12:50:12 -0700 Subject: [PATCH 11/16] improve cue error displaying for plans Signed-off-by: Andrea Luzzardi --- dagger/environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagger/environment.go b/dagger/environment.go index cddf579f..b12673bf 100644 --- a/dagger/environment.go +++ b/dagger/environment.go @@ -103,7 +103,7 @@ func (e *Environment) LoadPlan(ctx context.Context, s Solver) error { } plan, err := compiler.Build(sources) if err != nil { - return fmt.Errorf("plan config: %w", err) + return fmt.Errorf("plan config: %w", compiler.Err(err)) } e.plan = plan From 1e37a0862cd5206b917cefc831c972d78b4cb6dc Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 25 May 2021 13:03:44 -0700 Subject: [PATCH 12/16] fix workspace tests Signed-off-by: Andrea Luzzardi --- dagger/state/store_test.go | 39 ------------------ dagger/state/{store.go => workspace.go} | 13 ++++-- dagger/state/workspace_test.go | 55 +++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 43 deletions(-) delete mode 100644 dagger/state/store_test.go rename dagger/state/{store.go => workspace.go} (94%) create mode 100644 dagger/state/workspace_test.go diff --git a/dagger/state/store_test.go b/dagger/state/store_test.go deleted file mode 100644 index d5ec0afd..00000000 --- a/dagger/state/store_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package state - -import ( - "context" - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestStore(t *testing.T) { - ctx := context.TODO() - - // Init - root, err := os.MkdirTemp(os.TempDir(), "dagger-*") - require.NoError(t, err) - st, err := Init(ctx, root, "test") - require.Equal(t, "test", st.Name) - require.Equal(t, root, st.Path) - require.NoError(t, err) - - // Open - _, err = Open(ctx, "/tmp/not/exist") - require.Error(t, err) - require.ErrorIs(t, ErrNotInit, err) - - st, err = Open(ctx, root) - require.NoError(t, err) - require.Equal(t, "test", st.Name) - require.Equal(t, root, st.Path) - - // Save - computed := `{"hello": "world"}` - st.Computed = computed - require.NoError(t, Save(ctx, st)) - st, err = Open(ctx, root) - require.NoError(t, err) - require.Equal(t, computed, st.Computed) -} diff --git a/dagger/state/store.go b/dagger/state/workspace.go similarity index 94% rename from dagger/state/store.go rename to dagger/state/workspace.go index 86563483..79c05dbd 100644 --- a/dagger/state/store.go +++ b/dagger/state/workspace.go @@ -35,14 +35,19 @@ type Workspace struct { } func Init(ctx context.Context, dir string) (*Workspace, error) { - root := path.Join(dir, daggerDir) - if err := os.Mkdir(root, 0755); err != nil { + root, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + daggerRoot := path.Join(root, daggerDir) + if err := os.Mkdir(daggerRoot, 0755); err != nil { if errors.Is(err, os.ErrExist) { return nil, ErrAlreadyInit } return nil, err } - if err := os.Mkdir(path.Join(root, envDir), 0755); err != nil { + if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil { return nil, err } return &Workspace{ @@ -178,7 +183,7 @@ func (w *Workspace) Save(ctx context.Context, st *State) error { } // Only update the encrypted file if there were changes - if bytes.Compare(data, currentPlain) != 0 { + if !bytes.Equal(data, currentPlain) { encrypted, err := keychain.Reencrypt(ctx, manifestPath, data) if err != nil { return err diff --git a/dagger/state/workspace_test.go b/dagger/state/workspace_test.go new file mode 100644 index 00000000..a474f36a --- /dev/null +++ b/dagger/state/workspace_test.go @@ -0,0 +1,55 @@ +package state + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWorkspace(t *testing.T) { + ctx := context.TODO() + + root, err := os.MkdirTemp(os.TempDir(), "dagger-*") + require.NoError(t, err) + + // Open should fail since the directory is not initialized + _, err = Open(ctx, root) + require.ErrorIs(t, ErrNotInit, err) + + // Init + workspace, err := Init(ctx, root) + require.NoError(t, err) + require.Equal(t, root, workspace.Path) + + // Create + st, err := workspace.Create(ctx, "test") + require.NoError(t, err) + require.Equal(t, "test", st.Name) + + // Open + workspace, err = Open(ctx, root) + require.NoError(t, err) + require.Equal(t, root, workspace.Path) + + // List + envs, err := workspace.List(ctx) + require.NoError(t, err) + require.Len(t, envs, 1) + require.Equal(t, "test", envs[0].Name) + + // Get + env, err := workspace.Get(ctx, "test") + require.NoError(t, err) + require.Equal(t, "test", env.Name) + + // Save + require.NoError(t, env.SetInput("foo", TextInput("bar"))) + require.NoError(t, workspace.Save(ctx, env)) + workspace, err = Open(ctx, root) + require.NoError(t, err) + env, err = workspace.Get(ctx, "test") + require.NoError(t, err) + require.Contains(t, env.Inputs, "foo") +} From 0ff12432aeaa5dbddd0770472a0d2a1a5d7318b9 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 25 May 2021 13:27:51 -0700 Subject: [PATCH 13/16] add encryption tests Signed-off-by: Andrea Luzzardi --- dagger/environment_test.go | 2 +- dagger/state/workspace_test.go | 54 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/dagger/environment_test.go b/dagger/environment_test.go index cb200b8d..06fccdd1 100644 --- a/dagger/environment_test.go +++ b/dagger/environment_test.go @@ -19,5 +19,5 @@ func TestLocalDirs(t *testing.T) { localdirs := environment.LocalDirs() require.Len(t, localdirs, 2) require.Contains(t, localdirs, "/") - require.Contains(t, localdirs, "/tmp/source") + require.Contains(t, localdirs, "/tmp/source/plan") } diff --git a/dagger/state/workspace_test.go b/dagger/state/workspace_test.go index a474f36a..9b7b6afd 100644 --- a/dagger/state/workspace_test.go +++ b/dagger/state/workspace_test.go @@ -3,9 +3,12 @@ package state import ( "context" "os" + "path" + "strings" "testing" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) func TestWorkspace(t *testing.T) { @@ -53,3 +56,54 @@ func TestWorkspace(t *testing.T) { require.NoError(t, err) require.Contains(t, env.Inputs, "foo") } + +func TestEncryption(t *testing.T) { + ctx := context.TODO() + + readManifest := func(st *State) *State { + data, err := os.ReadFile(path.Join(st.Path, manifestFile)) + require.NoError(t, err) + m := State{} + require.NoError(t, yaml.Unmarshal(data, &m)) + return &m + } + + root, err := os.MkdirTemp(os.TempDir(), "dagger-*") + require.NoError(t, err) + workspace, err := Init(ctx, root) + require.NoError(t, err) + + _, err = workspace.Create(ctx, "test") + require.NoError(t, err) + + // Set a plaintext input, make sure it is not encrypted + st, err := workspace.Get(ctx, "test") + require.NoError(t, err) + require.NoError(t, st.SetInput("plain", TextInput("plain"))) + require.NoError(t, workspace.Save(ctx, st)) + o := readManifest(st) + require.Contains(t, o.Inputs, "plain") + require.Equal(t, "plain", string(*o.Inputs["plain"].Text)) + + // Set a secret input, make sure it's encrypted + st, err = workspace.Get(ctx, "test") + require.NoError(t, err) + require.NoError(t, st.SetInput("secret", SecretInput("secret"))) + require.NoError(t, workspace.Save(ctx, st)) + o = readManifest(st) + require.Contains(t, o.Inputs, "secret") + secretValue := string(*o.Inputs["secret"].Secret) + require.NotEqual(t, "secret", secretValue) + require.True(t, strings.HasPrefix(secretValue, "ENC[")) + + // Change another input, make sure our secret didn't change + st, err = workspace.Get(ctx, "test") + require.NoError(t, err) + require.NoError(t, st.SetInput("plain", TextInput("different"))) + require.NoError(t, workspace.Save(ctx, st)) + o = readManifest(st) + require.Contains(t, o.Inputs, "plain") + require.Equal(t, "different", string(*o.Inputs["plain"].Text)) + require.Contains(t, o.Inputs, "secret") + require.Equal(t, secretValue, string(*o.Inputs["secret"].Secret)) +} From 12b771a43b0d404864bff4b5c3c191255d31b501 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 25 May 2021 13:37:11 -0700 Subject: [PATCH 14/16] update zerolog Signed-off-by: Andrea Luzzardi --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ab46d73b..5e742046 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/morikuni/aec v1.0.0 github.com/opencontainers/go-digest v1.0.0 github.com/opentracing/opentracing-go v1.2.0 - github.com/rs/zerolog v1.21.0 + github.com/rs/zerolog v1.22.0 github.com/spf13/cobra v1.1.3 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index a85441b1..4ba6788e 100644 --- a/go.sum +++ b/go.sum @@ -880,8 +880,8 @@ github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.21.0 h1:Q3vdXlfLNT+OftyBHsU0Y445MD+8m8axjKgf2si0QcM= -github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= +github.com/rs/zerolog v1.22.0 h1:XrVUjV4K+izZpKXZHlPrYQiDtmdGiCylnT4i43AAWxg= +github.com/rs/zerolog v1.22.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= From c7c8c5fa8d6396ef1c788658b2b447602d96ee5b Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 25 May 2021 15:14:39 -0700 Subject: [PATCH 15/16] tests: fix integration tests Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/compute.go | 1 + dagger/environment_test.go | 1 + dagger/state/state.go | 7 ++- dagger/state/workspace.go | 2 + tests/cli.bats | 113 +++++++++++++++---------------------- tests/examples.bats | 4 +- tests/helpers.bash | 14 ++++- tests/stdlib.bats | 7 ++- 8 files changed, 74 insertions(+), 75 deletions(-) diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index c49e235e..2fb4b8b4 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -38,6 +38,7 @@ var computeCmd = &cobra.Command{ st := &state.State{ Name: "FIXME", Path: args[0], + Plan: args[0], } for _, input := range viper.GetStringSlice("input-string") { diff --git a/dagger/environment_test.go b/dagger/environment_test.go index 06fccdd1..3da6ba6c 100644 --- a/dagger/environment_test.go +++ b/dagger/environment_test.go @@ -10,6 +10,7 @@ import ( func TestLocalDirs(t *testing.T) { st := &state.State{ Path: "/tmp/source", + Plan: "/tmp/source/plan", } require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{}))) diff --git a/dagger/state/state.go b/dagger/state/state.go index becf5d1a..84bed925 100644 --- a/dagger/state/state.go +++ b/dagger/state/state.go @@ -1,7 +1,5 @@ package state -import "path" - // Contents of an environment serialized to a file type State struct { // State path @@ -10,6 +8,9 @@ type State struct { // Workspace path Workspace string `yaml:"-"` + // Plan path + Plan string `yaml:"-"` + // Human-friendly environment name. // A environment may have more than one name. // FIXME: store multiple names? @@ -25,7 +26,7 @@ type State struct { // Cue module containing the environment plan // The input's top-level artifact is used as a module directory. func (s *State) PlanSource() Input { - return DirInput(path.Join(s.Path, planDir), []string{"*.cue", "cue.mod"}) + return DirInput(s.Plan, []string{"*.cue", "cue.mod"}) } func (s *State) SetInput(key string, value Input) error { diff --git a/dagger/state/workspace.go b/dagger/state/workspace.go index 79c05dbd..4a516401 100644 --- a/dagger/state/workspace.go +++ b/dagger/state/workspace.go @@ -155,6 +155,7 @@ func (w *Workspace) Get(ctx context.Context, name string) (*State, error) { return nil, err } st.Path = envPath + st.Plan = path.Join(envPath, planDir) st.Workspace = w.Path computed, err := os.ReadFile(path.Join(envPath, stateDir, computedFile)) @@ -237,6 +238,7 @@ func (w *Workspace) Create(ctx context.Context, name string) (*State, error) { st := &State{ Path: envPath, Workspace: w.Path, + Plan: path.Join(envPath, planDir), Name: name, } data, err := yaml.Marshal(st) diff --git a/tests/cli.bats b/tests/cli.bats index ca9513ba..df4d8de8 100644 --- a/tests/cli.bats +++ b/tests/cli.bats @@ -4,55 +4,45 @@ setup() { common_setup } -@test "dagger list" { - run "$DAGGER" list +@test "dagger init" { + run "$DAGGER" init assert_success - assert_output "" - - "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple run "$DAGGER" list assert_success - assert_output --partial "simple" + refute_output + + run "$DAGGER" init + assert_failure } -@test "dagger new --plan-dir" { - run "$DAGGER" list - assert_success - assert_output "" - - "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple - - # duplicate name - run "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple +@test "dagger new" { + run "$DAGGER" new "test" assert_failure - # verify the plan works - "$DAGGER" up -e "simple" - - # verify we have the right plan - run "$DAGGER" query -f cue -e "simple" -c -f json + run "$DAGGER" init assert_success - assert_output --partial '{ - "bar": "another value", - "computed": "test", - "foo": "value" -}' -} -@test "dagger new --plan-git" { - "$DAGGER" new --plan-git https://github.com/samalba/dagger-test.git simple - "$DAGGER" up -e "simple" - run "$DAGGER" query -f cue -e "simple" -c + run "$DAGGER" list assert_success - assert_output --partial '{ - foo: "value" - bar: "another value" -}' + refute_output + + run "$DAGGER" new "test" + assert_success + + run "$DAGGER" list + assert_success + assert_output --partial "test" + + run "$DAGGER" new "test" + assert_failure } @test "dagger query" { - "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple + "$DAGGER" init + + dagger_new_with_plan simple "$TESTDIR"/cli/simple + run "$DAGGER" query -l error -e "simple" assert_success assert_output '{ @@ -93,24 +83,10 @@ setup() { }' } -@test "dagger plan" { - "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple - - # plan dir - "$DAGGER" -e "simple" plan dir "$TESTDIR"/cli/simple - run "$DAGGER" -e "simple" query - assert_success - assert_output --partial '"foo": "value"' - - # plan git - "$DAGGER" -e "simple" plan git https://github.com/samalba/dagger-test.git - run "$DAGGER" -e "simple" query - assert_success - assert_output --partial '"foo": "value"' -} - @test "dagger input text" { - "$DAGGER" new --plan-dir "$TESTDIR"/cli/input/simple "input" + "$DAGGER" init + + dagger_new_with_plan input "$TESTDIR"/cli/input/simple # simple input "$DAGGER" input -e "input" text "input" "my input" @@ -176,7 +152,9 @@ setup() { } @test "dagger input json" { - "$DAGGER" new --plan-dir "$TESTDIR"/cli/input/simple "input" + "$DAGGER" init + + dagger_new_with_plan input "$TESTDIR"/cli/input/simple # simple json "$DAGGER" input -e "input" json "structured" '{"a": "foo", "b": 42}' @@ -214,7 +192,9 @@ setup() { } @test "dagger input yaml" { - "$DAGGER" new --plan-dir "$TESTDIR"/cli/input/simple "input" + "$DAGGER" init + + dagger_new_with_plan input "$TESTDIR"/cli/input/simple # simple yaml "$DAGGER" input -e "input" yaml "structured" '{"a": "foo", "b": 42}' @@ -252,10 +232,17 @@ setup() { } @test "dagger input dir" { - "$DAGGER" new --plan-dir "$TESTDIR"/cli/input/artifact "input" + "$DAGGER" init - # input dir - "$DAGGER" input -e "input" dir "source" "$TESTDIR"/cli/input/artifact/testdata + dagger_new_with_plan input "$TESTDIR"/cli/input/artifact + + # input dir outside the workspace + run "$DAGGER" input -e "input" dir "source" /tmp + assert_failure + + # input dir inside the workspace + cp -R "$TESTDIR"/cli/input/artifact/testdata/ "$DAGGER_WORKSPACE"/testdata + "$DAGGER" input -e "input" dir "source" "$DAGGER_WORKSPACE"/testdata "$DAGGER" up -e "input" run "$DAGGER" -l error query -e "input" assert_success @@ -276,7 +263,9 @@ setup() { } @test "dagger input git" { - "$DAGGER" new --plan-dir "$TESTDIR"/cli/input/artifact "input" + "$DAGGER" init + + dagger_new_with_plan input "$TESTDIR"/cli/input/artifact # input git "$DAGGER" input -e "input" git "source" https://github.com/samalba/dagger-test-simple.git @@ -296,11 +285,3 @@ setup() { "foo": "bar" }' } - -@test "dagger input scan" { - "$DAGGER" new --plan-dir "$TESTDIR"/cli/input/scan "scan" - - # TODO "scan" option isn't implemented - run "$DAGGER" input scan -e "input" - assert_success -} diff --git a/tests/examples.bats b/tests/examples.bats index 665354e7..a35323ab 100644 --- a/tests/examples.bats +++ b/tests/examples.bats @@ -7,13 +7,13 @@ setup() { @test "example: react" { skip_unless_secrets_available "$TESTDIR"/examples/react/inputs.yaml - "$DAGGER" new --plan-dir "$TESTDIR"/../examples/react react + "$DAGGER" init + dagger_new_with_plan react "$TESTDIR"/../examples/react sops -d "$TESTDIR"/examples/react/inputs.yaml | "$DAGGER" -e "react" input yaml "" -f - "$DAGGER" up -e "react" # curl the URL we just deployed to check if it worked deployUrl=$("$DAGGER" query -l error -f text -e "react" www.deployUrl) - echo "=>$deployUrl<=" run curl -sS "$deployUrl" assert_success assert_output --partial "Todo App" diff --git a/tests/helpers.bash b/tests/helpers.bash index 00f94b44..7a796768 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -10,8 +10,18 @@ common_setup() { DAGGER_LOG_FORMAT="pretty" export DAGGER_LOG_FORMAT - DAGGER_STORE="$(mktemp -d -t dagger-store-XXXXXX)" - export DAGGER_STORE + DAGGER_WORKSPACE="$(mktemp -d -t dagger-workspace-XXXXXX)" + export DAGGER_WORKSPACE +} + +dagger_new_with_plan() { + local name="$1" + local sourcePlan="$2" + local targetPlan="$DAGGER_WORKSPACE"/.dagger/env/"$name"/plan + + "$DAGGER" new "$name" + rmdir "$targetPlan" + ln -s "$sourcePlan" "$targetPlan" } skip_unless_secrets_available() { diff --git a/tests/stdlib.bats b/tests/stdlib.bats index baf6a169..409169f1 100644 --- a/tests/stdlib.bats +++ b/tests/stdlib.bats @@ -91,8 +91,11 @@ setup() { @test "stdlib: terraform" { skip_unless_secrets_available "$TESTDIR"/stdlib/aws/inputs.yaml - "$DAGGER" new --plan-dir "$TESTDIR"/stdlib/terraform/s3 terraform - "$DAGGER" -e terraform input dir TestData "$TESTDIR"/stdlib/terraform/s3/testdata + "$DAGGER" init + dagger_new_with_plan terraform "$TESTDIR"/stdlib/terraform/s3 + + cp -R "$TESTDIR"/stdlib/terraform/s3/testdata "$DAGGER_WORKSPACE"/testdata + "$DAGGER" -e terraform input dir TestData "$DAGGER_WORKSPACE"/testdata sops -d "$TESTDIR"/stdlib/aws/inputs.yaml | "$DAGGER" -e "terraform" input yaml "" -f - # it must fail because of a missing var From d10b8cf7d0f97d1a79a7bf64c2998f7800f103ab Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 25 May 2021 15:20:44 -0700 Subject: [PATCH 16/16] docs: update docs for gitflow Signed-off-by: Andrea Luzzardi --- docs/programming.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/programming.md b/docs/programming.md index 1365b923..55b15146 100644 --- a/docs/programming.md +++ b/docs/programming.md @@ -26,15 +26,19 @@ To get started with Cue, we recommend the following resources: To create a Dagger plan: -1\. Create a new directory anywhere in your git repository. +1\. Initialize a Dagger workspace anywhere in your git repository. -For example: `mkdir staging`. +`dagger init`. -2\. Create a new file with the *.cue* extension, and open it with any text editor or IDE. +2\. Create a new environment. -For example: `staging.cue`. +For example: `dagger new staging`. -3\. Describe each relay in your plan as a field in the cue configuration. +3\. Create a new file with the *.cue* extension in `.dagger/env/staging/plan`, and open it with any text editor or IDE. + +For example: `.dagger/env/staging/plan/staging.cue`. + +4\. Describe each relay in your plan as a field in the cue configuration. For example: @@ -67,9 +71,9 @@ For more inspiration, see these examples: * [Add HTTP monitoring to your application](https://github.com/dagger/dagger/blob/main/examples/README.md#add-http-monitoring-to-your-application) * [Deploy an application to your Kubernetes cluster](https://github.com/dagger/dagger/blob/main/examples/README.md#deploy-an-application-to-your-kubernetes-cluster) -4\. Extend your plan with relay definitions from [Dagger Universe](../stdlib), an encyclopedia of cue packages curated by the Dagger community. +5\. Extend your plan with relay definitions from [Dagger Universe](../stdlib), an encyclopedia of cue packages curated by the Dagger community. -5\. If you can't find the relay you need in the Universe, you can simply create your own. +6\. If you can't find the relay you need in the Universe, you can simply create your own. For example: