kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

No more runtime spec validation

Browse Source 
Signed-off-by: Solomon Hykes <sh.github.6811@hykes.org>
This commit is contained in:
Solomon Hykes
2021-02-12 22:37:41 +00:00
parent ec56160307
commit e8527ddcf5
11 changed files with 209 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
dagger/env.go
View File

@@ -2,7 +2,6 @@ package dagger
import (
"context"
"os"
"cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow"
@@ -104,76 +103,14 @@ func (env *Env) Update(ctx context.Context, s Solver) error {
if err != nil {
return errors.Wrap(err, "base config")
}
final, err := applySpec(base)
if err != nil {
return err
}
// Commit
return env.set(
final,
base,
env.input,
env.output,
)
}
// Scan the env config for compute scripts, and merge the spec over them,
// for validation and default value expansion.
// This is done once when loading the env configuration, as opposed to dynamically
// during compute like in previous versions. Hopefully this will improve performance.
//
// Also note that performance was improved DRASTICALLY by splitting the #Component spec
// into individual #ComputableStruct, #ComputableString etc. It appears that it is massively
// faster to check for the type in Go, then apply the correct spec, than rely on a cue disjunction.
//
// FIXME: re-enable support for scalar types beyond string.
//
// FIXME: remove dependency on #Component def so it can be deprecated.
func applySpec(base *cc.Value) (*cc.Value, error) {
if os.Getenv("NO_APPLY_SPEC") != "" {
return base, nil
}
// Merge the spec to validate & expand buildkit scripts
computableStructs := []cue.Path{}
computableStrings := []cue.Path{}
base.Walk(
func(v *cc.Value) bool {
compute := v.Get("#dagger.compute")
if !compute.Exists() {
return true // keep scanning
}
if _, err := v.String(); err == nil {
// computable string
computableStrings = append(computableStrings, v.Path())
return false
}
if _, err := v.Struct(); err == nil {
// computable struct
computableStructs = append(computableStructs, v.Path())
return false
}
return false
},
nil,
)
structSpec := spec.Get("#ComputableStruct")
for _, target := range computableStructs {
newbase, err := base.MergePath(structSpec, target)
if err != nil {
return nil, err
}
base = newbase
}
stringSpec := spec.Get("#ComputableString")
for _, target := range computableStrings {
newbase, err := base.MergePath(stringSpec, target)
if err != nil {
return nil, err
}
base = newbase
}
return base, nil
}
func (env *Env) Base() *cc.Value {
return env.base
}

110
dagger/gen.go
View File

@@ -1,110 +0,0 @@
package dagger
// Generated by gen.sh. DO NOT EDIT.
var DaggerSpec = `
package dagger
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#ComputableStruct: {
#dagger: compute: [...#Op]
...
}
#ComputableString: {
string
#dagger: compute: [...#Op]
}
#Component: {
// Match structs
#dagger: #ComponentConfig
...
} | {
// Match embedded scalars
bool | int | float | string | bytes
#dagger: #ComponentConfig
}
// The contents of a #dagger annotation
#ComponentConfig: {
// script to compute the value
compute?: #Script
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
// One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir
// Export a value from fs state to cue
#Export: {
do: "export"
// Source path in the container
source: string
format: "json" | "yaml" | *"string"
}
#Local: {
do: "local"
dir: string
include: [...string] | *[]
}
// FIXME: bring back load (more efficient than copy)
#Load: {
do: "load"
from: #Component | #Script
}
#Subdir: {
do: "subdir"
dir: string | *"/"
}
#Exec: {
do: "exec"
args: [...string]
env?: [string]: string
always?: true | *false
dir: string | *"/"
mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
from: #Component
path: string | *"/"
}
#MountScript: {
from: #Script
path: string | *"/"
}
#FetchContainer: {
do: "fetch-container"
ref: string
}
#FetchGit: {
do: "fetch-git"
remote: string
ref: string
}
#Copy: {
do: "copy"
from: #Script | #Component
src: string | *"/"
dest: string | *"/"
}
`

16
dagger/gen.sh
View File

@@ -1,16 +0,0 @@
#!/bin/bash
set -e
(
cat <<'EOF'
package dagger
// Generated by gen.sh. DO NOT EDIT.
var DaggerSpec = `
EOF
cat spec.cue
cat <<'EOF'
`
EOF
) > gen.go

104
dagger/spec.cue
View File

@@ -1,104 +0,0 @@
package dagger
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#ComputableStruct: {
#dagger: compute: [...#Op]
...
}
#ComputableString: {
string
#dagger: compute: [...#Op]
}
#Component: {
// Match structs
#dagger: #ComponentConfig
...
} | {
// Match embedded scalars
bool | int | float | string | bytes
#dagger: #ComponentConfig
}
// The contents of a #dagger annotation
#ComponentConfig: {
// script to compute the value
compute?: #Script
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
// One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir
// Export a value from fs state to cue
#Export: {
do: "export"
// Source path in the container
source: string
format: "json" | "yaml" | *"string"
}
#Local: {
do: "local"
dir: string
include: [...string] | *[]
}
// FIXME: bring back load (more efficient than copy)
#Load: {
do: "load"
from: #Component | #Script
}
#Subdir: {
do: "subdir"
dir: string | *"/"
}
#Exec: {
do: "exec"
args: [...string]
env?: [string]: string
always?: true | *false
dir: string | *"/"
mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
from: #Component
path: string | *"/"
}
#MountScript: {
from: #Script
path: string | *"/"
}
#FetchContainer: {
do: "fetch-container"
ref: string
}
#FetchGit: {
do: "fetch-git"
remote: string
ref: string
}
#Copy: {
do: "copy"
from: #Script | #Component
src: string | *"/"
dest: string | *"/"
}

53
dagger/spec.go
View File

@@ -1,53 +0,0 @@
//go:generate sh gen.sh
package dagger
import (
cueerrors "cuelang.org/go/cue/errors"
"github.com/pkg/errors"
"dagger.cloud/go/dagger/cc"
)
var (
// Global shared dagger spec, generated from spec.cue
spec = NewSpec()
)
// Cue spec validator
type Spec struct {
root *cc.Value
}
func NewSpec() *Spec {
v, err := cc.Compile("spec.cue", DaggerSpec)
if err != nil {
panic(err)
}
if _, err := v.Struct(); err != nil {
panic(err)
}
return &Spec{
root: v,
}
}
// eg. Validate(op, "#Op")
func (s Spec) Validate(v *cc.Value, defpath string) error {
// Lookup def by name, eg. "#Script" or "#Copy"
// See dagger/spec.cue
def := s.root.Get(defpath)
if err := def.Fill(v); err != nil {
return errors.New(cueerrors.Details(err, nil))
}
return nil
}
func (s Spec) Match(v *cc.Value, defpath string) bool {
return s.Validate(v, defpath) == nil
}
func (s Spec) Get(target string) *cc.Value {
return s.root.Get(target)
}

59
dagger/spec_test.go
View File

@@ -1,59 +0,0 @@
package dagger
import (
"testing"
"dagger.cloud/go/dagger/cc"
)
func TestMatch(t *testing.T) {
var data = []struct {
Src string
Def string
}{
{
Src: `do: "exec", args: ["echo", "hello"]`,
Def: "#Exec",
},
{
Src: `do: "fetch-git", remote: "github.com/shykes/tests"`,
Def: "#FetchGit",
},
}
for _, d := range data {
testMatch(t, d.Src, d.Def)
}
}
// Test an example op for false positives and negatives
func testMatch(t *testing.T, src interface{}, def string) {
op := compile(t, src)
if def != "" {
if err := spec.Validate(op, def); err != nil {
t.Errorf("false negative: %s: %q: %s", def, src, err)
}
}
for _, cmpDef := range []string{
"#Exec",
"#FetchGit",
"#FetchContainer",
"#Export",
"#Copy",
"#Local",
} {
if cmpDef == def {
continue
}
if err := spec.Validate(op, cmpDef); err == nil {
t.Errorf("false positive: %s: %q", cmpDef, src)
}
}
}
func compile(t *testing.T, src interface{}) *cc.Value {
v, err := cc.Compile("", src)
if err != nil {
t.Fatal(err)
}
return v
}