From 226e47e5f4673931d4b791405df0d94bcf7beef8 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 11 Sep 2022 22:56:54 +0200 Subject: [PATCH] with pull into storage --- cmd/kraken/commands/process.go | 48 ++++++++++ cmd/kraken/commands/root.go | 14 +-- cmd/kraken/kraken.go | 4 +- go.mod | 26 +++++- go.sum | 130 +++++++++++++++++++++++++-- internal/commands/process_repos.go | 86 ++++++++++++++++++ internal/server/http_server.go | 51 ++++++----- internal/server/server.go | 2 +- internal/server/storage_server.go | 6 +- internal/serverdeps/server_deps.go | 17 +++- internal/services/actions/action.go | 4 +- internal/services/jobs/models.go | 4 + internal/services/providers/git.go | 118 ++++++++++++++++++++++++ internal/services/storage/storage.go | 25 +++++- roadmap.md | 3 +- 15 files changed, 485 insertions(+), 53 deletions(-) create mode 100644 cmd/kraken/commands/process.go create mode 100644 internal/commands/process_repos.go create mode 100644 internal/services/jobs/models.go create mode 100644 internal/services/providers/git.go diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go new file mode 100644 index 0000000..1097089 --- /dev/null +++ b/cmd/kraken/commands/process.go @@ -0,0 +1,48 @@ +package commands + +import ( + "bytes" + "encoding/json" + "net/http" + + "github.com/spf13/cobra" +) + +func CreateKrakenProcessCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "process", + Run: func(cmd *cobra.Command, args []string) { + client := http.Client{} + + var buf bytes.Buffer + err := json.NewEncoder(&buf). + Encode(struct { + RepositoryUrls []string `json:"repositoryUrls"` + }{ + RepositoryUrls: []string{"git@git.front.kjuulh.io:kjuulh/kraken.git"}}) + if err != nil { + panic(err) + } + + req, err := http.NewRequestWithContext( + cmd.Context(), + http.MethodPost, + "http://localhost:3000/commands/processRepos", + &buf, + ) + if err != nil { + panic(err) + } + + resp, err := client.Do(req) + if err != nil { + panic(err) + } + if resp.StatusCode >= 300 { + panic(resp.Status) + } + }, + } + + return cmd +} diff --git a/cmd/kraken/commands/root.go b/cmd/kraken/commands/root.go index 315a5c1..5fc73bd 100644 --- a/cmd/kraken/commands/root.go +++ b/cmd/kraken/commands/root.go @@ -4,19 +4,11 @@ import "github.com/spf13/cobra" func CreateKrakenCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "krakenserver", - Short: "A brief description of your application", - Long: `A longer description that spans multiple lines and likely contains -examples and usage of using your application. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - // Uncomment the following line if your bare application - // has an action associated with it: + Use: "kraken", // Run: func(cmd *cobra.Command, args []string) { }, } - cmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + + cmd.AddCommand(CreateKrakenProcessCmd()) return cmd } diff --git a/cmd/kraken/kraken.go b/cmd/kraken/kraken.go index 756aaf4..0da4212 100644 --- a/cmd/kraken/kraken.go +++ b/cmd/kraken/kraken.go @@ -3,7 +3,7 @@ package main import ( "os" - "git.front.kjuulh.io/kjuulh/kraken/cmd/server/commands" + "git.front.kjuulh.io/kjuulh/kraken/cmd/kraken/commands" ) func main() { @@ -11,7 +11,7 @@ func main() { } func Execute() { - err := commands.CreateServerCmd(nil).Execute() + err := commands.CreateKrakenCmd().Execute() if err != nil { os.Exit(1) } diff --git a/go.mod b/go.mod index 0b2c312..0ff6c24 100644 --- a/go.mod +++ b/go.mod @@ -5,35 +5,53 @@ go 1.19 require ( git.front.kjuulh.io/kjuulh/curre v1.2.2 github.com/gin-gonic/gin v1.8.1 + github.com/go-git/go-git v4.7.0+incompatible + github.com/go-git/go-git/v5 v5.4.2 github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.0 go.uber.org/zap v1.23.0 - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 + golang.org/x/net v0.0.0-20220909164309-bea034e7d591 ) require ( + github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.10.0 // indirect github.com/goccy/go-json v0.9.7 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/src-d/gcfg v1.4.0 // indirect github.com/ugorji/go/codec v1.2.7 // indirect + github.com/xanzy/ssh-agent v0.3.2 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect - golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect + golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 // indirect + golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/src-d/go-git.v4 v4.13.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8460cab..5156f58 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,47 @@ git.front.kjuulh.io/kjuulh/curre v1.2.2 h1:0OwWIfekrMykdQg9bdmG80I+Mjc2k4i+sy903phuDWs= git.front.kjuulh.io/kjuulh/curre v1.2.2/go.mod h1:m7WpSehONLqPh/XF3F0BI0UOpLOfGuDmDEFI1XsM6fE= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git v4.7.0+incompatible h1:+W9rgGY4DOKKdX2x6HxSR7HNeTxqiKrOvKnuittYVdA= +github.com/go-git/go-git v4.7.0+incompatible/go.mod h1:6+421e08gnZWn30y26Vchf7efgYLe4dl5OQbBSUXShE= +github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -21,46 +53,81 @@ github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXS github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= +github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -69,6 +136,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM= +github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -77,21 +148,55 @@ go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI= -golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho= +golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= @@ -99,9 +204,22 @@ google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscL google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= +gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= +gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go new file mode 100644 index 0000000..85ed345 --- /dev/null +++ b/internal/commands/process_repos.go @@ -0,0 +1,86 @@ +package commands + +import ( + "context" + "sync" + "time" + + "git.front.kjuulh.io/kjuulh/kraken/internal/services/actions" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" + "go.uber.org/zap" +) + +type ( + ProcessRepos struct { + logger *zap.Logger + storage *storage.Service + git *providers.Git + } + + ProcessReposDeps interface { + GetStorageService() *storage.Service + GetGitProvider() *providers.Git + } +) + +func NewProcessRepos(logger *zap.Logger, deps ProcessReposDeps) *ProcessRepos { + return &ProcessRepos{ + logger: logger, + storage: deps.GetStorageService(), + git: deps.GetGitProvider(), + } +} + +func (pr *ProcessRepos) Process(ctx context.Context, repositoryUrls []string, action *actions.Action) error { + // Clone repos + wg := sync.WaitGroup{} + wg.Add(len(repositoryUrls)) + errChan := make(chan error, 1) + + for _, repoUrl := range repositoryUrls { + go func(ctx context.Context, repoUrl string) { + defer func() { + wg.Done() + }() + pr.logger.Debug("Creating area", zap.String("repoUrl", repoUrl)) + area, err := pr.storage.CreateArea(ctx) + if err != nil { + pr.logger.Error("failed to allocate area", zap.Error(err)) + errChan <- err + return + } + + defer func(ctx context.Context) { + pr.logger.Debug("Removing area", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + err = pr.storage.RemoveArea(ctx, area) + if err != nil { + errChan <- err + return + } + + }(ctx) + + pr.logger.Debug("Cloning repo", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + ctx, _ = context.WithTimeout(ctx, time.Second*5) + _, err = pr.git.Clone(ctx, area, repoUrl) + if err != nil { + pr.logger.Error("could not clone repo", zap.Error(err)) + errChan <- err + return + } + + pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + }(ctx, repoUrl) + } + + wg.Wait() + close(errChan) + pr.logger.Debug("finished processing all repos") + + for err := range errChan { + return err + } + + return nil +} diff --git a/internal/server/http_server.go b/internal/server/http_server.go index ecd7e46..18f66a6 100644 --- a/internal/server/http_server.go +++ b/internal/server/http_server.go @@ -4,32 +4,18 @@ import ( "context" "errors" "net/http" + "time" "git.front.kjuulh.io/kjuulh/curre" + "git.front.kjuulh.io/kjuulh/kraken/internal/commands" "git.front.kjuulh.io/kjuulh/kraken/internal/serverdeps" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/jobs" "github.com/gin-gonic/gin" + "github.com/google/uuid" + "go.uber.org/zap" ) -func NewHttpServer(deps *serverdeps.ServerDeps) curre.Component { - return curre.NewFunctionalComponent(&curre.FunctionalComponent{ - StartFunc: func(_ *curre.FunctionalComponent, _ context.Context) error { - handler := http.NewServeMux() - handler.HandleFunc( - "/health/ready", - func(w http.ResponseWriter, _ *http.Request) { - w.Write([]byte("ready")) - w.WriteHeader(http.StatusOK) - }) - - http.ListenAndServe("127.0.0.1:3000", handler) - - return nil - }, - }, - ) -} - -func NewGinHttpServer(_ *serverdeps.ServerDeps) curre.Component { +func NewGinHttpServer(logger *zap.Logger, deps *serverdeps.ServerDeps) curre.Component { var app *gin.Engine var server *http.Server @@ -45,6 +31,30 @@ func NewGinHttpServer(_ *serverdeps.ServerDeps) curre.Component { }) }) + commandRoute := app.Group("commands") + commandRoute.POST("processRepos", func(c *gin.Context) { + type processReposRequest struct { + RepositoryUrls []string `json:"repositoryUrls"` + } + var request processReposRequest + err := c.BindJSON(&request) + if err != nil { + logger.Info("could not bind request", zap.String("request", "processRepo"), zap.Error(err)) + c.AbortWithStatus(http.StatusBadRequest) + return + } + + jobId := uuid.New().String() + + go func(repositoryUrls []string, jobId string) { + ctx := context.WithValue(context.Background(), jobs.JobId{}, jobId) + processRepos := commands.NewProcessRepos(logger, deps) + err = processRepos.Process(ctx, repositoryUrls, nil) + }(request.RepositoryUrls, jobId) + + c.Status(http.StatusAccepted) + }) + server = &http.Server{ Addr: "127.0.0.1:3000", Handler: app, @@ -62,6 +72,7 @@ func NewGinHttpServer(_ *serverdeps.ServerDeps) curre.Component { return nil }, StopFunc: func(_ *curre.FunctionalComponent, ctx context.Context) error { + ctx, _ = context.WithTimeout(ctx, time.Second*10) if server != nil { server.Shutdown(ctx) } diff --git a/internal/server/server.go b/internal/server/server.go index 39fd300..e968028 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -14,7 +14,7 @@ func Start(logger *zap.Logger) error { deps := serverdeps.NewServerDeps(logger) return curre.NewManager(). - Register(NewGinHttpServer(deps)). + Register(NewGinHttpServer(logger.With(zap.String("app", "ginHttpServer")), deps)). Register(NewStorageServer(logger.With(zap.String("app", "storageServer")), deps)). Run(ctx) } diff --git a/internal/server/storage_server.go b/internal/server/storage_server.go index cb79b1f..5be0a36 100644 --- a/internal/server/storage_server.go +++ b/internal/server/storage_server.go @@ -2,6 +2,7 @@ package server import ( "context" + "time" "git.front.kjuulh.io/kjuulh/curre" "git.front.kjuulh.io/kjuulh/kraken/internal/serverdeps" @@ -11,15 +12,16 @@ import ( func NewStorageServer(logger *zap.Logger, deps *serverdeps.ServerDeps) curre.Component { storage := deps.GetStorageService() return curre.NewFunctionalComponent(&curre.FunctionalComponent{ - InitFunc: func(fc *curre.FunctionalComponent, ctx context.Context) error { + InitFunc: func(_ *curre.FunctionalComponent, ctx context.Context) error { logger.Debug("Initializing storage") return storage.InitializeStorage(ctx) }, StartFunc: func(fc *curre.FunctionalComponent, ctx context.Context) error { return nil }, - StopFunc: func(fc *curre.FunctionalComponent, ctx context.Context) error { + StopFunc: func(_ *curre.FunctionalComponent, ctx context.Context) error { logger.Debug("Cleaning up storage") + ctx, _ = context.WithTimeout(ctx, time.Second*10) return storage.CleanupStorage(ctx) }, }) diff --git a/internal/serverdeps/server_deps.go b/internal/serverdeps/server_deps.go index 8673432..bec910b 100644 --- a/internal/serverdeps/server_deps.go +++ b/internal/serverdeps/server_deps.go @@ -1,6 +1,7 @@ package serverdeps import ( + "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" "go.uber.org/zap" ) @@ -8,6 +9,7 @@ import ( type ServerDeps struct { logger *zap.Logger storageConfig *storage.StorageConfig + gitCfg *providers.GitConfig } func NewServerDeps(logger *zap.Logger) *ServerDeps { @@ -21,9 +23,22 @@ func NewServerDeps(logger *zap.Logger) *ServerDeps { deps.storageConfig = storageCfg } + deps.gitCfg = &providers.GitConfig{ + AuthOption: providers.GIT_AUTH_SSH, + User: "git", + Password: "", + AccessToken: "", + SshPublicKeyFilePath: "/Users/kah/.ssh/id_ed25519", + SshPrivateKeyPassword: "", + } + return deps } func (deps *ServerDeps) GetStorageService() *storage.Service { - return storage.NewService(deps.storageConfig) + return storage.NewService(deps.logger.With(zap.String("app", "storage")), deps.storageConfig) +} + +func (deps *ServerDeps) GetGitProvider() *providers.Git { + return providers.NewGit(deps.logger.With(zap.String("app", "gitProvider")), deps.gitCfg) } diff --git a/internal/services/actions/action.go b/internal/services/actions/action.go index 9d39eb6..685f838 100644 --- a/internal/services/actions/action.go +++ b/internal/services/actions/action.go @@ -1,2 +1,4 @@ -package action +package actions +type Action struct { +} diff --git a/internal/services/jobs/models.go b/internal/services/jobs/models.go new file mode 100644 index 0000000..1dea285 --- /dev/null +++ b/internal/services/jobs/models.go @@ -0,0 +1,4 @@ +package jobs + +type JobId struct { +} diff --git a/internal/services/providers/git.go b/internal/services/providers/git.go new file mode 100644 index 0000000..02706dc --- /dev/null +++ b/internal/services/providers/git.go @@ -0,0 +1,118 @@ +package providers + +import ( + "context" + + "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "go.uber.org/zap" + "go.uber.org/zap/zapio" +) + +// Git is a native git provider, it can clone, pull +// , push and as in abstraction on native git operations +type Git struct { + logger *zap.Logger + gitConfig *GitConfig +} + +type GitRepo struct { + repo *git.Repository +} + +type GitAuth string + +const ( + GIT_AUTH_SSH GitAuth = "ssh" + GIT_AUTH_USERNAME_PASSWORD GitAuth = "username_password" + GIT_AUTH_ACCESS_TOKEN GitAuth = "access_token" + GIT_AUTH_ANONYMOUS GitAuth = "anonymous" + GIT_AUTH_SSH_AGENT GitAuth = "ssh_agent" +) + +type GitConfig struct { + AuthOption GitAuth + User string + Password string + AccessToken string + SshPublicKeyFilePath string + SshPrivateKeyPassword string +} + +func NewGit(logger *zap.Logger, gitConfig *GitConfig) *Git { + return &Git{logger: logger, gitConfig: gitConfig} +} + +func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl string) (*GitRepo, error) { + g.logger.Debug( + "cloning repository", + zap.String("repoUrl", repoUrl), + zap.String("path", storageArea.Path), + ) + + auth, err := g.GetAuth() + if err != nil { + return nil, err + } + + cloneOptions := git.CloneOptions{ + URL: repoUrl, + Auth: auth, + RemoteName: "origin", + ReferenceName: "refs/heads/main", + SingleBranch: true, + NoCheckout: false, + Depth: 1, + RecurseSubmodules: 1, + Progress: &zapio.Writer{ + Log: g.logger.With(zap.String("process", "go-git")), + Level: zap.DebugLevel, + }, + Tags: 0, + InsecureSkipTLS: false, + CABundle: []byte{}, + } + + repo, err := git.PlainCloneContext(ctx, storageArea.Path, false, &cloneOptions) + if err != nil { + return nil, err + } + + g.logger.Debug("done cloning repo") + + return &GitRepo{repo: repo}, nil +} + +func (g *Git) GetAuth() (transport.AuthMethod, error) { + switch g.gitConfig.AuthOption { + case GIT_AUTH_SSH: + sshKey, err := ssh.NewPublicKeysFromFile( + g.gitConfig.User, + g.gitConfig.SshPublicKeyFilePath, + g.gitConfig.SshPrivateKeyPassword, + ) + if err != nil { + return nil, err + } + return sshKey, nil + case GIT_AUTH_USERNAME_PASSWORD: + return &http.BasicAuth{ + Username: g.gitConfig.User, + Password: g.gitConfig.Password, + }, nil + case GIT_AUTH_ACCESS_TOKEN: + return &http.BasicAuth{ + Username: "required-username", + Password: g.gitConfig.AccessToken, + }, nil + case GIT_AUTH_ANONYMOUS: + return nil, nil + case GIT_AUTH_SSH_AGENT: + return ssh.NewSSHAgentAuth(g.gitConfig.User) + default: + return nil, nil + } +} diff --git a/internal/services/storage/storage.go b/internal/services/storage/storage.go index 7ef0ab5..01a2625 100644 --- a/internal/services/storage/storage.go +++ b/internal/services/storage/storage.go @@ -1,9 +1,11 @@ package storage import ( + "errors" "os" "path" + "go.uber.org/zap" "golang.org/x/net/context" ) @@ -25,11 +27,12 @@ func NewDefaultStorageConfig() (*StorageConfig, error) { } type Service struct { - cfg *StorageConfig + logger *zap.Logger + cfg *StorageConfig } -func NewService(cfg *StorageConfig) *Service { - return &Service{cfg: cfg} +func NewService(logger *zap.Logger, cfg *StorageConfig) *Service { + return &Service{logger: logger, cfg: cfg} } func (s *Service) getStoragePath(ctx context.Context) string { @@ -41,7 +44,21 @@ func (s *Service) InitializeStorage(ctx context.Context) error { } func (s *Service) CleanupStorage(ctx context.Context) error { - return os.RemoveAll(s.getStoragePath(ctx)) + doneRemovingChan := make(chan struct{}, 1) + go func(ctx context.Context) { + s.logger.Debug("Removing all temp storage") + os.RemoveAll(s.getStoragePath(ctx)) + doneRemovingChan <- struct{}{} + }(ctx) + + select { + case <-ctx.Done(): + return errors.New("could not cleanup storage aborting") + case <-doneRemovingChan: + return nil + } + + return nil } func (s *Service) CreateArea(ctx context.Context) (*Area, error) { diff --git a/roadmap.md b/roadmap.md index cc57dcd..f24be08 100644 --- a/roadmap.md +++ b/roadmap.md @@ -2,8 +2,9 @@ ## POC: +- [ ] Add cuddle - [x] Create storage mechanism -- [ ] Pull repository into storage +- [x] Pull repository into storage - [ ] Create test action to run on repository - [ ] List git repositories on git.front.kjuulh.io. - [ ] Run action on each repo in list (hard-coded) using ssh and gpg.