From a33a4f47494f08aba1304c3530045873f26565d4 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Tue, 17 Aug 2021 02:59:22 +0200 Subject: [PATCH 1/7] AWS - Implement backward compatible localstack Signed-off-by: Guillaume de Rouville --- stdlib/aws/aws.cue | 48 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/stdlib/aws/aws.cue b/stdlib/aws/aws.cue index 2e0a3ba5..8c462c7a 100644 --- a/stdlib/aws/aws.cue +++ b/stdlib/aws/aws.cue @@ -10,11 +10,13 @@ import ( // AWS Config shared by all AWS packages #Config: { // AWS region - region: string @dagger(input) + region: dagger.#Input & { string } // AWS access key - accessKey: dagger.#Secret @dagger(input) + accessKey: dagger.#Input & { dagger.#Secret } // AWS secret key - secretKey: dagger.#Secret @dagger(input) + secretKey: dagger.#Input & { dagger.#Secret } + // AWS localstack mode + localMode: dagger.#Input & { string | *null } } // Re-usable aws-cli component @@ -30,6 +32,9 @@ import ( "package": jq: "=~1.6" "package": curl: true "package": "aws-cli": "=~1.18" + if config.localMode != null { + "package": "py3-pip": true + } } }, op.#Exec & { @@ -40,14 +45,37 @@ import ( "-eo", "pipefail", "-c", - #""" - aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" - aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" + if config.localMode == null { + #""" + aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" + aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" - aws configure set default.region "$AWS_DEFAULT_REGION" - aws configure set default.cli_pager "" - aws configure set default.output "json" - """#, + aws configure set default.region "$AWS_DEFAULT_REGION" + aws configure set default.cli_pager "" + aws configure set default.output "json" + """#, + } + if config.localMode != null { + #""" + # Download awscli v3 and override aws + pip install awscli-local[v2] + mv /usr/bin/awslocal /usr/bin/aws + + # Configure + mkdir -p ~/.aws/ + + # Set up ~/.aws/config + echo "[default]" > ~/.aws/config + echo "region = $AWS_DEFAULT_REGION" >> ~/.aws/config + echo "cli_pager =" >> ~/.aws/config + echo "output = json" >> ~/.aws/config + + # Set up ~/.aws/credentials + echo "[default]" > ~/.aws/credentials + echo "aws_access_key_id = $(cat /run/secrets/access_key)" >> ~/.aws/credentials + echo "aws_secret_access_key = $(cat /run/secrets/secret_key)" >> ~/.aws/credentials + """#, + } ] mount: "/run/secrets/access_key": secret: config.accessKey mount: "/run/secrets/secret_key": secret: config.secretKey From 68707d93ae524cfb03a0f175b9ca3d3c245ff9eb Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Tue, 17 Aug 2021 02:59:55 +0200 Subject: [PATCH 2/7] S3: Implement localstack env Signed-off-by: Guillaume de Rouville --- .../.dagger/env/aws-s3-localstack/.gitignore | 2 ++ .../.dagger/env/aws-s3-localstack/values.yaml | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 stdlib/.dagger/env/aws-s3-localstack/.gitignore create mode 100644 stdlib/.dagger/env/aws-s3-localstack/values.yaml diff --git a/stdlib/.dagger/env/aws-s3-localstack/.gitignore b/stdlib/.dagger/env/aws-s3-localstack/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/stdlib/.dagger/env/aws-s3-localstack/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/stdlib/.dagger/env/aws-s3-localstack/values.yaml b/stdlib/.dagger/env/aws-s3-localstack/values.yaml new file mode 100644 index 00000000..22c4e683 --- /dev/null +++ b/stdlib/.dagger/env/aws-s3-localstack/values.yaml @@ -0,0 +1,33 @@ +plan: + package: ./aws/s3/tests +name: aws-s3-localstack +inputs: + TestConfig.awsConfig.accessKey: + secret: ENC[AES256_GCM,data:RJY8Cw==,iv:yZW4m74SdRl5qeDfyQVJPOvAktu2qt8jJB45hWUd5p8=,tag:7DCf5ys4Cblb1V1XHMCWAQ==,type:str] + TestConfig.awsConfig.localMode: + text: "true" + TestConfig.awsConfig.secretKey: + secret: ENC[AES256_GCM,data:iYxBnA==,iv:7mdUXVnmMJm2pz9Kt1MwrwFeAxwqZaH9o1D2FbUD860=,tag:fj8+JJbNS3Z/L2ft+BtJ1Q==,type:str] + TestDirectory: + dir: + path: ./aws/s3/tests/testdata +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeEk5MS9nVmFoOVNNOHdE + WnZCTXBWbW9LL1NJYndCYjhIM2JsNXNEUmxJCkUya0dlZjZ0dGRIM1pVdzg5eWFH + MVpiaE9PclNudGdUZm5FcytuVDZGTDAKLS0tIEQxWDdteHgzS3JkdmtNTVpxMUh1 + aXlvVWJVSGNTSkVyYmpZbi9nUVJZdmMK6csXZ2RMxFw5DB+Hb2TyhyoZT8c2/z7Y + Lc9Pe8gb8aUq5Ha+wCybYvY6JWEM5A9XYJKbE7f4borTfGKS72d6pw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-08-16T20:43:09Z" + mac: ENC[AES256_GCM,data:suv1ImWFc+7HyrpNw66vasTf75PLu07DivxKCyuqcLFqBBs9qgxg8VFXax1hiZRp7NLTG1cqJ3JLW8DsPzhFTs2xJEDQ8BBKfqNFGgHzucaAISlfKzKKQtPb4NM+56WEmuiF7ns0jKyJH385A1qttD8pVeyLnj4ps+9iU/iA5wo=,iv:LT5XzerWt6ctuPYd/Pm7sdSQni30G5tBNQni+Nfmd3s=,tag:OO+/JhjwcZbnFCIFvOM17g==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 From e29bdfb422184fc6beb3d5bcb49eca716fd70f37 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Tue, 17 Aug 2021 03:00:43 +0200 Subject: [PATCH 3/7] ECR: Implement localstack env Signed-off-by: Guillaume de Rouville --- .../.dagger/env/aws-ecr-localstack/.gitignore | 2 ++ .../env/aws-ecr-localstack/values.yaml | 30 +++++++++++++++++++ stdlib/aws/ecr/tests/ecr.cue | 10 ++++++- 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 stdlib/.dagger/env/aws-ecr-localstack/.gitignore create mode 100644 stdlib/.dagger/env/aws-ecr-localstack/values.yaml diff --git a/stdlib/.dagger/env/aws-ecr-localstack/.gitignore b/stdlib/.dagger/env/aws-ecr-localstack/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/stdlib/.dagger/env/aws-ecr-localstack/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/stdlib/.dagger/env/aws-ecr-localstack/values.yaml b/stdlib/.dagger/env/aws-ecr-localstack/values.yaml new file mode 100644 index 00000000..4dc687ba --- /dev/null +++ b/stdlib/.dagger/env/aws-ecr-localstack/values.yaml @@ -0,0 +1,30 @@ +plan: + package: ./aws/ecr/tests +name: aws-ecr-localstack +inputs: + TestConfig.awsConfig.accessKey: + secret: ENC[AES256_GCM,data:t/iOlA==,iv:oLhubqSKZqn5lVPFn8On//LPj1deAFPXKVTyBKdVODQ=,tag:PORmrxDIf/MHbFDrdxnlTQ==,type:str] + TestConfig.awsConfig.localMode: + text: "true" + TestConfig.awsConfig.secretKey: + secret: ENC[AES256_GCM,data:2HYF9w==,iv:f/a9dy4HLJOtKdn8G+zKTboXU1YhzXBEKT1WSvrsCvk=,tag:RKQO+Pg3wg23LIMAnyXhMA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeEk5MS9nVmFoOVNNOHdE + WnZCTXBWbW9LL1NJYndCYjhIM2JsNXNEUmxJCkUya0dlZjZ0dGRIM1pVdzg5eWFH + MVpiaE9PclNudGdUZm5FcytuVDZGTDAKLS0tIEQxWDdteHgzS3JkdmtNTVpxMUh1 + aXlvVWJVSGNTSkVyYmpZbi9nUVJZdmMK6csXZ2RMxFw5DB+Hb2TyhyoZT8c2/z7Y + Lc9Pe8gb8aUq5Ha+wCybYvY6JWEM5A9XYJKbE7f4borTfGKS72d6pw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-08-16T20:52:50Z" + mac: ENC[AES256_GCM,data:31GBJoqzJOt2q29ql9xTGvhg6XaMupZliAEx2XXD19CvJuAlPdkkbdOP1quEDjHqeBfWUZlTTnqqm2HdSjJ9MgS7aSpgIwb65g/KMez/13QKqlicKaOT55uuyPmWh9+p0rSBCWTHo1vcZldXEuQ9eohwgkV1xHGU3tV+IsmlfrM=,iv:ISdTPiT3rUpZrmMoMBDYmQEB5Q6Tg5fx8leAS3B2kBU=,tag:z/4/2MS6CSlEC++9kCKn5Q==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 diff --git a/stdlib/aws/ecr/tests/ecr.cue b/stdlib/aws/ecr/tests/ecr.cue index c78a68eb..561f2cd7 100644 --- a/stdlib/aws/ecr/tests/ecr.cue +++ b/stdlib/aws/ecr/tests/ecr.cue @@ -11,11 +11,19 @@ TestConfig: awsConfig: aws.#Config & { } TestECR: { + localMode: TestConfig.awsConfig.localMode + suffix: random.#String & { seed: "" } - repository: "125635003186.dkr.ecr.\(TestConfig.awsConfig.region).amazonaws.com/dagger-ci" + repository: string + if localMode == null { + repository: "125635003186.dkr.ecr.\(TestConfig.awsConfig.region).amazonaws.com/dagger-ci" + } + if localMode != null { + repository: "localhost:4514/dagger-ci" + } tag: "test-ecr-\(suffix.out)" creds: #Credentials & { From a7189084c044782397111386ffdb3464cb864f72 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Wed, 18 Aug 2021 15:09:28 +0200 Subject: [PATCH 4/7] Update Helpers.bash - Universe.bats Signed-off-by: Guillaume de Rouville --- stdlib/helpers.bash | 10 ++++++++++ stdlib/universe.bats | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/stdlib/helpers.bash b/stdlib/helpers.bash index f2f4b4c7..94707528 100644 --- a/stdlib/helpers.bash +++ b/stdlib/helpers.bash @@ -58,6 +58,16 @@ copy_to_sandbox() { cp -a "$source_package" "$target_package" fi } +# Check if there is a local kubernetes cluster. +# +# This is need to do kubernetes test in the CI. +skip_unless_local_localstack() { + if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ]; then + echo "Localstack available" + else + skip "Localstack not available" + fi +} # Check if there is a local kubernetes cluster. # diff --git a/stdlib/universe.bats b/stdlib/universe.bats index bfb92323..ddc2379b 100644 --- a/stdlib/universe.bats +++ b/stdlib/universe.bats @@ -45,10 +45,22 @@ setup() { dagger -e aws-ecr up } +@test "aws: ecr/localstack" { + skip_unless_local_localstack + + dagger -e aws-ecr-localstack up +} + @test "aws: s3" { dagger -e aws-s3 up } +@test "aws: s3/localstack" { + skip_unless_local_localstack + + dagger -e aws-s3-localstack up +} + @test "aws: eks" { dagger -e aws-eks up } From f42a2436921c64c6bacf123c9d811975fe3088b6 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Thu, 19 Aug 2021 17:07:28 +0200 Subject: [PATCH 5/7] Add Localstack inside CI Signed-off-by: Guillaume de Rouville --- .github/workflows/ci.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75a91212..782cd50a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,22 @@ jobs: name: Universe runs-on: ubuntu-latest timeout-minutes: 30 + services: + localstack: + image: localstack/localstack:latest + env: + SERVICES: s3, ecr + LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + ports: + - 4566:4566 + - 4571:4571 + options: >- + --health-cmd "curl -f http://localhost:4566/health" + --health-start-period 5s + --health-timeout 5s + --health-interval 5s + --health-retries 10 + steps: - name: Check out uses: actions/checkout@v2 @@ -116,7 +132,16 @@ jobs: run: | mkdir -p ~/.config/dagger echo "$DAGGER_AGE_KEY" > ~/.config/dagger/keys.txt - + + - name: Provision Localstack AWS resources + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: eu-east-2 + run: | + aws --endpoint-url=http://localhost:4566 s3 mb s3://dagger-ci + aws --endpoint-url=http://localhost:4566 ecr create-repository --repository-name dagger-ci + - name: Universe Test run: | make universe-test From 58209b87d108f43342f7d46c36ba50d6d5c6917a Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Thu, 19 Aug 2021 17:11:33 +0200 Subject: [PATCH 6/7] Doc update + linting + fix CI typo + fix port issue ECR Signed-off-by: Guillaume de Rouville --- .github/workflows/ci.yml | 5 +- docs/reference/universe/aws/README.md | 22 ++++--- docs/reference/universe/aws/cloudformation.md | 1 + docs/reference/universe/aws/ecr.md | 18 +++--- docs/reference/universe/aws/eks.md | 15 ++--- docs/reference/universe/aws/elb.md | 13 ++-- docs/reference/universe/aws/rds.md | 55 ++++++++-------- docs/reference/universe/aws/s3.md | 1 + stdlib/aws/aws.cue | 62 +++++++++---------- stdlib/aws/ecr/tests/ecr.cue | 6 +- stdlib/helpers.bash | 4 +- 11 files changed, 107 insertions(+), 95 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 782cd50a..a31984b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,13 +100,14 @@ jobs: timeout-minutes: 30 services: localstack: - image: localstack/localstack:latest + image: localstack/localstack:0.12.16 env: SERVICES: s3, ecr LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} ports: - 4566:4566 - 4571:4571 + - 4510:4510 options: >- --health-cmd "curl -f http://localhost:4566/health" --health-start-period 5s @@ -137,7 +138,7 @@ jobs: env: AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test - AWS_DEFAULT_REGION: eu-east-2 + AWS_DEFAULT_REGION: us-east-2 run: | aws --endpoint-url=http://localhost:4566 s3 mb s3://dagger-ci aws --endpoint-url=http://localhost:4566 ecr create-repository --repository-name dagger-ci diff --git a/docs/reference/universe/aws/README.md b/docs/reference/universe/aws/README.md index 7484c97e..ad23b78d 100644 --- a/docs/reference/universe/aws/README.md +++ b/docs/reference/universe/aws/README.md @@ -16,11 +16,12 @@ Re-usable aws-cli component ### aws.#CLI Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | ### aws.#CLI Outputs @@ -32,11 +33,12 @@ AWS Config shared by all AWS packages ### aws.#Config Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*region* | `string` |AWS region | -|*accessKey* | `dagger.#Secret` |AWS access key | -|*secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*region* | `string` |AWS region | +|*accessKey* | `dagger.#Secret` |AWS access key | +|*secretKey* | `dagger.#Secret` |AWS secret key | +|*localMode* | `*null \| string` |AWS localstack mode | ### aws.#Config Outputs diff --git a/docs/reference/universe/aws/cloudformation.md b/docs/reference/universe/aws/cloudformation.md index 8528db70..11b81e17 100644 --- a/docs/reference/universe/aws/cloudformation.md +++ b/docs/reference/universe/aws/cloudformation.md @@ -21,6 +21,7 @@ AWS CloudFormation Stack |*config.region* | `string` |AWS region | |*config.accessKey* | `dagger.#Secret` |AWS access key | |*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | |*source* | `string` |Source is the Cloudformation template (JSON/YAML string) | |*stackName* | `string` |Stackname is the cloudformation stack | |*parameters* | `struct` |Stack parameters | diff --git a/docs/reference/universe/aws/ecr.md b/docs/reference/universe/aws/ecr.md index c45d1824..bbfcd619 100644 --- a/docs/reference/universe/aws/ecr.md +++ b/docs/reference/universe/aws/ecr.md @@ -16,14 +16,16 @@ Convert ECR credentials to Docker Login format ### ecr.#Credentials Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*ctr.image.config.region* | `string` |AWS region | -|*ctr.image.config.accessKey* | `dagger.#Secret` |AWS access key | -|*ctr.image.config.secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*ctr.image.config.region* | `string` |AWS region | +|*ctr.image.config.accessKey* | `dagger.#Secret` |AWS access key | +|*ctr.image.config.secretKey* | `dagger.#Secret` |AWS secret key | +|*ctr.image.config.localMode* | `*null \| string` |AWS localstack mode | ### ecr.#Credentials Outputs diff --git a/docs/reference/universe/aws/eks.md b/docs/reference/universe/aws/eks.md index 7d445769..f7ca7423 100644 --- a/docs/reference/universe/aws/eks.md +++ b/docs/reference/universe/aws/eks.md @@ -16,13 +16,14 @@ KubeConfig config outputs a valid kube-auth-config for kubectl client ### eks.#KubeConfig Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*clusterName* | `string` |EKS cluster name | -|*version* | `*"v1.19.9" \| string` |Kubectl version | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*clusterName* | `string` |EKS cluster name | +|*version* | `*"v1.19.9" \| string` |Kubectl version | ### eks.#KubeConfig Outputs diff --git a/docs/reference/universe/aws/elb.md b/docs/reference/universe/aws/elb.md index adf122a4..97142576 100644 --- a/docs/reference/universe/aws/elb.md +++ b/docs/reference/universe/aws/elb.md @@ -16,12 +16,13 @@ Returns an unused rule priority (randomized in available range) ### elb.#RandomRulePriority Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*listenerArn* | `string` |ListenerArn | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*listenerArn* | `string` |ListenerArn | ### elb.#RandomRulePriority Outputs diff --git a/docs/reference/universe/aws/rds.md b/docs/reference/universe/aws/rds.md index 9c5735f1..df58a790 100644 --- a/docs/reference/universe/aws/rds.md +++ b/docs/reference/universe/aws/rds.md @@ -16,15 +16,16 @@ Creates a new Database on an existing RDS Instance ### rds.#Database Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*name* | `string` |DB name | -|*dbArn* | `string` |ARN of the database instance | -|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | -|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*name* | `string` |DB name | +|*dbArn* | `string` |ARN of the database instance | +|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | +|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | ### rds.#Database Outputs @@ -38,12 +39,13 @@ Fetches information on an existing RDS Instance ### rds.#Instance Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*dbArn* | `string` |ARN of the database instance | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*dbArn* | `string` |ARN of the database instance | ### rds.#Instance Outputs @@ -59,17 +61,18 @@ Creates a new user credentials on an existing RDS Instance ### rds.#User Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*username* | `string` |Username | -|*password* | `string` |Password | -|*dbArn* | `string` |ARN of the database instance | -|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | -|*grantDatabase* | `*"" \| string` |Name of the database to grants access to | -|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*username* | `string` |Username | +|*password* | `string` |Password | +|*dbArn* | `string` |ARN of the database instance | +|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | +|*grantDatabase* | `*"" \| string` |Name of the database to grants access to | +|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | ### rds.#User Outputs diff --git a/docs/reference/universe/aws/s3.md b/docs/reference/universe/aws/s3.md index ca7b54dd..0b1c2431 100644 --- a/docs/reference/universe/aws/s3.md +++ b/docs/reference/universe/aws/s3.md @@ -21,6 +21,7 @@ S3 Bucket object(s) sync |*config.region* | `string` |AWS region | |*config.accessKey* | `dagger.#Secret` |AWS access key | |*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | |*source* | `dagger.#Artifact` |Source Artifact to upload to S3 | |*target* | `string` |Target S3 URL (eg. s3://\/\/\) | |*delete* | `*false \| true` |Delete files that already exist on remote destination | diff --git a/stdlib/aws/aws.cue b/stdlib/aws/aws.cue index 8c462c7a..c3e24fec 100644 --- a/stdlib/aws/aws.cue +++ b/stdlib/aws/aws.cue @@ -10,13 +10,13 @@ import ( // AWS Config shared by all AWS packages #Config: { // AWS region - region: dagger.#Input & { string } + region: dagger.#Input & {string} // AWS access key - accessKey: dagger.#Input & { dagger.#Secret } + accessKey: dagger.#Input & {dagger.#Secret} // AWS secret key - secretKey: dagger.#Input & { dagger.#Secret } + secretKey: dagger.#Input & {dagger.#Secret} // AWS localstack mode - localMode: dagger.#Input & { string | *null } + localMode: dagger.#Input & {string | *null} } // Re-usable aws-cli component @@ -33,7 +33,7 @@ import ( "package": curl: true "package": "aws-cli": "=~1.18" if config.localMode != null { - "package": "py3-pip": true + package: "py3-pip": true } } }, @@ -46,36 +46,36 @@ import ( "pipefail", "-c", if config.localMode == null { - #""" - aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" - aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" + #""" + aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" + aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" - aws configure set default.region "$AWS_DEFAULT_REGION" - aws configure set default.cli_pager "" - aws configure set default.output "json" - """#, - } - if config.localMode != null { - #""" - # Download awscli v3 and override aws - pip install awscli-local[v2] - mv /usr/bin/awslocal /usr/bin/aws + aws configure set default.region "$AWS_DEFAULT_REGION" + aws configure set default.cli_pager "" + aws configure set default.output "json" + """# + }, + if config.localMode != null { + #""" + # Download awscli v3 and override aws + pip install awscli-local[v2] + mv /usr/bin/awslocal /usr/bin/aws - # Configure - mkdir -p ~/.aws/ + # Configure + mkdir -p ~/.aws/ - # Set up ~/.aws/config - echo "[default]" > ~/.aws/config - echo "region = $AWS_DEFAULT_REGION" >> ~/.aws/config - echo "cli_pager =" >> ~/.aws/config - echo "output = json" >> ~/.aws/config + # Set up ~/.aws/config + echo "[default]" > ~/.aws/config + echo "region = $AWS_DEFAULT_REGION" >> ~/.aws/config + echo "cli_pager =" >> ~/.aws/config + echo "output = json" >> ~/.aws/config - # Set up ~/.aws/credentials - echo "[default]" > ~/.aws/credentials - echo "aws_access_key_id = $(cat /run/secrets/access_key)" >> ~/.aws/credentials - echo "aws_secret_access_key = $(cat /run/secrets/secret_key)" >> ~/.aws/credentials - """#, - } + # Set up ~/.aws/credentials + echo "[default]" > ~/.aws/credentials + echo "aws_access_key_id = $(cat /run/secrets/access_key)" >> ~/.aws/credentials + echo "aws_secret_access_key = $(cat /run/secrets/secret_key)" >> ~/.aws/credentials + """# + }, ] mount: "/run/secrets/access_key": secret: config.accessKey mount: "/run/secrets/secret_key": secret: config.secretKey diff --git a/stdlib/aws/ecr/tests/ecr.cue b/stdlib/aws/ecr/tests/ecr.cue index 561f2cd7..d9b99c45 100644 --- a/stdlib/aws/ecr/tests/ecr.cue +++ b/stdlib/aws/ecr/tests/ecr.cue @@ -17,14 +17,14 @@ TestECR: { seed: "" } - repository: string + repository: string if localMode == null { repository: "125635003186.dkr.ecr.\(TestConfig.awsConfig.region).amazonaws.com/dagger-ci" } if localMode != null { - repository: "localhost:4514/dagger-ci" + repository: "localhost:4510/dagger-ci" } - tag: "test-ecr-\(suffix.out)" + tag: "test-ecr-\(suffix.out)" creds: #Credentials & { config: TestConfig.awsConfig diff --git a/stdlib/helpers.bash b/stdlib/helpers.bash index 94707528..436911fe 100644 --- a/stdlib/helpers.bash +++ b/stdlib/helpers.bash @@ -58,9 +58,9 @@ copy_to_sandbox() { cp -a "$source_package" "$target_package" fi } -# Check if there is a local kubernetes cluster. +# Check if there is a localstack instance. # -# This is need to do kubernetes test in the CI. +# This is needed to do docs test in the CI. skip_unless_local_localstack() { if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ]; then echo "Localstack available" From a2ef5dc6e08952f205767465e2262f693d874dd8 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Mon, 23 Aug 2021 14:42:03 +0200 Subject: [PATCH 7/7] Add cleanup and setup of Localstack instances on local mode Signed-off-by: Guillaume de Rouville --- stdlib/helpers.bash | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/stdlib/helpers.bash b/stdlib/helpers.bash index 436911fe..5df6078f 100644 --- a/stdlib/helpers.bash +++ b/stdlib/helpers.bash @@ -1,3 +1,20 @@ +# Instead of setup, this just runs once +setup_file() { + # Cleanup local Localstack instances + if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ] && \ + [ "$GITHUB_ACTIONS" != "true" ]; then + echo "Cleanup local LOCALSTACK" + + # S3 buckets cleanup + aws --endpoint-url=http://localhost:4566 s3 rm s3://dagger-ci 2>/dev/null || true + aws --endpoint-url=http://localhost:4566 s3 mb s3://dagger-ci 2>/dev/null || true + + # ECR repositories cleanup + aws --endpoint-url=http://localhost:4566 ecr delete-repository --repository-name dagger-ci 2>/dev/null || true + aws --endpoint-url=http://localhost:4566 ecr create-repository --repository-name dagger-ci 2>/dev/null || true + fi +} + common_setup() { load 'node_modules/bats-support/load' load 'node_modules/bats-assert/load' @@ -62,7 +79,7 @@ copy_to_sandbox() { # # This is needed to do docs test in the CI. skip_unless_local_localstack() { - if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ]; then + if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ]; then echo "Localstack available" else skip "Localstack not available"