12 Commits

Author SHA1 Message Date
cuddle-please
67d57f2b54 chore(release): 0.1.0
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2025-03-06 02:04:13 +00:00
489b30e028 fix(deps): update aws-sdk-rust monorepo
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2025-03-06 02:02:25 +00:00
430240d751 fix(deps): update rust crate aws-sdk-s3 to v1.76.0
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2025-02-15 01:49:37 +00:00
707d1e2263 chore(deps): update tokio-prost monorepo to v0.13.5
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2025-02-13 01:55:42 +00:00
d0bd8b938f fix(deps): update aws-sdk-rust monorepo
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2025-02-07 01:54:57 +00:00
6f3a5129bd fix(deps): update aws-sdk-rust monorepo
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2025-01-30 02:06:16 +00:00
68c9820aae fix(deps): update rust crate aws-sdk-s3 to v1.70.0
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2025-01-22 01:54:31 +00:00
e44cb6dd23 fix(deps): update aws-sdk-rust monorepo
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2025-01-16 01:55:25 +00:00
6aa881048c fix(deps): update aws-sdk-rust monorepo
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2024-12-29 01:50:13 +00:00
5356303e7b fix(deps): update rust crate serde to v1.0.216
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2024-12-11 05:44:02 +00:00
1bd6fc06c5 chore(deps): update tokio-prost monorepo to v0.13.4
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2024-12-07 01:51:42 +00:00
f342e3c0df fix(deps): update rust crate aws-sdk-s3 to v1.65.0
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2024-12-05 01:52:45 +00:00
2 changed files with 113 additions and 54 deletions

View File

@@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
## [0.1.0] - 2024-11-30
## [0.1.0] - 2025-03-06
### Added
- with updated notmad
@@ -28,6 +28,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- with dagger engine
- do cleanup
- add broker setup
this is mainly to decouple the actual sending of events, from the ingest.
we now ingest the data, and update consumer groups with the new offset. Consumer groups now in the background continously send out data from the update. they tick 1 second between checks, but if something takes long than a second, the next run just continues from where we left off
- update
- add basic streaming data
- add ingest layer
@@ -42,12 +45,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- add base
### Fixed
- *(deps)* update aws-sdk-rust monorepo
- *(deps)* update rust crate aws-sdk-s3 to v1.76.0
- *(deps)* update aws-sdk-rust monorepo
- *(deps)* update aws-sdk-rust monorepo
- *(deps)* update rust crate aws-sdk-s3 to v1.70.0
- *(deps)* update aws-sdk-rust monorepo
- *(deps)* update aws-sdk-rust monorepo
- *(deps)* update rust crate serde to v1.0.216
- *(deps)* update rust crate aws-sdk-s3 to v1.65.0
- *(deps)* update rust crate aws-sdk-s3 to v1.63.0
- *(deps)* update rust crate serde to v1.0.215
- *(deps)* update all dependencies
- offset be inclusive end
### Other
- *(deps)* update tokio-prost monorepo to v0.13.5
- *(deps)* update tokio-prost monorepo to v0.13.4
- *(deps)* update rust crate tracing-subscriber to v0.3.19
- *(deps)* update rust crate tracing to v0.1.41
- into files

151
Cargo.lock generated
View File

@@ -155,9 +155,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "aws-config"
version = "1.5.10"
version = "1.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924"
checksum = "90aff65e86db5fe300752551c1b015ef72b708ac54bded8ef43d0d53cb7cb0b1"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -165,7 +165,7 @@ dependencies = [
"aws-sdk-ssooidc",
"aws-sdk-sts",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http 0.61.1",
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
@@ -197,15 +197,15 @@ dependencies = [
[[package]]
name = "aws-runtime"
version = "1.4.3"
version = "1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468"
checksum = "76dd04d39cc12844c0994f2c9c5a6f5184c22e9188ec1ff723de41910a21dcad"
dependencies = [
"aws-credential-types",
"aws-sigv4",
"aws-smithy-async",
"aws-smithy-eventstream",
"aws-smithy-http",
"aws-smithy-http 0.60.12",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-types",
@@ -223,9 +223,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.63.0"
version = "1.78.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f43850204a109a5eea1ea93951cf0440268cef98b0d27dfef4534949e23735f7"
checksum = "3038614b6cf7dd68d9a7b5b39563d04337eb3678d1d4173e356e927b0356158a"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -233,7 +233,7 @@ dependencies = [
"aws-smithy-async",
"aws-smithy-checksums",
"aws-smithy-eventstream",
"aws-smithy-http",
"aws-smithy-http 0.61.1",
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
@@ -257,14 +257,14 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
version = "1.49.0"
version = "1.61.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09677244a9da92172c8dc60109b4a9658597d4d298b188dd0018b6a66b410ca4"
checksum = "e65ff295979977039a25f5a0bf067a64bc5e6aa38f3cef4037cf42516265553c"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http 0.61.1",
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
@@ -279,14 +279,14 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
version = "1.50.0"
version = "1.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81fea2f3a8bb3bd10932ae7ad59cc59f65f270fc9183a7e91f501dc5efbef7ee"
checksum = "91430a60f754f235688387b75ee798ef00cfd09709a582be2b7525ebb5306d4f"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http 0.61.1",
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
@@ -301,14 +301,14 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.50.0"
version = "1.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5"
checksum = "9276e139d39fff5a0b0c984fc2d30f970f9a202da67234f948fda02e5bea1dbe"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http 0.61.1",
"aws-smithy-json",
"aws-smithy-query",
"aws-smithy-runtime",
@@ -324,13 +324,13 @@ dependencies = [
[[package]]
name = "aws-sigv4"
version = "1.2.5"
version = "1.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1"
checksum = "9bfe75fad52793ce6dec0dc3d4b1f388f038b5eb866c8d4d7f3a8e21b5ea5051"
dependencies = [
"aws-credential-types",
"aws-smithy-eventstream",
"aws-smithy-http",
"aws-smithy-http 0.60.12",
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
@@ -353,9 +353,9 @@ dependencies = [
[[package]]
name = "aws-smithy-async"
version = "1.2.1"
version = "1.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c"
checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e"
dependencies = [
"futures-util",
"pin-project-lite",
@@ -364,15 +364,16 @@ dependencies = [
[[package]]
name = "aws-smithy-checksums"
version = "0.60.13"
version = "0.63.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795"
checksum = "db2dc8d842d872529355c72632de49ef8c5a2949a4472f10e802f28cf925770c"
dependencies = [
"aws-smithy-http",
"aws-smithy-http 0.60.12",
"aws-smithy-types",
"bytes",
"crc32c",
"crc32fast",
"crc64fast-nvme",
"hex",
"http 0.2.12",
"http-body 0.4.6",
@@ -385,9 +386,9 @@ dependencies = [
[[package]]
name = "aws-smithy-eventstream"
version = "0.60.5"
version = "0.60.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90"
checksum = "461e5e02f9864cba17cff30f007c2e37ade94d01e87cdb5204e44a84e6d38c17"
dependencies = [
"aws-smithy-types",
"bytes",
@@ -396,9 +397,29 @@ dependencies = [
[[package]]
name = "aws-smithy-http"
version = "0.60.11"
version = "0.60.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6"
checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
"bytes-utils",
"futures-core",
"http 0.2.12",
"http-body 0.4.6",
"once_cell",
"percent-encoding",
"pin-project-lite",
"pin-utils",
"tracing",
]
[[package]]
name = "aws-smithy-http"
version = "0.61.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6f276f21c7921fe902826618d1423ae5bf74cf8c1b8472aee8434f3dfd31824"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-runtime-api",
@@ -417,9 +438,9 @@ dependencies = [
[[package]]
name = "aws-smithy-json"
version = "0.60.7"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6"
checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422"
dependencies = [
"aws-smithy-types",
]
@@ -436,12 +457,12 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime"
version = "1.7.3"
version = "1.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2"
checksum = "d526a12d9ed61fadefda24abe2e682892ba288c2018bcb38b1b4c111d13f6d92"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http 0.60.12",
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
@@ -480,9 +501,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.2.9"
version = "1.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510"
checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042"
dependencies = [
"base64-simd",
"bytes",
@@ -515,9 +536,9 @@ dependencies = [
[[package]]
name = "aws-types"
version = "1.3.3"
version = "1.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef"
checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
@@ -800,6 +821,21 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crc32c"
version = "0.6.8"
@@ -818,6 +854,15 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crc64fast-nvme"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3"
dependencies = [
"crc",
]
[[package]]
name = "crypto-bigint"
version = "0.4.9"
@@ -1089,9 +1134,9 @@ dependencies = [
[[package]]
name = "fastrand"
version = "2.2.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "ff"
@@ -2233,9 +2278,9 @@ dependencies = [
[[package]]
name = "prost"
version = "0.13.3"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f"
checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
dependencies = [
"bytes",
"prost-derive",
@@ -2243,9 +2288,9 @@ dependencies = [
[[package]]
name = "prost-derive"
version = "0.13.3"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools",
@@ -2256,9 +2301,9 @@ dependencies = [
[[package]]
name = "prost-types"
version = "0.13.3"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670"
checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
dependencies = [
"prost",
]
@@ -2654,18 +2699,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]]
name = "serde"
version = "1.0.215"
version = "1.0.216"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.215"
version = "1.0.216"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
dependencies = [
"proc-macro2",
"quote",
@@ -3289,9 +3334,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.3"
version = "2.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada"
checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
dependencies = [
"form_urlencoded",
"idna",