commit 3162971c8905ecdd060bacaa3ad8ae496b1e0503 Author: kjuulh Date: Thu Feb 26 21:52:50 2026 +0100 feat: add initial Signed-off-by: kjuulh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2551a44 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/target +.env +*.swp +*.swo +*~ +.DS_Store +data/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..0930923 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1746 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "aws-lc-rs" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" + +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "drop-queue" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fd14e42a067434aa0550170cd62d5cca5790e71be1a3b40fa768524e82652b" +dependencies = [ + "anyhow", + "async-trait", + "notmad", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.182" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "notmad" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88f52fa65fdf2dc8bf9e0ba7e95f0966a3d7449f660922cc21d96fe382f5c82e" +dependencies = [ + "anyhow", + "futures", + "futures-util", + "rand", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "sq-cluster" +version = "0.1.0" +dependencies = [ + "anyhow", + "sq-grpc-interface", + "sq-models", + "sq-storage", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "sq-grpc-interface" +version = "0.1.0" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-prost", +] + +[[package]] +name = "sq-models" +version = "0.1.0" +dependencies = [ + "serde", +] + +[[package]] +name = "sq-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "sq-grpc-interface", + "sq-models", + "thiserror", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "sq-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "clap", + "dotenvy", + "drop-queue", + "http", + "notmad", + "prost", + "sq-cluster", + "sq-grpc-interface", + "sq-models", + "sq-sim", + "sq-storage", + "tokio", + "tonic", + "tower", + "tower-http", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "sq-sim" +version = "0.1.0" +dependencies = [ + "anyhow", + "tokio", + "tracing", +] + +[[package]] +name = "sq-storage" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "crc32fast", + "sq-models", + "sq-sim", + "tokio", + "tracing", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "flate2", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", + "webpki-roots", +] + +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zerocopy" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..fe6aa64 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,51 @@ +[workspace] +members = ["crates/*"] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2024" + +[workspace.dependencies] +sq-grpc-interface = { path = "crates/sq-grpc-interface" } +sq-models = { path = "crates/sq-models" } +sq-storage = { path = "crates/sq-storage" } +sq-cluster = { path = "crates/sq-cluster" } +sq-sdk = { path = "crates/sq-sdk" } +sq-sim = { path = "crates/sq-sim" } + +anyhow = { version = "1" } +tokio = { version = "1", features = ["full"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tracing = { version = "0.1", features = ["log"] } +thiserror = "2" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +clap = { version = "4", features = ["derive", "env", "string"] } +dotenvy = { version = "0.15" } +async-trait = "0.1" +futures = "0.3" +notmad = "0.11.0" +drop-queue = { version = "0.0.9", features = ["notmad"] } +tower = { version = "0.5", features = ["tokio", "tracing"] } +http = "1" + +bytes = "1" +prost = "0.14.1" +prost-types = "0.14.1" +tonic = { version = "=0.14.2", features = [ + "gzip", + "tls-aws-lc", + "tls-webpki-roots", +] } +tonic-prost = "=0.14.2" + +uuid = { version = "1", features = ["v4", "v7"] } +tokio-util = "0.7" +tokio-stream = { version = "0.1", features = ["sync"] } +crc32fast = "1" +zstd = "0.13" +object_store = { version = "0.12", features = ["aws"] } +rand = "0.9" +axum = "0.8" +tower-http = { version = "0.6", features = ["trace"] } diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 0000000..168a526 --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,10 @@ +version: v2 +managed: + enabled: true +plugins: + - remote: buf.build/community/neoeinstein-prost:v0.5.0 + out: ./crates/sq-grpc-interface/src/grpc/ + - remote: buf.build/community/neoeinstein-tonic:v0.5.0 + out: ./crates/sq-grpc-interface/src/grpc/ +inputs: + - directory: ./interface/proto diff --git a/buf.yaml b/buf.yaml new file mode 100644 index 0000000..c17ee74 --- /dev/null +++ b/buf.yaml @@ -0,0 +1,4 @@ +version: v2 +modules: + - path: interface/proto + name: buf.build/rawpotion/sq diff --git a/crates/sq-cluster/Cargo.toml b/crates/sq-cluster/Cargo.toml new file mode 100644 index 0000000..c5870a2 --- /dev/null +++ b/crates/sq-cluster/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sq-cluster" +version.workspace = true +edition.workspace = true + +[dependencies] +sq-models = { workspace = true } +sq-storage = { workspace = true } +sq-grpc-interface = { workspace = true } + +anyhow = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tonic = { workspace = true } diff --git a/crates/sq-cluster/src/lib.rs b/crates/sq-cluster/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/crates/sq-grpc-interface/Cargo.toml b/crates/sq-grpc-interface/Cargo.toml new file mode 100644 index 0000000..5fdf97d --- /dev/null +++ b/crates/sq-grpc-interface/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "sq-grpc-interface" +version.workspace = true +edition.workspace = true + +[dependencies] +prost = { workspace = true } +prost-types = { workspace = true } +tonic = { workspace = true } +tonic-prost = { workspace = true } diff --git a/crates/sq-grpc-interface/src/grpc/sq/v1/sq.v1.rs b/crates/sq-grpc-interface/src/grpc/sq/v1/sq.v1.rs new file mode 100644 index 0000000..192e234 --- /dev/null +++ b/crates/sq-grpc-interface/src/grpc/sq/v1/sq.v1.rs @@ -0,0 +1,2 @@ +// This file will be generated by `buf generate`. +// Placeholder for initial workspace compilation. diff --git a/crates/sq-grpc-interface/src/lib.rs b/crates/sq-grpc-interface/src/lib.rs new file mode 100644 index 0000000..d0f57d7 --- /dev/null +++ b/crates/sq-grpc-interface/src/lib.rs @@ -0,0 +1,6 @@ +#[path = "./grpc/sq/v1/sq.v1.rs"] +#[allow(clippy::all)] +pub mod grpc; + +#[allow(unused_imports)] +pub use grpc::*; diff --git a/crates/sq-models/Cargo.toml b/crates/sq-models/Cargo.toml new file mode 100644 index 0000000..ed328b3 --- /dev/null +++ b/crates/sq-models/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "sq-models" +version.workspace = true +edition.workspace = true + +[dependencies] +serde = { workspace = true } diff --git a/crates/sq-models/src/lib.rs b/crates/sq-models/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/crates/sq-sdk/Cargo.toml b/crates/sq-sdk/Cargo.toml new file mode 100644 index 0000000..c072130 --- /dev/null +++ b/crates/sq-sdk/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sq-sdk" +version.workspace = true +edition.workspace = true + +[dependencies] +sq-grpc-interface = { workspace = true } +sq-models = { workspace = true } + +anyhow = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tonic = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/sq-sdk/src/lib.rs b/crates/sq-sdk/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/crates/sq-server/Cargo.toml b/crates/sq-server/Cargo.toml new file mode 100644 index 0000000..97f895c --- /dev/null +++ b/crates/sq-server/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "sq-server" +version.workspace = true +edition.workspace = true + +[dependencies] +sq-grpc-interface = { workspace = true } +sq-models = { workspace = true } +sq-storage = { workspace = true } +sq-cluster = { workspace = true } +sq-sim = { workspace = true } + +anyhow = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +clap = { workspace = true } +dotenvy = { workspace = true } +notmad = { workspace = true } +drop-queue = { workspace = true } +tonic = { workspace = true } +prost = { workspace = true } +axum = { workspace = true } +tower = { workspace = true } +tower-http = { workspace = true } +http = { workspace = true } diff --git a/crates/sq-server/src/main.rs b/crates/sq-server/src/main.rs new file mode 100644 index 0000000..b94ed05 --- /dev/null +++ b/crates/sq-server/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("sq-server"); +} diff --git a/crates/sq-sim/Cargo.toml b/crates/sq-sim/Cargo.toml new file mode 100644 index 0000000..bc68009 --- /dev/null +++ b/crates/sq-sim/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "sq-sim" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["full", "test-util"] } diff --git a/crates/sq-sim/src/lib.rs b/crates/sq-sim/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/crates/sq-storage/Cargo.toml b/crates/sq-storage/Cargo.toml new file mode 100644 index 0000000..6780171 --- /dev/null +++ b/crates/sq-storage/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sq-storage" +version.workspace = true +edition.workspace = true + +[dependencies] +sq-models = { workspace = true } +sq-sim = { workspace = true } + +anyhow = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +crc32fast = { workspace = true } +bytes = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["full", "test-util"] } diff --git a/crates/sq-storage/src/lib.rs b/crates/sq-storage/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/interface/proto/sq/v1/health.proto b/interface/proto/sq/v1/health.proto new file mode 100644 index 0000000..14b3c03 --- /dev/null +++ b/interface/proto/sq/v1/health.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; +package sq.v1; + +service StatusService { + rpc Status(GetStatusRequest) returns (GetStatusResponse) {} +} + +message GetStatusRequest {} + +message GetStatusResponse { + string node_id = 1; + ClusterStatus cluster = 2; +} + +message ClusterStatus { + repeated NodeInfo nodes = 1; +} + +message NodeInfo { + string node_id = 1; + string address = 2; + string status = 3; +} diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..17cdc67 --- /dev/null +++ b/mise.toml @@ -0,0 +1,31 @@ +[tasks.build] +run = "cargo build --workspace" +description = "Build all crates" + +[tasks.test] +run = "cargo nextest run --workspace" +description = "Run all tests" + +[tasks.check] +run = "cargo check --workspace" +description = "Check all crates" + +[tasks.clippy] +run = "cargo clippy --workspace" +description = "Run clippy on all crates" + +[tasks."generate:proto"] +run = "buf generate" +description = "Generate protobuf code" + +[tasks."local:up"] +run = "docker compose -f templates/docker-compose.yaml up -d" +description = "Start local dev services" + +[tasks."local:down"] +run = "docker compose -f templates/docker-compose.yaml down" +description = "Stop local dev services" + +[tasks.develop] +run = "cargo run -p sq-server -- serve" +description = "Run SQ server in development mode" diff --git a/todos/SQ-000-workspace-skeleton.md b/todos/SQ-000-workspace-skeleton.md new file mode 100644 index 0000000..c42d0f7 --- /dev/null +++ b/todos/SQ-000-workspace-skeleton.md @@ -0,0 +1,46 @@ +# SQ-000: Workspace Skeleton + +**Status:** `[x] DONE` +**Blocked by:** None +**Priority:** Critical (everything depends on this) + +## Description + +Bootstrap the entire SQ workspace: Cargo workspace, all crate stubs, buf protobuf config, dev tooling, and gitignore. Every crate should compile with an empty `lib.rs` or `main.rs`. + +## Files to Create + +### Root +- `Cargo.toml` - workspace root with all members and shared dependencies +- `Cargo.lock` - (generated) +- `buf.yaml` - buf module definition (`buf.build/rawpotion/sq`) +- `buf.gen.yaml` - prost + tonic codegen config +- `mise.toml` - dev tasks (build, test, generate:proto, local:up/down) +- `.gitignore` - target/, .env, etc. +- `.env` - local dev environment vars + +### Proto +- `interface/proto/sq/v1/health.proto` - minimal Status service + +### Crate stubs +- `crates/sq-grpc-interface/Cargo.toml` + `src/lib.rs` +- `crates/sq-models/Cargo.toml` + `src/lib.rs` +- `crates/sq-storage/Cargo.toml` + `src/lib.rs` +- `crates/sq-cluster/Cargo.toml` + `src/lib.rs` +- `crates/sq-server/Cargo.toml` + `src/main.rs` +- `crates/sq-sdk/Cargo.toml` + `src/lib.rs` +- `crates/sq-sim/Cargo.toml` + `src/lib.rs` + +## Acceptance Criteria + +- [x] `cargo check --workspace` passes +- [x] `cargo test --workspace` passes (no tests yet, but no errors) +- [x] All 7 crates are listed in workspace members +- [x] Workspace dependencies match forest/fungus conventions (tonic 0.14.2, prost 0.14.1, etc.) + +## Notes + +- Follow forest's Cargo.toml pattern exactly: `/home/kjuulh/git/src.rawpotion.io/rawpotion/forest/Cargo.toml` +- Follow forest's buf.gen.yaml: `/home/kjuulh/git/src.rawpotion.io/rawpotion/forest/buf.gen.yaml` +- Use Rust edition 2024 +- sq-server is the only binary crate; all others are libraries diff --git a/todos/SQ-001-domain-types.md b/todos/SQ-001-domain-types.md new file mode 100644 index 0000000..7c032a6 --- /dev/null +++ b/todos/SQ-001-domain-types.md @@ -0,0 +1,54 @@ +# SQ-001: Domain Types + +**Status:** `[ ] TODO` +**Blocked by:** SQ-000 +**Priority:** High + +## Description + +Define the core domain types in `sq-models`: Message, Header, TopicName, Offset, and configuration types. These are the foundational types used throughout the entire system. + +## Files to Create/Modify + +- `crates/sq-models/src/lib.rs` - re-exports +- `crates/sq-models/src/message.rs` - Message, Header, TopicName, Offset types +- `crates/sq-models/src/config.rs` - TopicConfig, WalConfig + +## Key Types + +```rust +pub struct Message { + pub offset: u64, + pub topic: TopicName, + pub key: Option>, + pub value: Vec, + pub headers: Vec
, + pub timestamp_ms: u64, +} + +pub struct Header { + pub key: String, + pub value: Vec, +} + +pub struct TopicName(pub String); + +pub struct WalConfig { + pub max_segment_bytes: u64, // default 64MB + pub max_segment_age_secs: u64, // default 60s + pub data_dir: PathBuf, +} + +pub struct TopicConfig { + pub name: TopicName, + pub partitions: u32, // default 1 + pub replication_factor: u32, // default 3 +} +``` + +## Acceptance Criteria + +- [ ] All types compile and are public +- [ ] `Message` implements `Clone`, `Debug`, `PartialEq` +- [ ] Unit test: construct a Message, verify all fields +- [ ] Property test (optional): arbitrary Message roundtrip through Debug/PartialEq diff --git a/todos/SQ-002-wal-record-encoding.md b/todos/SQ-002-wal-record-encoding.md new file mode 100644 index 0000000..f9deecd --- /dev/null +++ b/todos/SQ-002-wal-record-encoding.md @@ -0,0 +1,49 @@ +# SQ-002: WAL Record Encoding/Decoding + +**Status:** `[ ] TODO` +**Blocked by:** SQ-001 +**Priority:** High + +## Description + +Implement binary encoding and decoding of individual WAL records, independent of file I/O. Each record is CRC32-protected for corruption detection. + +## Files to Create/Modify + +- `crates/sq-storage/src/wal/mod.rs` - module declaration +- `crates/sq-storage/src/wal/record.rs` - encode_record, decode_record, CRC validation +- `crates/sq-storage/src/lib.rs` - re-export wal module + +## Record Binary Format + +``` +[crc32: u32] - CRC32 over everything after this field +[length: u32] - total byte length of record (excluding crc32 and length) +[offset: u64] - monotonic offset +[timestamp_ms: u64] - wall clock millis +[key_len: u32] - 0 = no key +[key: [u8; key_len]] +[value_len: u32] +[value: [u8; value_len]] +[headers_count: u16] +[for each header:] + [hdr_key_len: u16] + [hdr_key: [u8; hdr_key_len]] + [hdr_val_len: u32] + [hdr_val: [u8; hdr_val_len]] +``` + +## Acceptance Criteria + +- [ ] `encode_record(&Message) -> Vec` produces correct binary +- [ ] `decode_record(&[u8]) -> Result<(Message, usize)>` parses correctly (returns bytes consumed) +- [ ] Roundtrip: encode then decode, verify equality +- [ ] Corruption test: flip a byte, decode returns CRC error +- [ ] Edge cases: empty value, empty key, no headers, many headers, max-size value +- [ ] Uses `crc32fast` crate for CRC computation + +## Notes + +- Use little-endian byte order throughout +- The `length` field allows skipping records without full parsing +- CRC is over the bytes AFTER the CRC field itself diff --git a/todos/SQ-003-simulation-io-traits.md b/todos/SQ-003-simulation-io-traits.md new file mode 100644 index 0000000..5c03227 --- /dev/null +++ b/todos/SQ-003-simulation-io-traits.md @@ -0,0 +1,66 @@ +# SQ-003: Simulation I/O Traits + +**Status:** `[ ] TODO` +**Blocked by:** SQ-000 +**Priority:** High + +## Description + +Define the trait abstractions for Clock and FileSystem that allow swapping real I/O for deterministic simulated I/O. This is the foundation of TigerBeetle-style testing. + +## Files to Create/Modify + +- `crates/sq-sim/src/lib.rs` - re-exports +- `crates/sq-sim/src/clock.rs` - Clock trait + RealClock + SimClock +- `crates/sq-sim/src/fs.rs` - FileSystem trait + FileHandle trait + RealFileSystem + InMemoryFileSystem + +## Key Traits + +```rust +pub trait Clock: Send + Sync { + fn now(&self) -> std::time::Instant; + async fn sleep(&self, duration: Duration); +} + +pub trait FileSystem: Send + Sync { + async fn create_dir_all(&self, path: &Path) -> Result<()>; + async fn open_read(&self, path: &Path) -> Result>; + async fn open_write(&self, path: &Path) -> Result>; + async fn open_append(&self, path: &Path) -> Result>; + async fn remove_file(&self, path: &Path) -> Result<()>; + async fn list_dir(&self, path: &Path) -> Result>; + async fn exists(&self, path: &Path) -> bool; +} + +pub trait FileHandle: Send + Sync { + async fn write_all(&mut self, buf: &[u8]) -> Result; + async fn read_exact(&mut self, buf: &mut [u8]) -> Result; + async fn read_to_end(&mut self, buf: &mut Vec) -> Result; + async fn fsync(&mut self) -> Result<()>; + fn position(&self) -> u64; + async fn seek(&mut self, pos: u64) -> Result<()>; +} +``` + +## Fault Injection (InMemoryFileSystem) + +```rust +impl InMemoryFileSystem { + pub fn fail_next_fsync(&self, error: io::Error); + pub fn simulate_disk_full(&self); + pub fn corrupt_bytes(&self, path: &Path, offset: u64, len: usize); + pub fn clear_faults(&self); +} +``` + +## Acceptance Criteria + +- [ ] InMemoryFileSystem: write, read back, verify content +- [ ] InMemoryFileSystem: create_dir_all, list_dir +- [ ] InMemoryFileSystem: fsync succeeds normally +- [ ] InMemoryFileSystem: fail_next_fsync causes next fsync to error +- [ ] InMemoryFileSystem: simulate_disk_full causes writes to fail +- [ ] SimClock: starts at time 0, advance(Duration) changes now() +- [ ] SimClock: sleep returns immediately when time is advanced +- [ ] RealClock: delegates to std::time +- [ ] RealFileSystem: delegates to tokio::fs (basic smoke test) diff --git a/todos/SQ-004-wal-segment-writer.md b/todos/SQ-004-wal-segment-writer.md new file mode 100644 index 0000000..3dab95e --- /dev/null +++ b/todos/SQ-004-wal-segment-writer.md @@ -0,0 +1,63 @@ +# SQ-004: WAL Segment Writer + +**Status:** `[ ] TODO` +**Blocked by:** SQ-002, SQ-003 +**Priority:** High + +## Description + +Implement the WAL segment writer that appends records to segment files with fsync for durability. Handles segment rotation when size or time thresholds are exceeded. + +## Files to Create/Modify + +- `crates/sq-storage/src/wal/writer.rs` - WalWriter with append + fsync + rotation +- `crates/sq-storage/src/wal/segment.rs` - segment header encoding/decoding + +## Segment Header Format (32 bytes) + +``` +[magic: [u8; 4]] = b"SQWL" +[version: u16] = 1 +[topic_len: u16] +[topic: [u8; 20]] (padded/truncated) +[partition: u32] +``` + +## WalWriter API + +```rust +pub struct WalWriter { + fs: Arc, + config: WalConfig, + topic: TopicName, + partition: u32, + active_segment: Option>, + segment_base_offset: u64, + segment_position: u64, + next_offset: u64, + segment_opened_at: Instant, +} + +impl WalWriter { + pub async fn new(fs: Arc, config: WalConfig, topic: TopicName, partition: u32) -> Result; + pub async fn append(&mut self, key: Option<&[u8]>, value: &[u8], headers: &[Header], timestamp_ms: u64) -> Result; // returns offset + pub async fn close_active_segment(&mut self) -> Result>; + pub fn next_offset(&self) -> u64; +} +``` + +## Acceptance Criteria (using InMemoryFileSystem) + +- [ ] Write 1 message, verify segment file exists with correct header + record +- [ ] Write 100 messages, verify all offsets are monotonically increasing (0, 1, 2, ...) +- [ ] Segment rotation: write until size > max_segment_bytes, verify new segment created +- [ ] Segment rotation: advance clock past max_segment_age, verify rotation on next write +- [ ] fsync failure: set fault on InMemoryFS, verify append() returns error +- [ ] fsync failure: offset is NOT advanced (can retry the write) +- [ ] Segment directory structure: `{data_dir}/{topic}/{partition}/{base_offset}.wal` + +## Notes + +- sq-storage depends on sq-sim for the FileSystem trait +- Writer must call fsync after every append (or batch of appends) +- `ClosedSegment` contains the path and offset range of the completed segment diff --git a/todos/SQ-005-wal-segment-reader.md b/todos/SQ-005-wal-segment-reader.md new file mode 100644 index 0000000..9d23c66 --- /dev/null +++ b/todos/SQ-005-wal-segment-reader.md @@ -0,0 +1,37 @@ +# SQ-005: WAL Segment Reader + +**Status:** `[ ] TODO` +**Blocked by:** SQ-004 +**Priority:** High + +## Description + +Implement the WAL segment reader that reads messages from segment files, supporting seek-to-offset and forward scanning. + +## Files to Create/Modify + +- `crates/sq-storage/src/wal/reader.rs` - WalReader with open, read_from, iterator + +## WalReader API + +```rust +pub struct WalReader { + fs: Arc, +} + +impl WalReader { + pub fn new(fs: Arc) -> Self; + pub async fn read_segment(&self, path: &Path) -> Result>; + pub async fn read_from_offset(&self, path: &Path, offset: u64) -> Result>; + pub async fn read_segment_header(&self, path: &Path) -> Result; +} +``` + +## Acceptance Criteria (using InMemoryFileSystem) + +- [ ] Write N messages with writer, read all back with reader, verify equality +- [ ] Read from a specific offset in the middle of a segment +- [ ] Corrupted record mid-segment: reader returns error for that record, can report partial results +- [ ] Empty segment (header only): reader yields zero messages +- [ ] Invalid magic bytes: reader returns descriptive error +- [ ] Truncated record at end of segment (partial write): reader stops cleanly at last complete record diff --git a/todos/SQ-006-sparse-offset-index.md b/todos/SQ-006-sparse-offset-index.md new file mode 100644 index 0000000..0a96b44 --- /dev/null +++ b/todos/SQ-006-sparse-offset-index.md @@ -0,0 +1,44 @@ +# SQ-006: Sparse Offset Index + +**Status:** `[ ] TODO` +**Blocked by:** SQ-005 +**Priority:** Medium + +## Description + +In-memory sparse offset index that maps offsets to segment file locations for fast consumer seeks. + +## Files to Create/Modify + +- `crates/sq-storage/src/index.rs` - OffsetIndex + +## OffsetIndex API + +```rust +pub struct OffsetIndex { + // Per topic-partition: sorted vec of (offset, segment_path, byte_position) + entries: BTreeMap<(TopicName, u32), Vec>, + sample_interval: u64, // e.g. every 1000th offset +} + +pub struct IndexEntry { + pub offset: u64, + pub segment_path: PathBuf, + pub byte_position: u64, +} + +impl OffsetIndex { + pub fn new(sample_interval: u64) -> Self; + pub fn add_entry(&mut self, topic: &TopicName, partition: u32, entry: IndexEntry); + pub fn lookup(&self, topic: &TopicName, partition: u32, offset: u64) -> Option<&IndexEntry>; + pub fn build_from_segments(fs: &F, segments: &[PathBuf]) -> Result; +} +``` + +## Acceptance Criteria + +- [ ] Build index from a set of written segments, look up first offset -> correct segment +- [ ] Look up offset in the middle -> returns nearest lower indexed entry +- [ ] Look up offset beyond all segments -> returns None +- [ ] Multiple topic-partitions are isolated +- [ ] Sample interval works: only every Nth offset is indexed diff --git a/todos/SQ-007-storage-engine-facade.md b/todos/SQ-007-storage-engine-facade.md new file mode 100644 index 0000000..67d34a5 --- /dev/null +++ b/todos/SQ-007-storage-engine-facade.md @@ -0,0 +1,42 @@ +# SQ-007: Storage Engine Facade + +**Status:** `[ ] TODO` +**Blocked by:** SQ-006 +**Priority:** High + +## Description + +Unified read/write interface wrapping WAL writer + reader + offset index. This is the single entry point for all storage operations used by the server. + +## Files to Create/Modify + +- `crates/sq-storage/src/engine.rs` - StorageEngine + +## StorageEngine API + +```rust +pub struct StorageEngine { + fs: Arc, + clock: Arc, + config: WalConfig, + writers: HashMap<(String, u32), WalWriter>, + index: OffsetIndex, +} + +impl StorageEngine { + pub async fn new(fs: Arc, clock: Arc, config: WalConfig) -> Result; + pub async fn append(&mut self, topic: &str, partition: u32, key: Option<&[u8]>, value: &[u8], headers: &[Header]) -> Result; + pub async fn read(&self, topic: &str, partition: u32, from_offset: u64, limit: usize) -> Result>; + pub async fn recover(&mut self) -> Result<()>; // Rebuild state from existing WAL files on startup + pub fn closed_segments(&self) -> Vec; // For S3 shipper +} +``` + +## Acceptance Criteria + +- [ ] Write 1000 messages, read from offset 0, verify all present +- [ ] Write, read from offset 500, verify correct slice returned +- [ ] Read with limit, verify at most N messages returned +- [ ] Write to multiple topics/partitions, verify isolation (no cross-contamination) +- [ ] Recovery: write messages, drop engine, create new engine, call recover(), read all messages back +- [ ] Segment rotation happens transparently during append diff --git a/todos/SQ-008-protobuf-api-definitions.md b/todos/SQ-008-protobuf-api-definitions.md new file mode 100644 index 0000000..8909ae8 --- /dev/null +++ b/todos/SQ-008-protobuf-api-definitions.md @@ -0,0 +1,42 @@ +# SQ-008: Protobuf API Definitions + +**Status:** `[ ] TODO` +**Blocked by:** SQ-000 +**Priority:** High + +## Description + +Define all protobuf service definitions and generate the Rust gRPC code via buf. + +## Files to Create/Modify + +- `interface/proto/sq/v1/data_plane.proto` - Publish, Subscribe, Ack RPCs +- `interface/proto/sq/v1/control_plane.proto` - CreateTopic, DeleteTopic, ListTopics, DescribeTopic, CreateConsumerGroup +- `interface/proto/sq/v1/health.proto` - Status service (may already exist from Phase 0) +- `interface/proto/sq/v1/cluster.proto` - ReplicateEntries, Join, Heartbeat, FetchSegment (internal) +- `crates/sq-grpc-interface/src/lib.rs` - updated module re-exports +- Regenerated code in `crates/sq-grpc-interface/src/grpc/` + +## Key Service Definitions + +### DataPlaneService +- `Publish(PublishRequest) -> PublishResponse` +- `Subscribe(SubscribeRequest) -> stream SubscribeResponse` +- `Ack(AckRequest) -> AckResponse` + +### ControlPlaneService +- `CreateTopic`, `DeleteTopic`, `ListTopics`, `DescribeTopic`, `CreateConsumerGroup` + +### ClusterService (internal) +- `ReplicateEntries`, `Join`, `Heartbeat`, `FetchSegment` (streaming) + +### StatusService +- `Status(GetStatusRequest) -> GetStatusResponse` + +## Acceptance Criteria + +- [ ] `buf lint` passes on all proto files +- [ ] `buf generate` produces code in sq-grpc-interface +- [ ] `cargo check -p sq-grpc-interface` passes +- [ ] All service traits are generated (DataPlaneService, ControlPlaneService, ClusterService, StatusService) +- [ ] AckMode enum has ALL, LOCAL, NONE variants diff --git a/todos/SQ-009-server-skeleton.md b/todos/SQ-009-server-skeleton.md new file mode 100644 index 0000000..0c66141 --- /dev/null +++ b/todos/SQ-009-server-skeleton.md @@ -0,0 +1,43 @@ +# SQ-009: Server Skeleton + +**Status:** `[ ] TODO` +**Blocked by:** SQ-008 +**Priority:** High + +## Description + +Running `sq-server` binary with CLI, notmad lifecycle management, health gRPC endpoint, and Axum HTTP health endpoint. Follows the fungus-server pattern exactly. + +## Files to Create/Modify + +- `crates/sq-server/src/main.rs` - entry point with logging setup +- `crates/sq-server/src/cli.rs` - clap Command enum with Serve subcommand +- `crates/sq-server/src/cli/serve.rs` - ServeCommand with notmad builder +- `crates/sq-server/src/state.rs` - State struct with Config +- `crates/sq-server/src/grpc/mod.rs` - GrpcServer as notmad::Component +- `crates/sq-server/src/grpc/health.rs` - StatusService impl +- `crates/sq-server/src/grpc/error.rs` - gRPC error mapping +- `crates/sq-server/src/servehttp.rs` - Axum health routes + +## Configuration + +``` +SQ_HOST=127.0.0.1:6060 # gRPC listen address +SQ_HTTP_HOST=127.0.0.1:6062 # HTTP listen address +SQ_DATA_DIR=./data # WAL storage directory +SQ_NODE_ID=node-1 # Unique node identifier +LOG_LEVEL=pretty # pretty|json|short +``` + +## Acceptance Criteria + +- [ ] `cargo run -p sq-server -- serve` starts and listens on configured ports +- [ ] gRPC Status RPC returns node_id +- [ ] HTTP GET / returns 200 with health message +- [ ] Graceful shutdown on SIGINT via notmad cancellation token +- [ ] Logging setup matches forest/fungus pattern (pretty/json/short) + +## Reference Files + +- `/home/kjuulh/git/src.rawpotion.io/rawpotion/fungus/crates/fungus-server/src/cli/serve.rs` +- `/home/kjuulh/git/src.rawpotion.io/rawpotion/fungus/crates/fungus-server/src/state.rs` diff --git a/todos/SQ-010-publish-endpoint.md b/todos/SQ-010-publish-endpoint.md new file mode 100644 index 0000000..49c844b --- /dev/null +++ b/todos/SQ-010-publish-endpoint.md @@ -0,0 +1,32 @@ +# SQ-010: Publish Endpoint (Single Node) + +**Status:** `[ ] TODO` +**Blocked by:** SQ-007, SQ-009 +**Priority:** High + +## Description + +Implement the Publish gRPC RPC. Clients send messages which are durably written to the local WAL via StorageEngine. + +## Files to Create/Modify + +- `crates/sq-server/src/grpc/data_plane.rs` - DataPlaneService Publish impl +- `crates/sq-server/src/state.rs` - add StorageEngine to State + +## Behavior + +1. Receive PublishRequest with batch of messages +2. For each message: call StorageEngine::append +3. Ack mode handling: + - ACK_MODE_ALL: return after fsync (default single-node behavior) + - ACK_MODE_LOCAL: same as ALL for single node + - ACK_MODE_NONE: return immediately, write async +4. Return PublishResponse with offset assignments + +## Acceptance Criteria + +- [ ] Publish 1 message, verify WAL file exists and contains the message +- [ ] Publish batch of 100 messages, verify all acked with sequential offsets +- [ ] ACK_MODE_NONE: response is immediate (no waiting for fsync) +- [ ] Invalid request (empty topic): returns InvalidArgument gRPC status +- [ ] Auto-create topic on first publish (if topic doesn't exist) diff --git a/todos/SQ-011-subscribe-endpoint.md b/todos/SQ-011-subscribe-endpoint.md new file mode 100644 index 0000000..ffcb649 --- /dev/null +++ b/todos/SQ-011-subscribe-endpoint.md @@ -0,0 +1,29 @@ +# SQ-011: Subscribe Endpoint (Single Node) + +**Status:** `[ ] TODO` +**Blocked by:** SQ-010 +**Priority:** High + +## Description + +Implement the Subscribe gRPC RPC with server-streaming. Server pushes messages to the client as they become available. + +## Files to Create/Modify + +- `crates/sq-server/src/grpc/data_plane.rs` - DataPlaneService Subscribe impl + +## Behavior + +1. Client sends SubscribeRequest with topic, partition, optional start_offset +2. Server reads from StorageEngine starting at the given offset +3. Server streams batches of ConsumedMessage to the client +4. When caught up, server polls for new messages at a configurable interval (e.g. 100ms) +5. Stream continues until client disconnects or cancellation + +## Acceptance Criteria + +- [ ] Publish 10 messages, then subscribe from offset 0, receive all 10 +- [ ] Subscribe from offset 5, receive messages 5-9 only +- [ ] Subscribe to empty topic from offset 0, then publish, receive new messages +- [ ] Client disconnect: server-side stream cleans up without error +- [ ] Subscribe to nonexistent topic: returns NotFound gRPC status diff --git a/todos/SQ-012-consumer-groups.md b/todos/SQ-012-consumer-groups.md new file mode 100644 index 0000000..e13fb88 --- /dev/null +++ b/todos/SQ-012-consumer-groups.md @@ -0,0 +1,40 @@ +# SQ-012: Consumer Groups & Offset Tracking + +**Status:** `[ ] TODO` +**Blocked by:** SQ-011 +**Priority:** Medium + +## Description + +Consumer group offset management: store committed offsets, use them as default start position for Subscribe, and implement the Ack RPC. + +## Files to Create/Modify + +- `crates/sq-storage/src/consumer_offsets.rs` - in-memory map with file persistence +- `crates/sq-server/src/grpc/data_plane.rs` - Ack RPC impl; update Subscribe to use committed offset +- `crates/sq-storage/src/engine.rs` - add consumer offset methods + +## ConsumerOffsets API + +```rust +pub struct ConsumerOffsets { + offsets: HashMap<(String, String, u32), u64>, // (group, topic, partition) -> offset + persist_path: PathBuf, +} + +impl ConsumerOffsets { + pub fn commit(&mut self, group: &str, topic: &str, partition: u32, offset: u64) -> Result<()>; + pub fn get_committed(&self, group: &str, topic: &str, partition: u32) -> Option; + pub async fn persist(&self) -> Result<()>; + pub async fn load(path: &Path) -> Result; +} +``` + +## Acceptance Criteria + +- [ ] Commit offset, query it back, verify correct +- [ ] Commit offset, persist to file, load from file, verify preserved +- [ ] Ack RPC: commit offset via gRPC, verify stored +- [ ] Subscribe without start_offset uses committed offset for the consumer group +- [ ] Subscribe with explicit start_offset overrides committed offset +- [ ] Two consumers in same group: both see same committed offset diff --git a/todos/SQ-013-topic-management.md b/todos/SQ-013-topic-management.md new file mode 100644 index 0000000..a4fb11c --- /dev/null +++ b/todos/SQ-013-topic-management.md @@ -0,0 +1,25 @@ +# SQ-013: Control Plane - Topic Management + +**Status:** `[ ] TODO` +**Blocked by:** SQ-012 +**Priority:** Medium + +## Description + +Implement the ControlPlane gRPC service for topic CRUD operations. + +## Files to Create/Modify + +- `crates/sq-server/src/grpc/control_plane.rs` - ControlPlaneService impl +- `crates/sq-storage/src/topic_metadata.rs` - topic registry (file-backed) +- `crates/sq-server/src/grpc/mod.rs` - register ControlPlane service + +## Acceptance Criteria + +- [ ] CreateTopic: creates topic with specified partitions and replication factor +- [ ] CreateTopic: duplicate name returns AlreadyExists +- [ ] ListTopics: returns all created topics +- [ ] DescribeTopic: returns partition info with earliest/latest offsets +- [ ] DeleteTopic: removes topic from registry +- [ ] Publish to deleted topic: returns NotFound +- [ ] Topic metadata persists across server restarts diff --git a/todos/SQ-014-sdk-producer.md b/todos/SQ-014-sdk-producer.md new file mode 100644 index 0000000..0de33cf --- /dev/null +++ b/todos/SQ-014-sdk-producer.md @@ -0,0 +1,42 @@ +# SQ-014: SDK Producer + +**Status:** `[ ] TODO` +**Blocked by:** SQ-010 +**Priority:** Medium + +## Description + +Ergonomic Rust producer client in sq-sdk. Handles connection management, batching, and retry logic. + +## Files to Create/Modify + +- `crates/sq-sdk/src/lib.rs` - re-exports +- `crates/sq-sdk/src/connection.rs` - gRPC channel management +- `crates/sq-sdk/src/producer.rs` - Producer with batching, linger timer, retry +- `crates/sq-sdk/src/error.rs` - SqError type + +## Producer API + +```rust +pub struct ProducerConfig { + pub server_addresses: Vec, + pub default_ack_mode: AckMode, + pub max_retries: u32, + pub retry_backoff_ms: u64, +} + +pub struct Producer { /* ... */ } + +impl Producer { + pub async fn connect(config: ProducerConfig) -> Result; + pub async fn send(&self, topic: &str, key: Option<&[u8]>, value: &[u8]) -> Result; + pub async fn send_batch(&self, messages: Vec) -> Result, SqError>; +} +``` + +## Acceptance Criteria + +- [ ] Producer connects to running server, sends message, gets offset back +- [ ] Send batch: all messages get sequential offsets +- [ ] Connection failure: returns appropriate error +- [ ] Multiple server addresses: round-robin or failover diff --git a/todos/SQ-015-sdk-consumer.md b/todos/SQ-015-sdk-consumer.md new file mode 100644 index 0000000..3529cd8 --- /dev/null +++ b/todos/SQ-015-sdk-consumer.md @@ -0,0 +1,42 @@ +# SQ-015: SDK Consumer + +**Status:** `[ ] TODO` +**Blocked by:** SQ-014, SQ-012 +**Priority:** Medium + +## Description + +Ergonomic Rust consumer client in sq-sdk. Wraps the server-streaming Subscribe RPC with poll-based interface and auto-commit support. + +## Files to Create/Modify + +- `crates/sq-sdk/src/consumer.rs` - Consumer with poll loop and auto-commit + +## Consumer API + +```rust +pub struct ConsumerConfig { + pub server_addresses: Vec, + pub consumer_group: String, + pub topics: Vec, + pub auto_commit: bool, + pub auto_commit_interval_ms: u64, + pub max_poll_records: u32, +} + +pub struct Consumer { /* ... */ } + +impl Consumer { + pub async fn connect(config: ConsumerConfig) -> Result; + pub async fn poll(&mut self) -> Result, SqError>; + pub async fn commit(&self, topic: &str, partition: u32, offset: u64) -> Result<(), SqError>; +} +``` + +## Acceptance Criteria + +- [ ] End-to-end: produce 100 messages with Producer, consume all with Consumer +- [ ] Auto-commit: consumed offsets are committed after interval +- [ ] Manual commit: explicit commit stores offset +- [ ] Poll returns empty vec when no new messages (non-blocking) +- [ ] Consumer group: two consumers resume from committed offset diff --git a/todos/SQ-016-object-store-shipping.md b/todos/SQ-016-object-store-shipping.md new file mode 100644 index 0000000..3a855b3 --- /dev/null +++ b/todos/SQ-016-object-store-shipping.md @@ -0,0 +1,45 @@ +# SQ-016: Object Store Shipping + +**Status:** `[ ] TODO` +**Blocked by:** SQ-007 +**Priority:** Medium + +## Description + +Background process that ships closed WAL segments to S3-compatible object storage for long-term durability. + +## Files to Create/Modify + +- `crates/sq-storage/src/object_store/mod.rs` - ObjectStore trait + S3 impl + Noop impl +- `crates/sq-storage/src/object_store/shipper.rs` - SegmentShipper as notmad::Component +- `crates/sq-storage/src/object_store/layout.rs` - S3 key naming convention + +## ObjectStore Trait + +```rust +pub trait ObjectStore: Send + Sync { + async fn put(&self, key: &str, data: Vec) -> Result<()>; + async fn get(&self, key: &str) -> Result>; + async fn list(&self, prefix: &str) -> Result>; + async fn delete(&self, key: &str) -> Result<()>; +} +``` + +## S3 Key Layout + +`{cluster_id}/{topic}/{partition}/{base_offset}-{end_offset}.sqseg` + +## Acceptance Criteria + +- [ ] Closed segment is detected and uploaded to object store +- [ ] S3 key matches expected layout +- [ ] Noop object store works for testing (stores in memory) +- [ ] Upload failure: segment stays local, retried on next cycle +- [ ] Successful upload is recorded (segment marked as "shipped") +- [ ] Uses zstd compression before upload + +## Notes + +- Uses `object_store` crate with AWS S3 features (same as nostore) +- Shipper runs as a notmad::Component in the background +- Poll interval: every 5 seconds check for closed segments diff --git a/todos/SQ-017-wal-trimming.md b/todos/SQ-017-wal-trimming.md new file mode 100644 index 0000000..030c7f2 --- /dev/null +++ b/todos/SQ-017-wal-trimming.md @@ -0,0 +1,27 @@ +# SQ-017: WAL Trimming + +**Status:** `[ ] TODO` +**Blocked by:** SQ-016 +**Priority:** Medium + +## Description + +Garbage collect local WAL segments after they have been confirmed in object storage. + +## Files to Create/Modify + +- `crates/sq-storage/src/wal/trimmer.rs` - WalTrimmer + +## Behavior + +1. Periodically scan for segments marked as "shipped" +2. Verify the segment exists in object storage (optional double-check) +3. Delete the local WAL segment file +4. Update the offset index to point to S3 location instead + +## Acceptance Criteria + +- [ ] Segment marked as shipped -> trimmer deletes local file +- [ ] Segment NOT marked as shipped -> trimmer leaves it +- [ ] After trimming, index entries point to S3 location +- [ ] Trimmer respects a minimum retention period (keep recent segments locally even if shipped) diff --git a/todos/SQ-018-s3-read-fallback.md b/todos/SQ-018-s3-read-fallback.md new file mode 100644 index 0000000..a62b054 --- /dev/null +++ b/todos/SQ-018-s3-read-fallback.md @@ -0,0 +1,29 @@ +# SQ-018: S3 Read Fallback + +**Status:** `[ ] TODO` +**Blocked by:** SQ-017 +**Priority:** Medium + +## Description + +When a consumer requests an offset from a trimmed segment, fetch it from S3 instead. + +## Files to Create/Modify + +- `crates/sq-storage/src/engine.rs` - update read path with S3 fallback +- `crates/sq-storage/src/object_store/reader.rs` - download + decompress segment from S3 + +## Behavior + +1. Read path checks local WAL first +2. If segment not found locally, check if it's in S3 via the index +3. Download segment from S3, decompress (zstd) +4. Read messages from the downloaded segment +5. Optionally cache downloaded segment locally for subsequent reads + +## Acceptance Criteria + +- [ ] Write messages, ship to S3, trim locally, read from S3 -> messages correct +- [ ] CRC validation on S3-fetched data passes +- [ ] S3 fetch failure: returns appropriate error +- [ ] Performance: subsequent reads of same trimmed segment use local cache diff --git a/todos/SQ-019-virtual-network.md b/todos/SQ-019-virtual-network.md new file mode 100644 index 0000000..353ee55 --- /dev/null +++ b/todos/SQ-019-virtual-network.md @@ -0,0 +1,45 @@ +# SQ-019: Virtual Network for Simulation + +**Status:** `[ ] TODO` +**Blocked by:** SQ-003 +**Priority:** Medium + +## Description + +Virtual network layer for multi-node simulation testing. Enables partition, latency, and packet drop injection. + +## Files to Create/Modify + +- `crates/sq-sim/src/network.rs` - VirtualNetwork with message queues and fault injection + +## VirtualNetwork API + +```rust +pub struct VirtualNetwork { + queues: HashMap<(NodeId, NodeId), VecDeque>>, + partitions: HashSet<(NodeId, NodeId)>, + latency: Option<(Duration, Duration)>, // min, max + drop_probability: f64, +} + +impl VirtualNetwork { + pub fn new() -> Self; + pub fn partition(&mut self, a: NodeId, b: NodeId); + pub fn heal(&mut self, a: NodeId, b: NodeId); + pub fn heal_all(&mut self); + pub fn set_latency(&mut self, min: Duration, max: Duration); + pub fn set_drop_probability(&mut self, prob: f64); + pub async fn send(&self, from: NodeId, to: NodeId, msg: Vec) -> Result<()>; + pub async fn recv(&self, node: NodeId) -> Result<(NodeId, Vec)>; + pub fn deliver_pending(&mut self); // Process queued messages +} +``` + +## Acceptance Criteria + +- [ ] Two virtual nodes exchange messages successfully +- [ ] Partition: messages from A to B are dropped +- [ ] Heal: messages resume flowing after heal +- [ ] Latency injection: messages are delayed +- [ ] Drop probability: some messages are randomly dropped +- [ ] Bidirectional partition: neither direction works diff --git a/todos/SQ-020-cluster-membership.md b/todos/SQ-020-cluster-membership.md new file mode 100644 index 0000000..a456480 --- /dev/null +++ b/todos/SQ-020-cluster-membership.md @@ -0,0 +1,42 @@ +# SQ-020: Cluster Membership (Gossip) + +**Status:** `[ ] TODO` +**Blocked by:** SQ-009, SQ-019 +**Priority:** Medium + +## Description + +Nodes discover each other via seed list and maintain a membership list through periodic heartbeats. + +## Files to Create/Modify + +- `crates/sq-cluster/src/lib.rs` - module exports +- `crates/sq-cluster/src/membership.rs` - seed list, join, heartbeat, failure detection +- `crates/sq-server/src/grpc/cluster.rs` - ClusterService Join/Heartbeat RPC impl +- `crates/sq-server/src/cli/serve.rs` - add --seeds CLI flag + +## Configuration + +``` +SQ_SEEDS=node1:6060,node2:6060 # Seed node addresses +SQ_NODE_ID=node-1 # Unique node ID +SQ_HEARTBEAT_INTERVAL_MS=5000 # Heartbeat every 5s +SQ_FAILURE_THRESHOLD=3 # Missed heartbeats before suspected +``` + +## Membership State Machine + +``` +Unknown -> Alive (on Join response or Heartbeat) +Alive -> Suspected (missed 3 heartbeats) +Suspected -> Dead (suspected for 30 seconds) +Dead -> Alive (on successful re-Join) +``` + +## Acceptance Criteria + +- [ ] Start 3 nodes with seed list, all discover each other +- [ ] Status RPC shows all 3 nodes as "alive" +- [ ] Stop one node, others detect it as "suspected" then "dead" +- [ ] Restart dead node, it re-joins and becomes "alive" +- [ ] Node with no seeds starts as single-node cluster diff --git a/todos/SQ-021-write-replication.md b/todos/SQ-021-write-replication.md new file mode 100644 index 0000000..9df827b --- /dev/null +++ b/todos/SQ-021-write-replication.md @@ -0,0 +1,33 @@ +# SQ-021: Write Replication + +**Status:** `[ ] TODO` +**Blocked by:** SQ-020, SQ-010 +**Priority:** High + +## Description + +Writes are replicated to N peers before ack to client. Simple quorum approach: coordinator writes locally, sends to peers, waits for majority ack. + +## Files to Create/Modify + +- `crates/sq-cluster/src/replication.rs` - Replicator with quorum logic +- `crates/sq-server/src/grpc/cluster.rs` - ReplicateEntries RPC impl +- `crates/sq-server/src/grpc/data_plane.rs` - update Publish to use Replicator + +## Replication Flow + +1. Coordinator receives Publish request +2. Coordinator writes to local WAL, assigns offset +3. Coordinator sends ReplicateEntries to all known alive peers +4. Coordinator waits for W acks (W = floor(N/2) + 1, where N = replication factor) +5. On quorum reached: ack to client +6. On quorum timeout: return error to client + +## Acceptance Criteria + +- [ ] 3-node cluster: publish message, verify all 3 nodes have it in WAL +- [ ] 3-node cluster, 1 node down: publish succeeds (2/3 quorum) +- [ ] 3-node cluster, 2 nodes down: publish fails (no quorum) +- [ ] ACK_MODE_LOCAL: ack after local WAL only (skip replication) +- [ ] ACK_MODE_NONE: return immediately, replicate async +- [ ] Replication timeout: configurable, default 5 seconds diff --git a/todos/SQ-022-simulation-tests.md b/todos/SQ-022-simulation-tests.md new file mode 100644 index 0000000..58c1630 --- /dev/null +++ b/todos/SQ-022-simulation-tests.md @@ -0,0 +1,49 @@ +# SQ-022: Multi-Node Simulation Tests + +**Status:** `[ ] TODO` +**Blocked by:** SQ-021, SQ-019 +**Priority:** High + +## Description + +Full TigerBeetle-inspired simulation test suite. Spin up multiple nodes with virtual I/O, inject faults, verify invariants. + +## Files to Create/Modify + +- `crates/sq-sim/src/runtime.rs` - test harness for multi-node simulation +- `crates/sq-sim/tests/invariants.rs` - invariant checker functions +- `crates/sq-sim/tests/scenarios/mod.rs` +- `crates/sq-sim/tests/scenarios/single_node.rs` - S01-S04 +- `crates/sq-sim/tests/scenarios/multi_node.rs` - S05-S08 +- `crates/sq-sim/tests/scenarios/failures.rs` - S09-S12 + +## Scenarios + +- **S01:** Single node, single producer, single consumer - baseline +- **S02:** Single node, concurrent producers - offset ordering +- **S03:** Single node, disk full during write - graceful error +- **S04:** Single node, crash and restart - WAL recovery +- **S05:** Three nodes, normal operation - replication works +- **S06:** Three nodes, one crashes - remaining two continue +- **S07:** Three nodes, network partition (2+1) - majority continues +- **S08:** Three nodes, S3 outage - local WAL accumulates +- **S09:** Consumer group, offset preservation +- **S10:** High throughput burst - no message loss +- **S11:** Slow consumer with WAL trimming - falls back to S3 +- **S12:** Node rejoins after long absence - catches up + +## Invariants (checked after every step) + +1. No acked message is ever lost +2. Offsets strictly monotonic, no gaps +3. CRC integrity on all reads +4. Consumer group offsets never regress +5. After network heal, replicas converge +6. WAL never trimmed before S3 confirmation + +## Acceptance Criteria + +- [ ] All 12 scenarios pass +- [ ] Each scenario runs with multiple random seeds (at least 10) +- [ ] Invariant violations produce clear diagnostic output +- [ ] Tests complete in < 60 seconds total diff --git a/todos/SQ-023-node-recovery.md b/todos/SQ-023-node-recovery.md new file mode 100644 index 0000000..f7d6c54 --- /dev/null +++ b/todos/SQ-023-node-recovery.md @@ -0,0 +1,31 @@ +# SQ-023: Node Recovery / Catch-Up + +**Status:** `[ ] TODO` +**Blocked by:** SQ-021, SQ-018 +**Priority:** Medium + +## Description + +A node that was offline catches up from peers or S3 when it rejoins. + +## Files to Create/Modify + +- `crates/sq-cluster/src/recovery.rs` - on-join catch-up logic +- `crates/sq-server/src/grpc/cluster.rs` - FetchSegment RPC impl +- `crates/sq-cluster/src/replication.rs` - integrate recovery into join flow + +## Recovery Flow + +1. Rejoining node contacts peers via seed list +2. For each topic-partition, compare local latest offset with peers +3. If peer has newer data: fetch missing segments via FetchSegment RPC +4. If peer has also trimmed: fetch from S3 +5. Replay fetched segments into local WAL +6. Mark node as "caught up" and start accepting writes + +## Acceptance Criteria + +- [ ] Node joins late: fetches missing data from peer, all messages readable +- [ ] Node catches up from S3 when peer has trimmed that segment +- [ ] Recovery doesn't block existing cluster operations +- [ ] Recovery progress is logged diff --git a/todos/SQ-024-docker-compose-e2e.md b/todos/SQ-024-docker-compose-e2e.md new file mode 100644 index 0000000..a75488e --- /dev/null +++ b/todos/SQ-024-docker-compose-e2e.md @@ -0,0 +1,32 @@ +# SQ-024: Docker Compose & E2E Example + +**Status:** `[ ] TODO` +**Blocked by:** SQ-023 +**Priority:** Low + +## Description + +Docker Compose setup for running a 3-node SQ cluster with MinIO, plus an example publish/subscribe program. + +## Files to Create/Modify + +- `templates/docker-compose.yaml` - 3 sq-server instances + MinIO +- `templates/sq-server.Dockerfile` - multi-stage build +- `examples/publish_subscribe/Cargo.toml` +- `examples/publish_subscribe/src/main.rs` +- `scripts/grpc.sh` - grpcurl testing helper + +## Docker Compose Services + +- `minio` - S3-compatible object storage +- `sq-1` - SQ node 1 (seeds: sq-2, sq-3) +- `sq-2` - SQ node 2 (seeds: sq-1, sq-3) +- `sq-3` - SQ node 3 (seeds: sq-1, sq-2) + +## Acceptance Criteria + +- [ ] `docker compose up` starts all 4 services +- [ ] All 3 SQ nodes discover each other (verify via Status RPC) +- [ ] Example program publishes and consumes messages successfully +- [ ] Kill one container, cluster continues operating +- [ ] Restart container, node catches up diff --git a/todos/SQ-025-compression-performance.md b/todos/SQ-025-compression-performance.md new file mode 100644 index 0000000..d699865 --- /dev/null +++ b/todos/SQ-025-compression-performance.md @@ -0,0 +1,29 @@ +# SQ-025: Compression & Performance Tuning + +**Status:** `[ ] TODO` +**Blocked by:** SQ-024 +**Priority:** Low + +## Description + +Add zstd compression to S3 segment shipping and create a benchmark suite. + +## Files to Create/Modify + +- `crates/sq-storage/src/object_store/shipper.rs` - add zstd compression +- `crates/sq-storage/src/object_store/reader.rs` - add zstd decompression +- `crates/sq-storage/benches/` - throughput benchmarks + +## Benchmarks + +- Write throughput: messages/sec at various payload sizes +- Read throughput: messages/sec sequential scan +- Compression ratio: raw vs compressed segment size +- S3 round-trip: write, ship, trim, read from S3 + +## Acceptance Criteria + +- [ ] Compressed segments round-trip correctly (write -> compress -> upload -> download -> decompress -> read) +- [ ] Compression ratio metrics are logged +- [ ] Benchmarks produce readable output +- [ ] No correctness regressions (all existing tests still pass)