feat: add s3 and deployment

Signed-off-by: kjuulh <contact@kjuulh.io>

crates/nodata/src/grpc.rs

@@ -61,7 +61,12 @@ impl no_data_service_server::NoDataService for GrpcServer {
         self.counter.inc();
 
         self.state.ingest().publish(req).await.map_err(|e| {
-            tracing::warn!(error = e.to_string(), "failed to handle ingest of data");
+            let caused_by = e
+                .chain()
+                .map(|e| e.to_string())
+                .collect::<Vec<String>>()
+                .join(", ");
+            tracing::warn!("failed to handle ingest of data: {}: {}", e, caused_by);
             tonic::Status::internal(e.to_string())
         })?;
 

crates/nodata/src/services/consumers.rs

@@ -288,7 +288,7 @@ mod test {
         let topic = "some-topic".to_string();
         let offset = 9usize;
 
-        let staging = Staging::new();
+        let staging = Staging::new().await?;
         // Publish 10 messages
         for _ in 0..10 {
             let offset = staging

crates/nodata/src/services/staging.rs

@@ -1,5 +1,6 @@
 use std::{collections::BTreeMap, sync::Arc};
 
+use nodata_storage::backend::local::LocalStorageBackend;
 use tokio::sync::RwLock;
 
 use crate::state::SharedState;
@@ -23,11 +24,11 @@ pub struct Staging {
 }
 
 impl Staging {
-    pub fn new() -> Self {
-        Self {
+    pub async fn new() -> anyhow::Result<Self> {
+        Ok(Self {
             store: Arc::default(),
-            storage: nodata_storage::Storage::new(nodata_storage::backend::StorageBackend::temp()),
-        }
+            storage: nodata_storage::Storage::new_from_env().await?,
+        })
     }
 
     pub async fn publish(

crates/nodata/src/state.rs

@@ -1,8 +1,6 @@
 use std::{ops::Deref, sync::Arc};
 
-use anyhow::Context;
 use prometheus::Registry;
-use sqlx::{Pool, Postgres};
 
 use crate::services::{consumers::Consumers, handler::Handler, staging::Staging};
 
@@ -24,7 +22,6 @@ impl Deref for SharedState {
 }
 
 pub struct State {
-    pub _db: Pool<Postgres>,
     pub staging: Staging,
     pub consumers: Consumers,
     pub handler: Handler,
@@ -33,23 +30,10 @@ pub struct State {
 
 impl State {
     pub async fn new() -> anyhow::Result<Self> {
-        let db = sqlx::PgPool::connect(
-            &std::env::var("DATABASE_URL").context("DATABASE_URL is not set")?,
-        )
-        .await?;
-
-        sqlx::migrate!("migrations/crdb")
-            .set_locking(false)
-            .run(&db)
-            .await?;
-
-        let _ = sqlx::query("SELECT 1;").fetch_one(&db).await?;
-
-        let staging = Staging::new();
+        let staging = Staging::new().await?;
         let handler = Handler::new(staging.clone());
 
         Ok(Self {
-            _db: db,
             consumers: Consumers::new(),
             staging,
             handler,