feat: add prometheus and protobuf messages

Signed-off-by: kjuulh <contact@kjuulh.io>

crates/nodata-storage/Cargo.toml

@@ -8,6 +8,9 @@ anyhow.workspace = true
 tokio.workspace = true
 uuid.workspace = true
 tracing.workspace = true
+prost.workspace = true
+prost-types.workspace = true
+bytes.workspace = true
 
 hex = "0.4.3"
 sha2 = "0.10.8"

crates/nodata-storage/src/backend.rs

@@ -0,0 +1,84 @@
+use std::{
+    env::temp_dir,
+    path::{Path, PathBuf},
+    time::{SystemTime, UNIX_EPOCH},
+};
+
+use anyhow::Context;
+use tokio::io::AsyncWriteExt;
+
+pub struct StorageBackend {
+    location: PathBuf,
+}
+
+impl StorageBackend {
+    pub fn new(location: &Path) -> Self {
+        Self {
+            location: location.into(),
+        }
+    }
+
+    pub fn temp() -> Self {
+        Self::new(&temp_dir().join("nodata"))
+    }
+
+    pub async fn flush_segment(&self, topic: &str, buffer: &[u8]) -> anyhow::Result<String> {
+        let segment_key = uuid::Uuid::now_v7();
+        let segment_path = PathBuf::from("logs")
+            .join(topic)
+            .join(segment_key.to_string());
+        tracing::trace!("writing segment file: {}", segment_path.display());
+        let file_location = self.location.join(&segment_path);
+        if let Some(parent) = file_location.parent() {
+            tokio::fs::create_dir_all(parent)
+                .await
+                .context("failed to create storage backend dir")?;
+        }
+
+        let mut segment_file = tokio::fs::File::create(&file_location).await?;
+        segment_file.write_all(buffer).await?;
+        segment_file.flush().await?;
+
+        Ok(segment_key.to_string())
+    }
+
+    pub async fn append_index(
+        &self,
+        topic: &str,
+        segment_file: &str,
+        time: SystemTime,
+    ) -> anyhow::Result<()> {
+        let index_path = PathBuf::from("indexes").join(topic);
+        tracing::trace!("writing index file: {}", index_path.display());
+        let file_location = self.location.join(&index_path);
+        if let Some(parent) = file_location.parent() {
+            tokio::fs::create_dir_all(parent)
+                .await
+                .context("failed to create storage backend dir, index")?;
+        }
+
+        if !file_location.exists() {
+            tokio::fs::File::create(&file_location).await?;
+        }
+
+        let mut index_file = tokio::fs::File::options()
+            .append(true)
+            .open(&file_location)
+            .await?;
+        index_file
+            .write_all(
+                format!(
+                    "{},{}\n",
+                    time.duration_since(UNIX_EPOCH)
+                        .expect("to be able to get time")
+                        .as_secs(),
+                    segment_file
+                )
+                .as_bytes(),
+            )
+            .await?;
+        index_file.flush().await?;
+
+        Ok(())
+    }
+}

crates/nodata-storage/src/gen/nodata.v1.rs

@@ -0,0 +1,61 @@
+// @generated
+// This file is @generated by prost-build.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct PublishEventRequest {
+    #[prost(string, tag="1")]
+    pub topic: ::prost::alloc::string::String,
+    #[prost(bytes="vec", tag="2")]
+    pub value: ::prost::alloc::vec::Vec<u8>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+pub struct PublishEventResponse {
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+pub struct GetTopicsRequest {
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetTopicsResponse {
+    #[prost(string, repeated, tag="1")]
+    pub topics: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SubscribeRequest {
+    #[prost(string, tag="1")]
+    pub topic: ::prost::alloc::string::String,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SubscribeResponse {
+    #[prost(message, optional, tag="2")]
+    pub published: ::core::option::Option<::prost_types::Timestamp>,
+    #[prost(bytes="vec", tag="4")]
+    pub value: ::prost::alloc::vec::Vec<u8>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+pub struct HandleMsgRequest {
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+pub struct HandleMsgResponse {
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+pub struct PingRequest {
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+pub struct PingResponse {
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Log {
+    #[prost(bytes="vec", repeated, tag="1")]
+    pub messages: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec<u8>>,
+}
+// @@protoc_insertion_point(module)

crates/nodata-storage/src/lib.rs

@@ -9,103 +9,21 @@ use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
 
 use anyhow::Context;
 use backend::StorageBackend;
+use proto::ProtoStorage;
 use sha2::{Digest, Sha256};
 use tokio::sync::Mutex;
 
 type TopicHashKey = String;
 
-pub mod backend {
-    use std::{
-        env::temp_dir,
-        path::{Path, PathBuf},
-        time::{SystemTime, UNIX_EPOCH},
-    };
-
-    use anyhow::Context;
-    use tokio::io::AsyncWriteExt;
-
-    pub struct StorageBackend {
-        location: PathBuf,
-    }
-
-    impl StorageBackend {
-        pub fn new(location: &Path) -> Self {
-            Self {
-                location: location.into(),
-            }
-        }
-
-        pub fn temp() -> Self {
-            Self::new(&temp_dir().join("nodata"))
-        }
-
-        pub async fn flush_segment(&self, topic: &str, buffer: &[u8]) -> anyhow::Result<String> {
-            let segment_key = uuid::Uuid::now_v7();
-            let segment_path = PathBuf::from("logs")
-                .join(topic)
-                .join(segment_key.to_string());
-            tracing::trace!("writing segment file: {}", segment_path.display());
-            let file_location = self.location.join(&segment_path);
-            if let Some(parent) = file_location.parent() {
-                tokio::fs::create_dir_all(parent)
-                    .await
-                    .context("failed to create storage backend dir")?;
-            }
-
-            let mut segment_file = tokio::fs::File::create(&file_location).await?;
-            segment_file.write_all(buffer).await?;
-            segment_file.flush().await?;
-
-            Ok(segment_key.to_string())
-        }
-
-        pub async fn append_index(
-            &self,
-            topic: &str,
-            segment_file: &str,
-            time: SystemTime,
-        ) -> anyhow::Result<()> {
-            let index_path = PathBuf::from("indexes").join(topic);
-            tracing::trace!("writing index file: {}", index_path.display());
-            let file_location = self.location.join(&index_path);
-            if let Some(parent) = file_location.parent() {
-                tokio::fs::create_dir_all(parent)
-                    .await
-                    .context("failed to create storage backend dir, index")?;
-            }
-
-            if !file_location.exists() {
-                tokio::fs::File::create(&file_location).await?;
-            }
-
-            let mut index_file = tokio::fs::File::options()
-                .append(true)
-                .open(&file_location)
-                .await?;
-            index_file
-                .write_all(
-                    format!(
-                        "{},{}\n",
-                        time.duration_since(UNIX_EPOCH)
-                            .expect("to be able to get time")
-                            .as_secs(),
-                        segment_file
-                    )
-                    .as_bytes(),
-                )
-                .await?;
-            index_file.flush().await?;
-
-            Ok(())
-        }
-    }
-}
+pub mod backend;
 
 #[derive(Clone)]
 pub struct Storage {
     segment_size_bytes: usize,
-    buffer: Arc<Mutex<BTreeMap<TopicHashKey, Vec<u8>>>>,
+    buffer: Arc<Mutex<BTreeMap<TopicHashKey, Vec<Vec<u8>>>>>,
     backend: Arc<StorageBackend>,
+
+    codec: ProtoStorage,
 }
 
 impl Storage {
@@ -113,7 +31,9 @@ impl Storage {
         Self {
             segment_size_bytes: 4096 * 1000, // 4MB
             buffer: Arc::default(),
+
             backend: Arc::new(backend),
+            codec: ProtoStorage::default(),
         }
     }
 
@@ -136,20 +56,30 @@ impl Storage {
             }
 
             let log_buffer = buffer.get_mut(&topic_key).unwrap();
-            if log_buffer.len() + value.len() >= self.segment_size_bytes {
+            if log_buffer
+                .iter()
+                .map(|b| b.len())
+                .reduce(|acc, i| acc + i)
+                .unwrap_or_default()
+                >= self.segment_size_bytes
+            {
+                let log_buffer = buffer.remove(&topic_key).unwrap();
                 let time = SystemTime::now();
                 let segment_id = self
                     .backend
-                    .flush_segment(&topic_key, log_buffer)
+                    .flush_segment(&topic_key, &self.codec.format_log_message(log_buffer))
                     .await
                     .context("failed to write segment")?;
                 self.backend
                     .append_index(&topic_key, &segment_id, time)
                     .await?;
-                log_buffer.clear();
-            }
 
-            log_buffer.extend(value);
+                let mut log_buf = Vec::with_capacity(self.segment_size_bytes);
+                log_buf.push(value.to_vec());
+                buffer.insert(topic_key.clone(), log_buf);
+            } else {
+                log_buffer.push(value.to_vec());
+            }
         }
 
         Ok(())
@@ -168,3 +98,20 @@ impl Storage {
         hex::encode(Sha256::digest(input.as_bytes()))
     }
 }
+
+mod proto {
+    use prost::Message;
+
+    mod gen {
+        include!("gen/nodata.v1.rs");
+    }
+
+    #[derive(Default, Clone)]
+    pub struct ProtoStorage {}
+
+    impl ProtoStorage {
+        pub fn format_log_message(&self, messages: Vec<Vec<u8>>) -> Vec<u8> {
+            gen::Log { messages }.encode_to_vec()
+        }
+    }
+}