kjuulh/nodata
1
0
Fork 0
Code Issues 1 Pull Requests 6 Actions Packages Projects Releases Wiki Activity

feat: add s3 and deployment

Browse Source 
Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
Kasper Juul Hermansen
2024-11-17 15:43:37 +01:00
parent 2e8d14f5a6
commit c8f4bae1f2
12 changed files with 1093 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
crates/nodata-storage/src/lib.rs
View File

@@ -8,7 +8,7 @@
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
use anyhow::Context;
use backend::StorageBackend;
use backend::{local::LocalStorageBackend, StorageBackend};
use proto::ProtoStorage;
use sha2::{Digest, Sha256};
use tokio::sync::Mutex;
@@ -21,19 +21,43 @@ pub mod backend;
pub struct Storage {
segment_size_bytes: usize,
buffer: Arc<Mutex<BTreeMap<TopicHashKey, Vec<Vec<u8>>>>>,
backend: Arc<StorageBackend>,
backend: Arc<dyn StorageBackend + Send + Sync + 'static>,
codec: ProtoStorage,
}
impl Storage {
pub fn new(backend: StorageBackend) -> Self {
pub fn new(backend: LocalStorageBackend) -> Self {
Self {
segment_size_bytes: 4096 * 1000, // 4MB
buffer: Arc::default(),
codec: ProtoStorage::default(),
backend: Arc::new(backend),
codec: ProtoStorage::default(),
}
}
pub async fn new_from_env() -> anyhow::Result<Self> {
match std::env::var("STORAGE_BACKEND")
.context("failed to find STORAGE_BACKEND in env")?
.as_str()
{
"local" => Ok(Self {
segment_size_bytes: 4096 * 1000, // 4MB
buffer: Arc::default(),
codec: ProtoStorage::default(),
backend: Arc::new(LocalStorageBackend::new_from_env()?),
}),
#[cfg(feature = "s3")]
"s3" => Ok(Self {
segment_size_bytes: 4 * 1024 * 1000, // 4MB
buffer: Arc::default(),
codec: ProtoStorage::default(),
backend: Arc::new(backend::s3::S3StorageBackend::new_from_env().await?),
}),
backend => anyhow::bail!("backend is not supported: {}", backend),
}
}