feat: add post3 s3 proxy for postgresql

Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
2026-02-27 11:37:48 +01:00
commit 21bac4a33f
67 changed files with 14403 additions and 0 deletions

View File

@@ -0,0 +1,17 @@
[package]
name = "post3-sdk"
version.workspace = true
edition.workspace = true
[dependencies]
aws-sdk-s3 = "1"
aws-credential-types = { version = "1", features = ["hardcoded-credentials"] }
aws-types = "1"
aws-config = "1"
bytes.workspace = true
thiserror.workspace = true
chrono.workspace = true
[dev-dependencies]
tokio.workspace = true
anyhow.workspace = true

View File

@@ -0,0 +1,107 @@
//! Use aws-sdk-s3 directly against post3 (without the post3-sdk wrapper).
//! Shows the raw configuration needed.
//!
//! Prerequisites: post3-server running on localhost:9000
//! mise run up && mise run dev
//!
//! Run:
//! cargo run -p post3-sdk --example aws_sdk_direct
use post3_sdk::aws_sdk_s3;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let endpoint = std::env::var("POST3_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:9000".to_string());
// Configure aws-sdk-s3 manually for post3
let creds = aws_sdk_s3::config::Credentials::new(
"test", // access key (any value works when auth is disabled)
"test", // secret key
None, // session token
None, // expiry
"example", // provider name
);
let config = aws_sdk_s3::Config::builder()
.behavior_version_latest()
.region(aws_sdk_s3::config::Region::new("us-east-1"))
.endpoint_url(&endpoint)
.credentials_provider(creds)
.force_path_style(true) // Required: post3 uses path-style, not virtual-hosted
.build();
let client = aws_sdk_s3::Client::from_conf(config);
// Create bucket
println!("Creating bucket...");
client
.create_bucket()
.bucket("direct-bucket")
.send()
.await?;
// Put object
println!("Putting object...");
client
.put_object()
.bucket("direct-bucket")
.key("greeting.txt")
.body(Vec::from(&b"Hello from aws-sdk-s3!"[..]).into())
.send()
.await?;
// Get object
let resp = client
.get_object()
.bucket("direct-bucket")
.key("greeting.txt")
.send()
.await?;
let body = resp.body.collect().await?.into_bytes();
println!("Got: {}", String::from_utf8_lossy(&body));
// List objects
let list = client
.list_objects_v2()
.bucket("direct-bucket")
.send()
.await?;
println!("Objects:");
for obj in list.contents() {
println!(
" {} ({} bytes)",
obj.key().unwrap_or("?"),
obj.size().unwrap_or(0)
);
}
// Head object
let head = client
.head_object()
.bucket("direct-bucket")
.key("greeting.txt")
.send()
.await?;
println!(
"Head: size={}, etag={:?}",
head.content_length().unwrap_or(0),
head.e_tag()
);
// Cleanup
client
.delete_object()
.bucket("direct-bucket")
.key("greeting.txt")
.send()
.await?;
client
.delete_bucket()
.bucket("direct-bucket")
.send()
.await?;
println!("Done!");
Ok(())
}

View File

@@ -0,0 +1,76 @@
//! Basic post3 usage: create a bucket, put/get/delete objects, list objects.
//!
//! Prerequisites: post3-server running on localhost:9000
//! mise run up && mise run dev
//!
//! Run:
//! cargo run -p post3-sdk --example basic
use post3_sdk::Post3Client;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let endpoint = std::env::var("POST3_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:9000".to_string());
let client = Post3Client::new(&endpoint);
// Create a bucket
println!("Creating bucket 'example-bucket'...");
client.create_bucket("example-bucket").await?;
// List buckets
let buckets = client.list_buckets().await?;
println!("Buckets: {:?}", buckets);
// Put an object
println!("Putting 'hello.txt'...");
client
.put_object("example-bucket", "hello.txt", b"Hello, post3!")
.await?;
// Get the object back
let data = client.get_object("example-bucket", "hello.txt").await?;
println!("Got: {}", String::from_utf8_lossy(&data));
// Put a few more objects
client
.put_object("example-bucket", "docs/readme.md", b"# README")
.await?;
client
.put_object("example-bucket", "docs/guide.md", b"# Guide")
.await?;
// List all objects
let objects = client.list_objects("example-bucket", None).await?;
println!("All objects:");
for obj in &objects {
println!(" {} ({} bytes)", obj.key, obj.size);
}
// List with prefix filter
let docs = client
.list_objects("example-bucket", Some("docs/"))
.await?;
println!("Objects under docs/:");
for obj in &docs {
println!(" {} ({} bytes)", obj.key, obj.size);
}
// Delete objects
println!("Cleaning up...");
client
.delete_object("example-bucket", "hello.txt")
.await?;
client
.delete_object("example-bucket", "docs/readme.md")
.await?;
client
.delete_object("example-bucket", "docs/guide.md")
.await?;
// Delete the bucket
client.delete_bucket("example-bucket").await?;
println!("Done!");
Ok(())
}

View File

@@ -0,0 +1,161 @@
//! Stress test: upload and verify large files.
//!
//! Tests progressively larger files to find limits and measure performance.
//! Generates deterministic pseudo-random data so we can verify integrity
//! without keeping the full payload in memory twice.
//!
//! Prerequisites: post3-server running on localhost:9000
//! mise run up && mise run dev
//!
//! Run:
//! cargo run -p post3-sdk --example large_upload --release
//!
//! Or with custom sizes (in MB):
//! POST3_SIZES=10,50,100,500,1024 cargo run -p post3-sdk --example large_upload --release
use post3_sdk::Post3Client;
use std::time::Instant;
fn generate_data(size_bytes: usize) -> Vec<u8> {
// Deterministic pattern: repeating 256-byte blocks with position-dependent content
let mut data = Vec::with_capacity(size_bytes);
let mut state: u64 = 0xdeadbeef;
while data.len() < size_bytes {
// Simple xorshift64 PRNG for fast deterministic data
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
data.extend_from_slice(&state.to_le_bytes());
}
data.truncate(size_bytes);
data
}
fn format_size(bytes: usize) -> String {
if bytes >= 1024 * 1024 * 1024 {
format!("{:.1} GiB", bytes as f64 / (1024.0 * 1024.0 * 1024.0))
} else if bytes >= 1024 * 1024 {
format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0))
} else if bytes >= 1024 {
format!("{:.1} KiB", bytes as f64 / 1024.0)
} else {
format!("{} B", bytes)
}
}
fn format_throughput(bytes: usize, duration: std::time::Duration) -> String {
let secs = duration.as_secs_f64();
if secs == 0.0 {
return "".to_string();
}
let mb_per_sec = bytes as f64 / (1024.0 * 1024.0) / secs;
format!("{:.1} MiB/s", mb_per_sec)
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let endpoint = std::env::var("POST3_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:9000".to_string());
let client = Post3Client::new(&endpoint);
// Parse sizes from env or use defaults
let sizes_mb: Vec<usize> = std::env::var("POST3_SIZES")
.unwrap_or_else(|_| "1,10,50,100,500,1024,2048".to_string())
.split(',')
.filter_map(|s| s.trim().parse().ok())
.collect();
println!("=== post3 Large File Stress Test ===");
println!("Endpoint: {}", endpoint);
println!("Sizes: {:?} MB", sizes_mb);
println!();
client.create_bucket("stress-test").await?;
for size_mb in &sizes_mb {
let size_bytes = size_mb * 1024 * 1024;
let key = format!("test-{}mb.bin", size_mb);
println!("--- {} ---", format_size(size_bytes));
// Generate data
print!(" Generating data... ");
let gen_start = Instant::now();
let data = generate_data(size_bytes);
println!("done ({:.1}s)", gen_start.elapsed().as_secs_f64());
// Upload
print!(" Uploading... ");
let upload_start = Instant::now();
match client.put_object("stress-test", &key, &data).await {
Ok(()) => {
let upload_dur = upload_start.elapsed();
println!(
"done ({:.1}s, {})",
upload_dur.as_secs_f64(),
format_throughput(size_bytes, upload_dur)
);
}
Err(e) => {
println!("FAILED: {}", e);
println!(" Skipping remaining sizes (hit server limit)");
break;
}
}
// Head (verify metadata)
let head = client.head_object("stress-test", &key).await?;
if let Some(info) = &head {
println!(
" Head: size={}, etag={:?}",
format_size(info.size as usize),
info.etag
);
}
// Download
print!(" Downloading... ");
let download_start = Instant::now();
match client.get_object("stress-test", &key).await {
Ok(downloaded) => {
let download_dur = download_start.elapsed();
println!(
"done ({:.1}s, {})",
download_dur.as_secs_f64(),
format_throughput(size_bytes, download_dur)
);
// Verify integrity
print!(" Verifying... ");
if downloaded.len() != data.len() {
println!(
"FAILED: size mismatch (expected {}, got {})",
data.len(),
downloaded.len()
);
} else if downloaded.as_ref() == data.as_slice() {
println!("OK (byte-for-byte match)");
} else {
// Find first mismatch
let pos = data
.iter()
.zip(downloaded.iter())
.position(|(a, b)| a != b)
.unwrap_or(0);
println!("FAILED: mismatch at byte {}", pos);
}
}
Err(e) => {
println!("FAILED: {}", e);
}
}
// Cleanup this object
client.delete_object("stress-test", &key).await?;
println!();
}
client.delete_bucket("stress-test").await?;
println!("=== Done ===");
Ok(())
}

View File

@@ -0,0 +1,78 @@
//! Demonstrate custom metadata (x-amz-meta-*) with post3.
//!
//! Prerequisites: post3-server running on localhost:9000
//! mise run up && mise run dev
//!
//! Run:
//! cargo run -p post3-sdk --example metadata
use post3_sdk::Post3Client;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let endpoint = std::env::var("POST3_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:9000".to_string());
let client = Post3Client::new(&endpoint);
client.create_bucket("meta-bucket").await?;
// Use the inner aws-sdk-s3 client to set custom metadata
let inner = client.inner();
println!("Putting object with custom metadata...");
inner
.put_object()
.bucket("meta-bucket")
.key("report.pdf")
.body(Vec::from(&b"fake pdf content"[..]).into())
.content_type("application/pdf")
.metadata("author", "alice")
.metadata("department", "engineering")
.metadata("version", "2")
.send()
.await?;
// Retrieve metadata via head_object
let head = inner
.head_object()
.bucket("meta-bucket")
.key("report.pdf")
.send()
.await?;
println!("Content-Type: {:?}", head.content_type());
println!("Content-Length: {:?}", head.content_length());
println!("ETag: {:?}", head.e_tag());
if let Some(metadata) = head.metadata() {
println!("Custom metadata:");
for (k, v) in metadata {
println!(" x-amz-meta-{}: {}", k, v);
}
}
// Retrieve the full object with metadata
let resp = inner
.get_object()
.bucket("meta-bucket")
.key("report.pdf")
.send()
.await?;
println!("\nGet object response:");
println!(" Content-Type: {:?}", resp.content_type());
if let Some(metadata) = resp.metadata() {
println!(" Metadata:");
for (k, v) in metadata {
println!(" x-amz-meta-{}: {}", k, v);
}
}
let body = resp.body.collect().await?.into_bytes();
println!(" Body: {}", String::from_utf8_lossy(&body));
// Cleanup
client.delete_object("meta-bucket", "report.pdf").await?;
client.delete_bucket("meta-bucket").await?;
println!("\nDone!");
Ok(())
}

View File

@@ -0,0 +1,177 @@
//! Stress test: multipart upload and verify huge files (416 GiB).
//!
//! Uses the SDK's multipart_upload convenience method which splits data into
//! parts and uploads them sequentially via CreateMultipartUpload / UploadPart /
//! CompleteMultipartUpload.
//!
//! Prerequisites: post3-server running on localhost:9000
//! mise run up && mise run dev
//!
//! Run:
//! cargo run -p post3-sdk --example multipart_upload --release
//!
//! Or with custom sizes (in MB) and part size:
//! POST3_SIZES=4096,8192,16384 POST3_PART_SIZE=64 cargo run -p post3-sdk --example multipart_upload --release
use post3_sdk::Post3Client;
use std::time::Instant;
fn generate_data(size_bytes: usize) -> Vec<u8> {
let mut data = Vec::with_capacity(size_bytes);
let mut state: u64 = 0xdeadbeef;
while data.len() < size_bytes {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
data.extend_from_slice(&state.to_le_bytes());
}
data.truncate(size_bytes);
data
}
fn format_size(bytes: usize) -> String {
if bytes >= 1024 * 1024 * 1024 {
format!("{:.1} GiB", bytes as f64 / (1024.0 * 1024.0 * 1024.0))
} else if bytes >= 1024 * 1024 {
format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0))
} else if bytes >= 1024 {
format!("{:.1} KiB", bytes as f64 / 1024.0)
} else {
format!("{} B", bytes)
}
}
fn format_throughput(bytes: usize, duration: std::time::Duration) -> String {
let secs = duration.as_secs_f64();
if secs == 0.0 {
return "".to_string();
}
let mb_per_sec = bytes as f64 / (1024.0 * 1024.0) / secs;
format!("{:.1} MiB/s", mb_per_sec)
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let endpoint = std::env::var("POST3_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:9000".to_string());
let client = Post3Client::new(&endpoint);
let sizes_mb: Vec<usize> = std::env::var("POST3_SIZES")
.unwrap_or_else(|_| "100,1024,4096,8192,16384".to_string())
.split(',')
.filter_map(|s| s.trim().parse().ok())
.collect();
// Part size in MiB (default 64 MiB — good balance of part count vs memory)
let part_size_mb: usize = std::env::var("POST3_PART_SIZE")
.unwrap_or_else(|_| "64".to_string())
.parse()
.unwrap_or(64);
let part_size = part_size_mb * 1024 * 1024;
println!("=== post3 Multipart Upload Stress Test ===");
println!("Endpoint: {}", endpoint);
println!("Sizes: {:?} MB", sizes_mb);
println!("Part size: {} MiB", part_size_mb);
println!();
client.create_bucket("mp-stress").await?;
for size_mb in &sizes_mb {
let size_bytes = size_mb * 1024 * 1024;
let key = format!("mp-test-{}mb.bin", size_mb);
let num_parts = (size_bytes + part_size - 1) / part_size;
println!("--- {} ({} parts of {} each) ---",
format_size(size_bytes),
num_parts,
format_size(part_size.min(size_bytes)),
);
// Generate data
print!(" Generating data... ");
let gen_start = Instant::now();
let data = generate_data(size_bytes);
println!("done ({:.1}s)", gen_start.elapsed().as_secs_f64());
// Multipart upload
print!(" Uploading (multipart)... ");
let upload_start = Instant::now();
match client.multipart_upload("mp-stress", &key, &data, part_size).await {
Ok(()) => {
let upload_dur = upload_start.elapsed();
println!(
"done ({:.1}s, {})",
upload_dur.as_secs_f64(),
format_throughput(size_bytes, upload_dur)
);
}
Err(e) => {
println!("FAILED: {}", e);
println!(" Skipping remaining sizes");
break;
}
}
// Head (verify metadata)
let head = client.head_object("mp-stress", &key).await?;
if let Some(info) = &head {
println!(
" Head: size={}, etag={:?}",
format_size(info.size as usize),
info.etag
);
// Verify the compound ETag format (md5-N)
if let Some(ref etag) = info.etag {
let stripped = etag.trim_matches('"');
if stripped.contains('-') {
let parts_str = stripped.split('-').last().unwrap_or("?");
println!(" ETag format: compound ({} parts)", parts_str);
}
}
}
// Download and verify
print!(" Downloading... ");
let download_start = Instant::now();
match client.get_object("mp-stress", &key).await {
Ok(downloaded) => {
let download_dur = download_start.elapsed();
println!(
"done ({:.1}s, {})",
download_dur.as_secs_f64(),
format_throughput(size_bytes, download_dur)
);
print!(" Verifying... ");
if downloaded.len() != data.len() {
println!(
"FAILED: size mismatch (expected {}, got {})",
data.len(),
downloaded.len()
);
} else if downloaded.as_ref() == data.as_slice() {
println!("OK (byte-for-byte match)");
} else {
let pos = data
.iter()
.zip(downloaded.iter())
.position(|(a, b)| a != b)
.unwrap_or(0);
println!("FAILED: mismatch at byte {}", pos);
}
}
Err(e) => {
println!("FAILED: {}", e);
}
}
// Cleanup
client.delete_object("mp-stress", &key).await?;
println!();
}
client.delete_bucket("mp-stress").await?;
println!("=== Done ===");
Ok(())
}

408
crates/post3-sdk/src/lib.rs Normal file
View File

@@ -0,0 +1,408 @@
use aws_credential_types::Credentials;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::Client;
use bytes::Bytes;
pub use aws_sdk_s3;
pub use bytes;
/// Error type for post3-sdk operations.
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("bucket not found: {0}")]
BucketNotFound(String),
#[error("object not found: {bucket}/{key}")]
ObjectNotFound { bucket: String, key: String },
#[error("s3 error: {0}")]
S3(String),
}
impl<E: std::fmt::Display> From<aws_sdk_s3::error::SdkError<E>> for Error {
fn from(err: aws_sdk_s3::error::SdkError<E>) -> Self {
Error::S3(err.to_string())
}
}
pub type Result<T> = std::result::Result<T, Error>;
/// Summary of an object returned by list operations.
#[derive(Debug, Clone)]
pub struct ObjectInfo {
pub key: String,
pub size: i64,
pub etag: Option<String>,
pub last_modified: Option<chrono::DateTime<chrono::Utc>>,
}
/// A client for post3 that wraps `aws-sdk-s3` with ergonomic defaults.
///
/// # Example
///
/// ```no_run
/// # async fn example() -> post3_sdk::Result<()> {
/// let client = post3_sdk::Post3Client::new("http://localhost:9000");
///
/// client.create_bucket("my-bucket").await?;
/// client.put_object("my-bucket", "hello.txt", b"hello world").await?;
///
/// let data = client.get_object("my-bucket", "hello.txt").await?;
/// assert_eq!(data.as_ref(), b"hello world");
/// # Ok(())
/// # }
/// ```
pub struct Post3Client {
inner: Client,
}
impl Post3Client {
/// Create a client with default configuration (dummy credentials, us-east-1, path-style).
pub fn new(endpoint_url: impl Into<String>) -> Self {
Self::builder().endpoint_url(endpoint_url).build()
}
/// Access the underlying `aws_sdk_s3::Client` for advanced operations.
pub fn inner(&self) -> &Client {
&self.inner
}
/// Start building a client with custom configuration.
pub fn builder() -> Post3ClientBuilder {
Post3ClientBuilder::default()
}
// -- Bucket operations --
pub async fn create_bucket(&self, name: &str) -> Result<()> {
self.inner
.create_bucket()
.bucket(name)
.send()
.await?;
Ok(())
}
pub async fn head_bucket(&self, name: &str) -> Result<bool> {
match self.inner.head_bucket().bucket(name).send().await {
Ok(_) => Ok(true),
Err(err) => {
if err
.as_service_error()
.map_or(false, |e| e.is_not_found())
{
Ok(false)
} else {
Err(Error::S3(err.to_string()))
}
}
}
}
pub async fn delete_bucket(&self, name: &str) -> Result<()> {
self.inner
.delete_bucket()
.bucket(name)
.send()
.await?;
Ok(())
}
pub async fn list_buckets(&self) -> Result<Vec<String>> {
let resp = self.inner.list_buckets().send().await?;
Ok(resp
.buckets()
.iter()
.filter_map(|b| b.name().map(|s| s.to_string()))
.collect())
}
// -- Object operations --
pub async fn put_object(
&self,
bucket: &str,
key: &str,
body: impl AsRef<[u8]>,
) -> Result<()> {
let body = Bytes::copy_from_slice(body.as_ref());
self.inner
.put_object()
.bucket(bucket)
.key(key)
.body(body.into())
.send()
.await?;
Ok(())
}
pub async fn get_object(&self, bucket: &str, key: &str) -> Result<Bytes> {
let resp = self
.inner
.get_object()
.bucket(bucket)
.key(key)
.send()
.await
.map_err(|e| {
if e.as_service_error()
.map_or(false, |se| se.is_no_such_key())
{
Error::ObjectNotFound {
bucket: bucket.to_string(),
key: key.to_string(),
}
} else {
Error::S3(e.to_string())
}
})?;
let data = resp
.body
.collect()
.await
.map_err(|e| Error::S3(e.to_string()))?;
Ok(data.into_bytes())
}
pub async fn head_object(
&self,
bucket: &str,
key: &str,
) -> Result<Option<ObjectInfo>> {
match self
.inner
.head_object()
.bucket(bucket)
.key(key)
.send()
.await
{
Ok(resp) => Ok(Some(ObjectInfo {
key: key.to_string(),
size: resp.content_length().unwrap_or(0),
etag: resp.e_tag().map(|s| s.to_string()),
last_modified: resp
.last_modified()
.and_then(|t| {
chrono::DateTime::from_timestamp(t.secs(), t.subsec_nanos())
}),
})),
Err(err) => {
if err
.as_service_error()
.map_or(false, |e| e.is_not_found())
{
Ok(None)
} else {
Err(Error::S3(err.to_string()))
}
}
}
}
pub async fn delete_object(&self, bucket: &str, key: &str) -> Result<()> {
self.inner
.delete_object()
.bucket(bucket)
.key(key)
.send()
.await?;
Ok(())
}
/// Upload an object using multipart upload, splitting into parts of the given size.
///
/// This is useful for large files where multipart upload provides better performance
/// through parallelism and resumability.
pub async fn multipart_upload(
&self,
bucket: &str,
key: &str,
data: impl AsRef<[u8]>,
part_size: usize,
) -> Result<()> {
let data = data.as_ref();
// Create multipart upload
let create_resp = self
.inner
.create_multipart_upload()
.bucket(bucket)
.key(key)
.send()
.await?;
let upload_id = create_resp
.upload_id()
.ok_or_else(|| Error::S3("missing upload_id in response".to_string()))?
.to_string();
// Upload parts
let mut completed_parts = Vec::new();
let mut part_number = 1i32;
for chunk in data.chunks(part_size) {
let body = Bytes::copy_from_slice(chunk);
let upload_resp = self
.inner
.upload_part()
.bucket(bucket)
.key(key)
.upload_id(&upload_id)
.part_number(part_number)
.body(body.into())
.send()
.await
.map_err(|e| {
// Try to abort on failure
Error::S3(e.to_string())
})?;
let etag = upload_resp
.e_tag()
.ok_or_else(|| Error::S3("missing ETag in upload_part response".to_string()))?
.to_string();
completed_parts.push(
CompletedPart::builder()
.part_number(part_number)
.e_tag(etag)
.build(),
);
part_number += 1;
}
// Complete multipart upload
let mut builder = CompletedMultipartUpload::builder();
for part in completed_parts {
builder = builder.parts(part);
}
self.inner
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(&upload_id)
.multipart_upload(builder.build())
.send()
.await?;
Ok(())
}
pub async fn list_objects(
&self,
bucket: &str,
prefix: Option<&str>,
) -> Result<Vec<ObjectInfo>> {
let mut req = self
.inner
.list_objects_v2()
.bucket(bucket);
if let Some(p) = prefix {
req = req.prefix(p);
}
let resp = req.send().await?;
Ok(resp
.contents()
.iter()
.map(|obj| ObjectInfo {
key: obj.key().unwrap_or_default().to_string(),
size: obj.size().unwrap_or(0),
etag: obj.e_tag().map(|s| s.to_string()),
last_modified: obj
.last_modified()
.and_then(|t| {
chrono::DateTime::from_timestamp(t.secs(), t.subsec_nanos())
}),
})
.collect())
}
}
/// Builder for `Post3Client` with custom configuration.
pub struct Post3ClientBuilder {
endpoint_url: Option<String>,
access_key: String,
secret_key: String,
region: String,
}
impl Default for Post3ClientBuilder {
fn default() -> Self {
Self {
endpoint_url: None,
access_key: "test".to_string(),
secret_key: "test".to_string(),
region: "us-east-1".to_string(),
}
}
}
impl Post3ClientBuilder {
pub fn endpoint_url(mut self, url: impl Into<String>) -> Self {
self.endpoint_url = Some(url.into());
self
}
pub fn credentials(mut self, access_key: impl Into<String>, secret_key: impl Into<String>) -> Self {
self.access_key = access_key.into();
self.secret_key = secret_key.into();
self
}
pub fn region(mut self, region: impl Into<String>) -> Self {
self.region = region.into();
self
}
pub fn build(self) -> Post3Client {
let creds = Credentials::new(
&self.access_key,
&self.secret_key,
None,
None,
"post3-sdk",
);
let mut config = aws_sdk_s3::Config::builder()
.behavior_version_latest()
.region(aws_types::region::Region::new(self.region))
.credentials_provider(creds)
.force_path_style(true);
if let Some(url) = self.endpoint_url {
config = config.endpoint_url(url);
}
Post3Client {
inner: Client::from_conf(config.build()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_construct_client() {
let client = Post3Client::new("http://localhost:9000");
// Verify we can access the inner client
let _inner = client.inner();
}
#[test]
fn test_builder_custom_creds() {
let client = Post3Client::builder()
.endpoint_url("http://localhost:9000")
.credentials("my-access-key", "my-secret-key")
.region("eu-west-1")
.build();
let _inner = client.inner();
}
}

View File

@@ -0,0 +1,37 @@
[package]
name = "post3-server"
version.workspace = true
edition.workspace = true
[dependencies]
post3.workspace = true
anyhow.workspace = true
tokio.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
clap.workspace = true
dotenvy.workspace = true
uuid.workspace = true
bytes.workspace = true
axum.workspace = true
tower.workspace = true
tower-http.workspace = true
notmad.workspace = true
tokio-util.workspace = true
sqlx.workspace = true
chrono.workspace = true
quick-xml.workspace = true
md-5.workspace = true
hex.workspace = true
serde.workspace = true
[dev-dependencies]
aws-config = "1"
aws-sdk-s3 = "1"
aws-credential-types = "1"
aws-types = "1"
tokio = { workspace = true, features = ["test-util"] }
tower.workspace = true
tracing-subscriber.workspace = true
tempfile.workspace = true

View File

@@ -0,0 +1,58 @@
pub mod serve;
use anyhow::Context;
use clap::{Parser, Subcommand};
use post3::{FilesystemBackend, PostgresBackend};
use sqlx::PgPool;
use crate::state::State;
#[derive(Parser)]
#[command(name = "post3-server", about = "S3-compatible storage server")]
struct App {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
Serve(serve::ServeCommand),
}
pub async fn execute() -> anyhow::Result<()> {
let app = App::parse();
match app.command {
Commands::Serve(cmd) => match cmd.backend {
serve::BackendType::Pg => {
let database_url =
std::env::var("DATABASE_URL").context("DATABASE_URL not set")?;
let pool = PgPool::connect(&database_url).await?;
sqlx::migrate!("../post3/migrations/")
.set_locking(false)
.run(&pool)
.await?;
tracing::info!("database migrations applied");
let backend = PostgresBackend::new(pool);
let state = State { store: backend };
cmd.run(&state).await
}
serve::BackendType::Fs => {
let data_dir = cmd
.data_dir
.as_ref()
.context("--data-dir is required when using --backend fs")?;
std::fs::create_dir_all(data_dir)?;
tracing::info!(path = %data_dir.display(), "using filesystem backend");
let backend = FilesystemBackend::new(data_dir);
let state = State { store: backend };
cmd.run(&state).await
}
},
}
}

View File

@@ -0,0 +1,44 @@
use std::net::SocketAddr;
use std::path::PathBuf;
use clap::{Parser, ValueEnum};
use post3::StorageBackend;
use crate::s3::S3Server;
use crate::state::State;
#[derive(Clone, ValueEnum)]
pub enum BackendType {
/// PostgreSQL backend (requires DATABASE_URL)
Pg,
/// Local filesystem backend
Fs,
}
#[derive(Parser)]
pub struct ServeCommand {
#[arg(long, env = "POST3_HOST", default_value = "127.0.0.1:9000")]
pub host: SocketAddr,
/// Storage backend to use
#[arg(long, default_value = "pg")]
pub backend: BackendType,
/// Data directory for filesystem backend
#[arg(long)]
pub data_dir: Option<PathBuf>,
}
impl ServeCommand {
pub async fn run<B: StorageBackend>(&self, state: &State<B>) -> anyhow::Result<()> {
notmad::Mad::builder()
.add(S3Server {
host: self.host,
state: state.clone(),
})
.run()
.await?;
Ok(())
}
}

View File

@@ -0,0 +1,2 @@
pub mod s3;
pub mod state;

View File

@@ -0,0 +1,18 @@
mod cli;
pub mod s3;
pub mod state;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
dotenvy::dotenv().ok();
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive("post3_server=debug".parse()?)
.add_directive("post3=debug".parse()?),
)
.init();
cli::execute().await
}

View File

@@ -0,0 +1,55 @@
use serde::Deserialize;
/// Query params for GET /{bucket} — dispatches between ListObjectsV2, ListMultipartUploads,
/// ListObjectVersions, and GetBucketLocation.
#[derive(Debug, Default, Deserialize)]
pub struct BucketGetQuery {
/// Presence of `?uploads` signals ListMultipartUploads
pub uploads: Option<String>,
/// Presence of `?versions` signals ListObjectVersions
pub versions: Option<String>,
/// Presence of `?location` signals GetBucketLocation
pub location: Option<String>,
#[serde(rename = "list-type")]
pub list_type: Option<i32>,
pub prefix: Option<String>,
#[serde(rename = "max-keys")]
pub max_keys: Option<i64>,
#[serde(rename = "continuation-token")]
pub continuation_token: Option<String>,
#[serde(rename = "start-after")]
pub start_after: Option<String>,
/// ListObjects v1 pagination marker
pub marker: Option<String>,
pub delimiter: Option<String>,
#[serde(rename = "encoding-type")]
pub encoding_type: Option<String>,
#[serde(rename = "key-marker")]
pub key_marker: Option<String>,
#[serde(rename = "upload-id-marker")]
pub upload_id_marker: Option<String>,
#[serde(rename = "max-uploads")]
pub max_uploads: Option<i32>,
}
/// Query params for POST /{bucket} — dispatches between DeleteObjects and other ops.
#[derive(Debug, Default, Deserialize)]
pub struct BucketPostQuery {
/// Presence of `?delete` signals DeleteObjects
pub delete: Option<String>,
}
/// Query params for /{bucket}/{*key} dispatchers (PUT, GET, DELETE, POST).
#[derive(Debug, Default, Deserialize)]
pub struct ObjectKeyQuery {
#[serde(rename = "uploadId")]
pub upload_id: Option<String>,
#[serde(rename = "partNumber")]
pub part_number: Option<i32>,
/// Presence of `?uploads` signals CreateMultipartUpload (POST only)
pub uploads: Option<String>,
#[serde(rename = "max-parts")]
pub max_parts: Option<i32>,
#[serde(rename = "part-number-marker")]
pub part_number_marker: Option<i32>,
}

View File

@@ -0,0 +1,187 @@
use axum::{
extract::{Path, State},
http::StatusCode,
response::IntoResponse,
};
use post3::{Post3Error, StorageBackend};
use crate::s3::responses;
use crate::state::State as AppState;
fn is_valid_bucket_name(name: &str) -> bool {
let len = name.len();
if len < 3 || len > 63 {
return false;
}
// Must contain only lowercase letters, numbers, hyphens, and periods
if !name
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'.')
{
return false;
}
// Must start and end with a letter or number
let first = name.as_bytes()[0];
let last = name.as_bytes()[len - 1];
if !(first.is_ascii_lowercase() || first.is_ascii_digit()) {
return false;
}
if !(last.is_ascii_lowercase() || last.is_ascii_digit()) {
return false;
}
// Must not be formatted as an IP address
if name.split('.').count() == 4
&& name
.split('.')
.all(|part| part.parse::<u8>().is_ok())
{
return false;
}
true
}
pub async fn create_bucket<B: StorageBackend>(
State(state): State<AppState<B>>,
Path(bucket): Path<String>,
) -> impl IntoResponse {
if !is_valid_bucket_name(&bucket) {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml".to_string())],
responses::error_xml(
"InvalidBucketName",
"The specified bucket is not valid.",
&bucket,
),
)
.into_response();
}
match state.store.create_bucket(&bucket).await {
Ok(_) => (
StatusCode::OK,
[
("Location", format!("/{bucket}")),
(
"x-amz-request-id",
uuid::Uuid::new_v4().to_string(),
),
],
)
.into_response(),
Err(Post3Error::BucketAlreadyExists(_)) => (
StatusCode::CONFLICT,
[("Content-Type", "application/xml".to_string())],
responses::error_xml(
"BucketAlreadyOwnedByYou",
"Your previous request to create the named bucket succeeded and you already own it.",
&bucket,
),
)
.into_response(),
Err(e) => {
tracing::error!("create_bucket error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml".to_string())],
responses::error_xml("InternalError", &e.to_string(), &bucket),
)
.into_response()
}
}
}
pub async fn head_bucket<B: StorageBackend>(
State(state): State<AppState<B>>,
Path(bucket): Path<String>,
) -> impl IntoResponse {
match state.store.head_bucket(&bucket).await {
Ok(Some(_)) => (
StatusCode::OK,
[
("x-amz-request-id", uuid::Uuid::new_v4().to_string()),
("x-amz-bucket-region", "us-east-1".to_string()),
],
)
.into_response(),
Ok(None) => (
StatusCode::NOT_FOUND,
[("x-amz-request-id", uuid::Uuid::new_v4().to_string())],
)
.into_response(),
Err(e) => {
tracing::error!("head_bucket error: {e}");
StatusCode::INTERNAL_SERVER_ERROR.into_response()
}
}
}
pub async fn delete_bucket<B: StorageBackend>(
State(state): State<AppState<B>>,
Path(bucket): Path<String>,
) -> impl IntoResponse {
match state.store.delete_bucket(&bucket).await {
Ok(()) => (
StatusCode::NO_CONTENT,
[("x-amz-request-id", uuid::Uuid::new_v4().to_string())],
)
.into_response(),
Err(Post3Error::BucketNotFound(_)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml".to_string())],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&bucket,
),
)
.into_response(),
Err(Post3Error::BucketNotEmpty(_)) => (
StatusCode::CONFLICT,
[("Content-Type", "application/xml".to_string())],
responses::error_xml(
"BucketNotEmpty",
"The bucket you tried to delete is not empty",
&bucket,
),
)
.into_response(),
Err(e) => {
tracing::error!("delete_bucket error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml".to_string())],
responses::error_xml("InternalError", &e.to_string(), &bucket),
)
.into_response()
}
}
}
pub async fn list_buckets<B: StorageBackend>(
State(state): State<AppState<B>>,
) -> impl IntoResponse {
match state.store.list_buckets().await {
Ok(buckets) => (
StatusCode::OK,
[
("Content-Type", "application/xml".to_string()),
(
"x-amz-request-id",
uuid::Uuid::new_v4().to_string(),
),
],
responses::list_buckets_xml(&buckets),
)
.into_response(),
Err(e) => {
tracing::error!("list_buckets error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml".to_string())],
responses::error_xml("InternalError", &e.to_string(), "/"),
)
.into_response()
}
}
}

View File

@@ -0,0 +1,3 @@
pub mod buckets;
pub mod multipart;
pub mod objects;

View File

@@ -0,0 +1,509 @@
use std::collections::HashMap;
use axum::{
extract::{Path, Query, State},
http::{HeaderMap, HeaderValue, StatusCode},
response::{IntoResponse, Response},
};
use bytes::Bytes;
use post3::{Post3Error, StorageBackend};
use crate::s3::extractors::{BucketGetQuery, ObjectKeyQuery};
use crate::s3::responses;
use crate::state::State as AppState;
pub async fn create_multipart_upload<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
headers: HeaderMap,
) -> Response {
let content_type = headers
.get("content-type")
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
let mut metadata = HashMap::new();
for (name, value) in headers.iter() {
let name_str = name.as_str();
if let Some(meta_key) = name_str.strip_prefix("x-amz-meta-") {
if let Ok(v) = value.to_str() {
metadata.insert(meta_key.to_string(), v.to_string());
}
}
}
match state
.store
.create_multipart_upload(&bucket, &key, content_type.as_deref(), metadata)
.await
{
Ok(result) => {
let mut response_headers = HeaderMap::new();
response_headers
.insert("Content-Type", HeaderValue::from_static("application/xml"));
response_headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(
StatusCode::OK,
response_headers,
responses::initiate_multipart_upload_xml(
&result.bucket,
&result.key,
&result.upload_id,
),
)
.into_response()
}
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&b,
),
)
.into_response(),
Err(e) => {
tracing::error!("create_multipart_upload error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml(
"InternalError",
&e.to_string(),
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
}
}
pub async fn upload_part<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
Query(query): Query<ObjectKeyQuery>,
body: Bytes,
) -> Response {
let upload_id = match &query.upload_id {
Some(id) => id.clone(),
None => {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidRequest",
"Missing uploadId parameter",
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
};
let part_number = match query.part_number {
Some(n) => n,
None => {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidRequest",
"Missing partNumber parameter",
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
};
match state
.store
.upload_part(&bucket, &key, &upload_id, part_number, body)
.await
{
Ok(result) => {
let mut headers = HeaderMap::new();
headers.insert("ETag", result.etag.parse().unwrap());
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(StatusCode::OK, headers).into_response()
}
Err(Post3Error::UploadNotFound(id)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchUpload",
"The specified multipart upload does not exist",
&id,
),
)
.into_response(),
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&b,
),
)
.into_response(),
Err(e) => {
tracing::error!("upload_part error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml(
"InternalError",
&e.to_string(),
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
}
}
pub async fn complete_multipart_upload<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
Query(query): Query<ObjectKeyQuery>,
body: Bytes,
) -> Response {
let upload_id = match &query.upload_id {
Some(id) => id.clone(),
None => {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidRequest",
"Missing uploadId parameter",
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
};
let part_etags = match responses::parse_complete_multipart_xml(&body) {
Ok(parts) => parts,
Err(msg) => {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml("MalformedXML", &msg, &format!("/{bucket}/{key}")),
)
.into_response()
}
};
match state
.store
.complete_multipart_upload(&bucket, &key, &upload_id, part_etags)
.await
{
Ok(result) => {
let location = format!("/{}/{}", bucket, key);
let mut headers = HeaderMap::new();
headers
.insert("Content-Type", HeaderValue::from_static("application/xml"));
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(
StatusCode::OK,
headers,
responses::complete_multipart_upload_xml(
&location,
&result.bucket,
&result.key,
&result.etag,
),
)
.into_response()
}
Err(Post3Error::UploadNotFound(id)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchUpload",
"The specified multipart upload does not exist",
&id,
),
)
.into_response(),
Err(Post3Error::InvalidPart {
upload_id: _,
part_number,
}) => (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidPart",
&format!("Part {part_number} not found or not uploaded"),
&format!("/{bucket}/{key}"),
),
)
.into_response(),
Err(Post3Error::ETagMismatch {
part_number,
expected,
got,
}) => (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidPart",
&format!(
"ETag mismatch for part {part_number}: expected {expected}, got {got}"
),
&format!("/{bucket}/{key}"),
),
)
.into_response(),
Err(Post3Error::InvalidPartOrder) => (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidPartOrder",
"Parts must be in ascending order",
&format!("/{bucket}/{key}"),
),
)
.into_response(),
Err(Post3Error::EntityTooSmall {
part_number,
size,
}) => (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"EntityTooSmall",
&format!(
"Your proposed upload is smaller than the minimum allowed size. Part {part_number} has size {size}."
),
&format!("/{bucket}/{key}"),
),
)
.into_response(),
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&b,
),
)
.into_response(),
Err(e) => {
tracing::error!("complete_multipart_upload error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml(
"InternalError",
&e.to_string(),
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
}
}
pub async fn abort_multipart_upload<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
Query(query): Query<ObjectKeyQuery>,
) -> Response {
let upload_id = match &query.upload_id {
Some(id) => id.clone(),
None => {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidRequest",
"Missing uploadId parameter",
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
};
match state
.store
.abort_multipart_upload(&bucket, &key, &upload_id)
.await
{
Ok(()) => {
let mut headers = HeaderMap::new();
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(StatusCode::NO_CONTENT, headers).into_response()
}
Err(Post3Error::UploadNotFound(id)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchUpload",
"The specified multipart upload does not exist",
&id,
),
)
.into_response(),
Err(e) => {
tracing::error!("abort_multipart_upload error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml(
"InternalError",
&e.to_string(),
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
}
}
pub async fn list_parts<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
Query(query): Query<ObjectKeyQuery>,
) -> Response {
let upload_id = match &query.upload_id {
Some(id) => id.clone(),
None => {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidRequest",
"Missing uploadId parameter",
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
};
match state
.store
.list_parts(
&bucket,
&key,
&upload_id,
query.max_parts,
query.part_number_marker,
)
.await
{
Ok(result) => {
let max_parts = query.max_parts.unwrap_or(1000);
let mut headers = HeaderMap::new();
headers
.insert("Content-Type", HeaderValue::from_static("application/xml"));
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(
StatusCode::OK,
headers,
responses::list_parts_xml(&result, max_parts),
)
.into_response()
}
Err(Post3Error::UploadNotFound(id)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchUpload",
"The specified multipart upload does not exist",
&id,
),
)
.into_response(),
Err(e) => {
tracing::error!("list_parts error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml(
"InternalError",
&e.to_string(),
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
}
}
pub async fn list_multipart_uploads<B: StorageBackend>(
State(state): State<AppState<B>>,
Path(bucket): Path<String>,
Query(query): Query<BucketGetQuery>,
) -> Response {
match state
.store
.list_multipart_uploads(
&bucket,
query.prefix.as_deref(),
query.key_marker.as_deref(),
query.upload_id_marker.as_deref(),
query.max_uploads,
)
.await
{
Ok(result) => {
let max_uploads = query.max_uploads.unwrap_or(1000);
let mut headers = HeaderMap::new();
headers
.insert("Content-Type", HeaderValue::from_static("application/xml"));
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(
StatusCode::OK,
headers,
responses::list_multipart_uploads_xml(&result, max_uploads),
)
.into_response()
}
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&b,
),
)
.into_response(),
Err(e) => {
tracing::error!("list_multipart_uploads error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml("InternalError", &e.to_string(), &bucket),
)
.into_response()
}
}
}

View File

@@ -0,0 +1,598 @@
use std::collections::HashMap;
use axum::{
body::Body,
extract::{Path, Query, State},
http::{header::HeaderName, HeaderMap, HeaderValue, StatusCode},
response::{IntoResponse, Response},
};
use bytes::Bytes;
use post3::{Post3Error, StorageBackend};
use crate::s3::extractors::{BucketGetQuery, ObjectKeyQuery};
use crate::s3::handlers::multipart;
use crate::s3::responses;
use crate::state::State as AppState;
// --- Dispatch functions ---
/// PUT /{bucket}/{*key} — dispatches to upload_part or put_object based on query params.
pub async fn put_dispatch<B: StorageBackend>(
state: State<AppState<B>>,
path: Path<(String, String)>,
query: Query<ObjectKeyQuery>,
headers: HeaderMap,
body: Bytes,
) -> Response {
if query.upload_id.is_some() && query.part_number.is_some() {
multipart::upload_part(state, path, query, body).await
} else {
put_object(state, path, headers, body).await
}
}
/// GET /{bucket}/{*key} — dispatches to list_parts or get_object based on query params.
pub async fn get_dispatch<B: StorageBackend>(
state: State<AppState<B>>,
path: Path<(String, String)>,
query: Query<ObjectKeyQuery>,
) -> Response {
if query.upload_id.is_some() {
multipart::list_parts(state, path, query).await
} else {
get_object(state, path).await
}
}
/// DELETE /{bucket}/{*key} — dispatches to abort_multipart_upload or delete_object.
pub async fn delete_dispatch<B: StorageBackend>(
state: State<AppState<B>>,
path: Path<(String, String)>,
query: Query<ObjectKeyQuery>,
) -> Response {
if query.upload_id.is_some() {
multipart::abort_multipart_upload(state, path, query).await
} else {
delete_object(state, path).await
}
}
/// POST /{bucket}/{*key} — dispatches to create_multipart_upload or complete_multipart_upload.
pub async fn post_dispatch<B: StorageBackend>(
state: State<AppState<B>>,
path: Path<(String, String)>,
query: Query<ObjectKeyQuery>,
headers: HeaderMap,
body: Bytes,
) -> Response {
if query.uploads.is_some() {
multipart::create_multipart_upload(state, path, headers).await
} else if query.upload_id.is_some() {
multipart::complete_multipart_upload(state, path, query, body).await
} else {
(
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidRequest",
"POST requires ?uploads or ?uploadId parameter",
&format!("/{}/{}", path.0 .0, path.0 .1),
),
)
.into_response()
}
}
// --- Object handlers ---
pub async fn put_object<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
headers: HeaderMap,
body: Bytes,
) -> Response {
let content_type = headers
.get("content-type")
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
// Extract x-amz-meta-* user metadata
let mut metadata = HashMap::new();
for (name, value) in headers.iter() {
let name_str = name.as_str();
if let Some(meta_key) = name_str.strip_prefix("x-amz-meta-") {
if let Ok(v) = value.to_str() {
metadata.insert(meta_key.to_string(), v.to_string());
}
}
}
match state
.store
.put_object(&bucket, &key, content_type.as_deref(), metadata, body)
.await
{
Ok(result) => {
let mut response_headers = HeaderMap::new();
response_headers.insert("ETag", result.etag.parse().unwrap());
response_headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(StatusCode::OK, response_headers).into_response()
}
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&b,
),
)
.into_response(),
Err(e) => {
tracing::error!("put_object error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml(
"InternalError",
&e.to_string(),
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
}
}
pub async fn get_object<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
) -> Response {
match state.store.get_object(&bucket, &key).await {
Ok(result) => {
let mut headers = HeaderMap::new();
headers.insert(
"Content-Type",
HeaderValue::from_str(&result.metadata.content_type).unwrap(),
);
headers.insert(
"Content-Length",
HeaderValue::from_str(&result.metadata.size.to_string()).unwrap(),
);
headers.insert("ETag", HeaderValue::from_str(&result.metadata.etag).unwrap());
headers.insert(
"Last-Modified",
HeaderValue::from_str(
&result
.metadata
.last_modified
.format("%a, %d %b %Y %H:%M:%S GMT")
.to_string(),
)
.unwrap(),
);
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
// Return user metadata as x-amz-meta-* headers
for (k, v) in &result.user_metadata {
let header_name = format!("x-amz-meta-{k}");
if let (Ok(name), Ok(val)) = (
header_name.parse::<HeaderName>(),
HeaderValue::from_str(v),
) {
headers.insert(name, val);
}
}
(StatusCode::OK, headers, Body::from(result.body)).into_response()
}
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&b,
),
)
.into_response(),
Err(Post3Error::ObjectNotFound { bucket: b, key: k }) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchKey",
"The specified key does not exist.",
&format!("/{b}/{k}"),
),
)
.into_response(),
Err(e) => {
tracing::error!("get_object error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml(
"InternalError",
&e.to_string(),
&format!("/{bucket}/{key}"),
),
)
.into_response()
}
}
}
pub async fn head_object<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
) -> Response {
match state.store.head_object(&bucket, &key).await {
Ok(Some(result)) => {
let mut headers = HeaderMap::new();
headers.insert(
"Content-Type",
HeaderValue::from_str(&result.object.content_type).unwrap(),
);
headers.insert(
"Content-Length",
HeaderValue::from_str(&result.object.size.to_string()).unwrap(),
);
headers.insert("ETag", HeaderValue::from_str(&result.object.etag).unwrap());
headers.insert(
"Last-Modified",
HeaderValue::from_str(
&result
.object
.last_modified
.format("%a, %d %b %Y %H:%M:%S GMT")
.to_string(),
)
.unwrap(),
);
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
for (k, v) in &result.user_metadata {
let header_name = format!("x-amz-meta-{k}");
if let (Ok(name), Ok(val)) = (
header_name.parse::<HeaderName>(),
HeaderValue::from_str(v),
) {
headers.insert(name, val);
}
}
(StatusCode::OK, headers).into_response()
}
Ok(None) => {
let mut headers = HeaderMap::new();
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(StatusCode::NOT_FOUND, headers).into_response()
}
Err(e) => {
tracing::error!("head_object error: {e}");
StatusCode::INTERNAL_SERVER_ERROR.into_response()
}
}
}
pub async fn delete_object<B: StorageBackend>(
State(state): State<AppState<B>>,
Path((bucket, key)): Path<(String, String)>,
) -> Response {
match state.store.delete_object(&bucket, &key).await {
Ok(()) => {
let mut headers = HeaderMap::new();
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(StatusCode::NO_CONTENT, headers).into_response()
}
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&b,
),
)
.into_response(),
Err(e) => {
tracing::error!("delete_object error: {e}");
StatusCode::INTERNAL_SERVER_ERROR.into_response()
}
}
}
/// Handles GET /{bucket} — dispatches to ListMultipartUploads, ListObjectVersions,
/// GetBucketLocation, or ListObjects (v1/v2).
pub async fn list_or_get<B: StorageBackend>(
State(state): State<AppState<B>>,
Path(bucket): Path<String>,
Query(query): Query<BucketGetQuery>,
) -> Response {
// ?uploads → ListMultipartUploads
if query.uploads.is_some() {
return multipart::list_multipart_uploads(
State(state),
Path(bucket),
Query(query),
)
.await;
}
// ?location → GetBucketLocation
if query.location.is_some() {
return get_bucket_location(State(state), Path(bucket)).await;
}
// ?versions → ListObjectVersions
if query.versions.is_some() {
return list_object_versions(State(state), Path(bucket), Query(query)).await;
}
// Default: ListObjects (v1 or v2)
let is_v2 = query.list_type == Some(2);
let continuation_token = if is_v2 {
// v2: use continuation-token if present, else start-after
query
.continuation_token
.as_deref()
.or(query.start_after.as_deref())
} else {
query.marker.as_deref()
};
// Treat empty delimiter as absent (S3 spec: empty delimiter = no delimiter)
let delimiter = query
.delimiter
.as_deref()
.filter(|d| !d.is_empty());
match state
.store
.list_objects_v2(
&bucket,
query.prefix.as_deref(),
continuation_token,
query.max_keys,
delimiter,
)
.await
{
Ok(result) => {
let max_keys = query.max_keys.unwrap_or(1000);
let mut headers = HeaderMap::new();
headers.insert("Content-Type", HeaderValue::from_static("application/xml"));
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
let xml = if is_v2 {
responses::list_objects_v2_xml(
&bucket,
&result,
max_keys,
query.continuation_token.as_deref(),
query.start_after.as_deref(),
)
} else {
responses::list_objects_v1_xml(
&bucket,
&result,
max_keys,
query.marker.as_deref(),
)
};
(StatusCode::OK, headers, xml).into_response()
}
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&b,
),
)
.into_response(),
Err(e) => {
tracing::error!("list_objects error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml("InternalError", &e.to_string(), &bucket),
)
.into_response()
}
}
}
/// GET /{bucket}?versions — ListObjectVersions (stub: returns all as version "null").
async fn list_object_versions<B: StorageBackend>(
State(state): State<AppState<B>>,
Path(bucket): Path<String>,
Query(query): Query<BucketGetQuery>,
) -> Response {
let delimiter = query.delimiter.as_deref().filter(|d| !d.is_empty());
match state
.store
.list_objects_v2(
&bucket,
query.prefix.as_deref(),
query.key_marker.as_deref(),
query.max_keys,
delimiter,
)
.await
{
Ok(result) => {
let max_keys = query.max_keys.unwrap_or(1000);
let mut headers = HeaderMap::new();
headers.insert("Content-Type", HeaderValue::from_static("application/xml"));
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(
StatusCode::OK,
headers,
responses::list_object_versions_xml(
&bucket,
&result,
max_keys,
query.key_marker.as_deref(),
),
)
.into_response()
}
Err(Post3Error::BucketNotFound(b)) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml("NoSuchBucket", "The specified bucket does not exist", &b),
)
.into_response(),
Err(e) => {
tracing::error!("list_object_versions error: {e}");
(
StatusCode::INTERNAL_SERVER_ERROR,
[("Content-Type", "application/xml")],
responses::error_xml("InternalError", &e.to_string(), &bucket),
)
.into_response()
}
}
}
/// GET /{bucket}?location — GetBucketLocation.
async fn get_bucket_location<B: StorageBackend>(
State(state): State<AppState<B>>,
Path(bucket): Path<String>,
) -> Response {
match state.store.head_bucket(&bucket).await {
Ok(Some(_)) => {
let mut headers = HeaderMap::new();
headers.insert("Content-Type", HeaderValue::from_static("application/xml"));
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(StatusCode::OK, headers, responses::get_bucket_location_xml()).into_response()
}
Ok(None) => (
StatusCode::NOT_FOUND,
[("Content-Type", "application/xml")],
responses::error_xml(
"NoSuchBucket",
"The specified bucket does not exist",
&bucket,
),
)
.into_response(),
Err(e) => {
tracing::error!("get_bucket_location error: {e}");
StatusCode::INTERNAL_SERVER_ERROR.into_response()
}
}
}
/// POST /{bucket} — dispatches to DeleteObjects based on ?delete query param.
pub async fn bucket_post_dispatch<B: StorageBackend>(
state: State<AppState<B>>,
path: Path<String>,
query: Query<crate::s3::extractors::BucketPostQuery>,
body: Bytes,
) -> Response {
if query.delete.is_some() {
delete_objects(state, path, body).await
} else {
(
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"InvalidRequest",
"POST on bucket requires ?delete parameter",
&format!("/{}", path.0),
),
)
.into_response()
}
}
/// POST /{bucket}?delete — DeleteObjects (batch delete).
async fn delete_objects<B: StorageBackend>(
State(state): State<AppState<B>>,
Path(bucket): Path<String>,
body: Bytes,
) -> Response {
let (keys, quiet) = match responses::parse_delete_objects_xml(&body) {
Ok(result) => result,
Err(msg) => {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml("MalformedXML", &msg, &format!("/{bucket}")),
)
.into_response();
}
};
// S3 limits DeleteObjects to 1000 keys
if keys.len() > 1000 {
return (
StatusCode::BAD_REQUEST,
[("Content-Type", "application/xml")],
responses::error_xml(
"MalformedXML",
"The number of keys in a DeleteObjects request cannot exceed 1000",
&format!("/{bucket}"),
),
)
.into_response();
}
let mut deleted = Vec::new();
let mut errors: Vec<(String, String, String)> = Vec::new();
for key in keys {
match state.store.delete_object(&bucket, &key).await {
Ok(()) => {
if !quiet {
deleted.push(key);
}
}
Err(e) => {
errors.push((key, "InternalError".to_string(), e.to_string()));
}
}
}
let mut headers = HeaderMap::new();
headers.insert("Content-Type", HeaderValue::from_static("application/xml"));
headers.insert(
"x-amz-request-id",
HeaderValue::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(),
);
(
StatusCode::OK,
headers,
responses::delete_objects_result_xml(&deleted, &errors),
)
.into_response()
}

View File

@@ -0,0 +1,42 @@
pub mod extractors;
pub mod handlers;
pub mod responses;
pub mod router;
use std::net::SocketAddr;
use notmad::{Component, ComponentInfo, MadError};
use post3::StorageBackend;
use tokio::net::TcpListener;
use tokio_util::sync::CancellationToken;
use crate::state::State;
pub struct S3Server<B: StorageBackend> {
pub host: SocketAddr,
pub state: State<B>,
}
impl<B: StorageBackend> Component for S3Server<B> {
fn info(&self) -> ComponentInfo {
"post3/s3".into()
}
async fn run(&self, cancellation_token: CancellationToken) -> Result<(), MadError> {
let app = router::build_router(self.state.clone());
tracing::info!("post3 s3-compatible server listening on {}", self.host);
let listener = TcpListener::bind(&self.host).await.map_err(|e| {
MadError::Inner(anyhow::anyhow!("failed to bind: {e}"))
})?;
axum::serve(listener, app.into_make_service())
.with_graceful_shutdown(async move {
cancellation_token.cancelled().await;
})
.await
.map_err(|e| MadError::Inner(anyhow::anyhow!("server error: {e}")))?;
Ok(())
}
}

View File

@@ -0,0 +1,538 @@
use post3::models::{BucketInfo, ListMultipartUploadsResult, ListObjectsResult, ListPartsResult};
use serde::Deserialize;
pub fn list_buckets_xml(buckets: &[BucketInfo]) -> String {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
<Owner><ID>post3</ID><DisplayName>post3</DisplayName></Owner>\
<Buckets>",
);
for b in buckets {
xml.push_str("<Bucket><Name>");
xml.push_str(&xml_escape(&b.name));
xml.push_str("</Name><CreationDate>");
xml.push_str(&b.created_at.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string());
xml.push_str("</CreationDate></Bucket>");
}
xml.push_str("</Buckets></ListAllMyBucketsResult>");
xml
}
pub fn list_objects_v2_xml(
bucket_name: &str,
result: &ListObjectsResult,
max_keys: i64,
continuation_token: Option<&str>,
start_after: Option<&str>,
) -> String {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
);
xml.push_str("<Name>");
xml.push_str(&xml_escape(bucket_name));
xml.push_str("</Name>");
xml.push_str("<Prefix>");
if let Some(ref pfx) = result.prefix {
xml.push_str(&xml_escape(pfx));
}
xml.push_str("</Prefix>");
if let Some(sa) = start_after {
xml.push_str("<StartAfter>");
xml.push_str(&xml_escape(sa));
xml.push_str("</StartAfter>");
}
xml.push_str("<KeyCount>");
xml.push_str(&result.key_count.to_string());
xml.push_str("</KeyCount>");
xml.push_str("<MaxKeys>");
xml.push_str(&max_keys.to_string());
xml.push_str("</MaxKeys>");
xml.push_str("<IsTruncated>");
xml.push_str(if result.is_truncated { "true" } else { "false" });
xml.push_str("</IsTruncated>");
if let Some(ref delim) = result.delimiter {
xml.push_str("<Delimiter>");
xml.push_str(&xml_escape(delim));
xml.push_str("</Delimiter>");
}
if let Some(token) = continuation_token {
xml.push_str("<ContinuationToken>");
xml.push_str(&xml_escape(token));
xml.push_str("</ContinuationToken>");
}
if let Some(ref token) = result.next_continuation_token {
xml.push_str("<NextContinuationToken>");
xml.push_str(&xml_escape(token));
xml.push_str("</NextContinuationToken>");
}
for obj in &result.objects {
xml.push_str("<Contents>");
xml.push_str("<Key>");
xml.push_str(&xml_escape(&obj.key));
xml.push_str("</Key>");
xml.push_str("<LastModified>");
xml.push_str(
&obj.last_modified
.format("%Y-%m-%dT%H:%M:%S%.3fZ")
.to_string(),
);
xml.push_str("</LastModified>");
xml.push_str("<ETag>");
xml.push_str(&xml_escape(&obj.etag));
xml.push_str("</ETag>");
xml.push_str("<Size>");
xml.push_str(&obj.size.to_string());
xml.push_str("</Size>");
xml.push_str("<StorageClass>STANDARD</StorageClass>");
xml.push_str("</Contents>");
}
for cp in &result.common_prefixes {
xml.push_str("<CommonPrefixes><Prefix>");
xml.push_str(&xml_escape(cp));
xml.push_str("</Prefix></CommonPrefixes>");
}
xml.push_str("</ListBucketResult>");
xml
}
pub fn list_objects_v1_xml(
bucket_name: &str,
result: &ListObjectsResult,
max_keys: i64,
marker: Option<&str>,
) -> String {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
);
xml.push_str("<Name>");
xml.push_str(&xml_escape(bucket_name));
xml.push_str("</Name>");
xml.push_str("<Prefix>");
if let Some(ref pfx) = result.prefix {
xml.push_str(&xml_escape(pfx));
}
xml.push_str("</Prefix>");
xml.push_str("<Marker>");
if let Some(m) = marker {
xml.push_str(&xml_escape(m));
}
xml.push_str("</Marker>");
xml.push_str("<MaxKeys>");
xml.push_str(&max_keys.to_string());
xml.push_str("</MaxKeys>");
xml.push_str("<IsTruncated>");
xml.push_str(if result.is_truncated { "true" } else { "false" });
xml.push_str("</IsTruncated>");
if let Some(ref token) = result.next_continuation_token {
xml.push_str("<NextMarker>");
xml.push_str(&xml_escape(token));
xml.push_str("</NextMarker>");
}
if let Some(ref delim) = result.delimiter {
xml.push_str("<Delimiter>");
xml.push_str(&xml_escape(delim));
xml.push_str("</Delimiter>");
}
for obj in &result.objects {
xml.push_str("<Contents>");
xml.push_str("<Key>");
xml.push_str(&xml_escape(&obj.key));
xml.push_str("</Key>");
xml.push_str("<LastModified>");
xml.push_str(
&obj.last_modified
.format("%Y-%m-%dT%H:%M:%S%.3fZ")
.to_string(),
);
xml.push_str("</LastModified>");
xml.push_str("<ETag>");
xml.push_str(&xml_escape(&obj.etag));
xml.push_str("</ETag>");
xml.push_str("<Size>");
xml.push_str(&obj.size.to_string());
xml.push_str("</Size>");
xml.push_str("<Owner><ID>post3</ID><DisplayName>post3</DisplayName></Owner>");
xml.push_str("<StorageClass>STANDARD</StorageClass>");
xml.push_str("</Contents>");
}
for cp in &result.common_prefixes {
xml.push_str("<CommonPrefixes><Prefix>");
xml.push_str(&xml_escape(cp));
xml.push_str("</Prefix></CommonPrefixes>");
}
xml.push_str("</ListBucketResult>");
xml
}
pub fn list_object_versions_xml(
bucket_name: &str,
result: &ListObjectsResult,
max_keys: i64,
key_marker: Option<&str>,
) -> String {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
);
xml.push_str("<Name>");
xml.push_str(&xml_escape(bucket_name));
xml.push_str("</Name>");
xml.push_str("<Prefix>");
if let Some(ref pfx) = result.prefix {
xml.push_str(&xml_escape(pfx));
}
xml.push_str("</Prefix>");
// Echo back input markers
xml.push_str("<KeyMarker>");
if let Some(km) = key_marker {
xml.push_str(&xml_escape(km));
}
xml.push_str("</KeyMarker>");
xml.push_str("<VersionIdMarker/>");
xml.push_str("<MaxKeys>");
xml.push_str(&max_keys.to_string());
xml.push_str("</MaxKeys>");
xml.push_str("<IsTruncated>");
xml.push_str(if result.is_truncated { "true" } else { "false" });
xml.push_str("</IsTruncated>");
for obj in &result.objects {
xml.push_str("<Version>");
xml.push_str("<Key>");
xml.push_str(&xml_escape(&obj.key));
xml.push_str("</Key>");
xml.push_str("<VersionId>null</VersionId>");
xml.push_str("<IsLatest>true</IsLatest>");
xml.push_str("<LastModified>");
xml.push_str(
&obj.last_modified
.format("%Y-%m-%dT%H:%M:%S%.3fZ")
.to_string(),
);
xml.push_str("</LastModified>");
xml.push_str("<ETag>");
xml.push_str(&xml_escape(&obj.etag));
xml.push_str("</ETag>");
xml.push_str("<Size>");
xml.push_str(&obj.size.to_string());
xml.push_str("</Size>");
xml.push_str("<StorageClass>STANDARD</StorageClass>");
xml.push_str("<Owner><ID>post3</ID><DisplayName>post3</DisplayName></Owner>");
xml.push_str("</Version>");
}
// Include NextKeyMarker/NextVersionIdMarker when truncated for pagination
if result.is_truncated {
if let Some(last_obj) = result.objects.last() {
xml.push_str("<NextKeyMarker>");
xml.push_str(&xml_escape(&last_obj.key));
xml.push_str("</NextKeyMarker>");
xml.push_str("<NextVersionIdMarker>null</NextVersionIdMarker>");
}
}
xml.push_str("</ListVersionsResult>");
xml
}
pub fn get_bucket_location_xml() -> String {
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>"
.to_string()
}
// --- DeleteObjects ---
#[derive(Debug, Deserialize)]
#[serde(rename = "Delete")]
struct DeleteObjectsRequest {
#[serde(rename = "Object")]
objects: Vec<DeleteObjectEntry>,
#[serde(rename = "Quiet", default)]
quiet: Option<bool>,
}
#[derive(Debug, Deserialize)]
struct DeleteObjectEntry {
#[serde(rename = "Key")]
key: String,
}
pub fn parse_delete_objects_xml(body: &[u8]) -> Result<(Vec<String>, bool), String> {
let request: DeleteObjectsRequest =
quick_xml::de::from_reader(body).map_err(|e| format!("invalid XML: {e}"))?;
let quiet = request.quiet.unwrap_or(false);
let keys = request.objects.into_iter().map(|o| o.key).collect();
Ok((keys, quiet))
}
pub fn delete_objects_result_xml(deleted: &[String], errors: &[(String, String, String)]) -> String {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
);
for key in deleted {
xml.push_str("<Deleted><Key>");
xml.push_str(&xml_escape(key));
xml.push_str("</Key></Deleted>");
}
for (key, code, message) in errors {
xml.push_str("<Error><Key>");
xml.push_str(&xml_escape(key));
xml.push_str("</Key><Code>");
xml.push_str(&xml_escape(code));
xml.push_str("</Code><Message>");
xml.push_str(&xml_escape(message));
xml.push_str("</Message></Error>");
}
xml.push_str("</DeleteResult>");
xml
}
pub fn error_xml(code: &str, message: &str, resource: &str) -> String {
let request_id = uuid::Uuid::new_v4().to_string();
format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<Error>\
<Code>{code}</Code>\
<Message>{message}</Message>\
<Resource>{resource}</Resource>\
<RequestId>{request_id}</RequestId>\
</Error>",
code = xml_escape(code),
message = xml_escape(message),
resource = xml_escape(resource),
request_id = request_id,
)
}
// --- Multipart upload responses ---
pub fn initiate_multipart_upload_xml(bucket: &str, key: &str, upload_id: &str) -> String {
format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
<Bucket>{bucket}</Bucket>\
<Key>{key}</Key>\
<UploadId>{upload_id}</UploadId>\
</InitiateMultipartUploadResult>",
bucket = xml_escape(bucket),
key = xml_escape(key),
upload_id = xml_escape(upload_id),
)
}
pub fn complete_multipart_upload_xml(
location: &str,
bucket: &str,
key: &str,
etag: &str,
) -> String {
format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
<Location>{location}</Location>\
<Bucket>{bucket}</Bucket>\
<Key>{key}</Key>\
<ETag>{etag}</ETag>\
</CompleteMultipartUploadResult>",
location = xml_escape(location),
bucket = xml_escape(bucket),
key = xml_escape(key),
etag = xml_escape(etag),
)
}
pub fn list_parts_xml(result: &ListPartsResult, max_parts: i32) -> String {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<ListPartsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
);
xml.push_str("<Bucket>");
xml.push_str(&xml_escape(&result.bucket));
xml.push_str("</Bucket>");
xml.push_str("<Key>");
xml.push_str(&xml_escape(&result.key));
xml.push_str("</Key>");
xml.push_str("<UploadId>");
xml.push_str(&xml_escape(&result.upload_id));
xml.push_str("</UploadId>");
xml.push_str("<MaxParts>");
xml.push_str(&max_parts.to_string());
xml.push_str("</MaxParts>");
xml.push_str("<IsTruncated>");
xml.push_str(if result.is_truncated { "true" } else { "false" });
xml.push_str("</IsTruncated>");
if let Some(marker) = result.next_part_number_marker {
xml.push_str("<NextPartNumberMarker>");
xml.push_str(&marker.to_string());
xml.push_str("</NextPartNumberMarker>");
}
for part in &result.parts {
xml.push_str("<Part>");
xml.push_str("<PartNumber>");
xml.push_str(&part.part_number.to_string());
xml.push_str("</PartNumber>");
xml.push_str("<LastModified>");
xml.push_str(
&part
.created_at
.format("%Y-%m-%dT%H:%M:%S%.3fZ")
.to_string(),
);
xml.push_str("</LastModified>");
xml.push_str("<ETag>");
xml.push_str(&xml_escape(&part.etag));
xml.push_str("</ETag>");
xml.push_str("<Size>");
xml.push_str(&part.size.to_string());
xml.push_str("</Size>");
xml.push_str("</Part>");
}
xml.push_str("</ListPartsResult>");
xml
}
pub fn list_multipart_uploads_xml(
result: &ListMultipartUploadsResult,
max_uploads: i32,
) -> String {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<ListMultipartUploadsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
);
xml.push_str("<Bucket>");
xml.push_str(&xml_escape(&result.bucket));
xml.push_str("</Bucket>");
xml.push_str("<Prefix>");
if let Some(ref pfx) = result.prefix {
xml.push_str(&xml_escape(pfx));
}
xml.push_str("</Prefix>");
xml.push_str("<MaxUploads>");
xml.push_str(&max_uploads.to_string());
xml.push_str("</MaxUploads>");
xml.push_str("<IsTruncated>");
xml.push_str(if result.is_truncated {
"true"
} else {
"false"
});
xml.push_str("</IsTruncated>");
if let Some(ref marker) = result.next_key_marker {
xml.push_str("<NextKeyMarker>");
xml.push_str(&xml_escape(marker));
xml.push_str("</NextKeyMarker>");
}
if let Some(ref marker) = result.next_upload_id_marker {
xml.push_str("<NextUploadIdMarker>");
xml.push_str(&xml_escape(marker));
xml.push_str("</NextUploadIdMarker>");
}
for upload in &result.uploads {
xml.push_str("<Upload>");
xml.push_str("<Key>");
xml.push_str(&xml_escape(&upload.key));
xml.push_str("</Key>");
xml.push_str("<UploadId>");
xml.push_str(&xml_escape(&upload.upload_id));
xml.push_str("</UploadId>");
xml.push_str("<Initiated>");
xml.push_str(
&upload
.initiated
.format("%Y-%m-%dT%H:%M:%S%.3fZ")
.to_string(),
);
xml.push_str("</Initiated>");
xml.push_str("</Upload>");
}
xml.push_str("</ListMultipartUploadsResult>");
xml
}
// --- XML request parsing for CompleteMultipartUpload ---
#[derive(Debug, Deserialize)]
#[serde(rename = "CompleteMultipartUpload")]
struct CompleteMultipartUploadRequest {
#[serde(rename = "Part")]
parts: Vec<CompletePart>,
}
#[derive(Debug, Deserialize)]
struct CompletePart {
#[serde(rename = "PartNumber")]
part_number: i32,
#[serde(rename = "ETag")]
etag: String,
}
pub fn parse_complete_multipart_xml(body: &[u8]) -> Result<Vec<(i32, String)>, String> {
let request: CompleteMultipartUploadRequest =
quick_xml::de::from_reader(body).map_err(|e| format!("invalid XML: {e}"))?;
Ok(request
.parts
.into_iter()
.map(|p| (p.part_number, p.etag))
.collect())
}
fn xml_escape(s: &str) -> String {
s.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&apos;")
}

View File

@@ -0,0 +1,48 @@
use axum::{
extract::{DefaultBodyLimit, Request},
http::StatusCode,
response::IntoResponse,
routing::{delete, get, head, post, put},
Router,
};
use post3::StorageBackend;
use tower_http::trace::TraceLayer;
use super::handlers::{buckets, objects};
use crate::state::State;
pub fn build_router<B: StorageBackend>(state: State<B>) -> Router {
Router::new()
// Service-level
.route("/", get(buckets::list_buckets::<B>))
// Bucket-level (with and without trailing slash for SDK compat)
.route("/{bucket}", put(buckets::create_bucket::<B>))
.route("/{bucket}/", put(buckets::create_bucket::<B>))
.route("/{bucket}", head(buckets::head_bucket::<B>))
.route("/{bucket}/", head(buckets::head_bucket::<B>))
.route("/{bucket}", delete(buckets::delete_bucket::<B>))
.route("/{bucket}/", delete(buckets::delete_bucket::<B>))
.route("/{bucket}", get(objects::list_or_get::<B>))
.route("/{bucket}/", get(objects::list_or_get::<B>))
.route("/{bucket}", post(objects::bucket_post_dispatch::<B>))
.route("/{bucket}/", post(objects::bucket_post_dispatch::<B>))
// Object-level (wildcard key for nested paths like "a/b/c")
.route("/{bucket}/{*key}", put(objects::put_dispatch::<B>))
.route("/{bucket}/{*key}", get(objects::get_dispatch::<B>))
.route("/{bucket}/{*key}", head(objects::head_object::<B>))
.route("/{bucket}/{*key}", delete(objects::delete_dispatch::<B>))
.route("/{bucket}/{*key}", post(objects::post_dispatch::<B>))
.fallback(fallback)
.layer(DefaultBodyLimit::max(5 * 1024 * 1024 * 1024)) // 5 GiB
.layer(TraceLayer::new_for_http())
.with_state(state)
}
async fn fallback(req: Request) -> impl IntoResponse {
tracing::warn!(
method = %req.method(),
uri = %req.uri(),
"unmatched request"
);
StatusCode::NOT_FOUND
}

View File

@@ -0,0 +1,6 @@
use post3::StorageBackend;
#[derive(Clone)]
pub struct State<B: StorageBackend> {
pub store: B,
}

View File

@@ -0,0 +1,106 @@
use std::net::SocketAddr;
use aws_credential_types::Credentials;
use aws_sdk_s3::Client;
use post3::PostgresBackend;
use sqlx::PgPool;
use tokio::net::TcpListener;
use tokio_util::sync::CancellationToken;
static TRACING: std::sync::Once = std::sync::Once::new();
fn init_tracing() {
TRACING.call_once(|| {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive("post3_server=debug".parse().unwrap())
.add_directive("tower_http=debug".parse().unwrap()),
)
.with_test_writer()
.init();
});
}
pub struct TestServer {
pub addr: SocketAddr,
pub client: Client,
cancel: CancellationToken,
pool: PgPool,
}
impl TestServer {
pub async fn start() -> Self {
init_tracing();
let db_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
"postgresql://devuser:devpassword@localhost:5435/post3_dev".into()
});
let pool = sqlx::pool::PoolOptions::new()
.max_connections(5)
.connect(&db_url)
.await
.unwrap();
// Run migrations
sqlx::migrate!("../post3/migrations/")
.set_locking(false)
.run(&pool)
.await
.unwrap();
// Clean slate
sqlx::query("DELETE FROM upload_parts").execute(&pool).await.unwrap();
sqlx::query("DELETE FROM multipart_upload_metadata").execute(&pool).await.unwrap();
sqlx::query("DELETE FROM multipart_uploads").execute(&pool).await.unwrap();
sqlx::query("DELETE FROM blocks").execute(&pool).await.unwrap();
sqlx::query("DELETE FROM object_metadata").execute(&pool).await.unwrap();
sqlx::query("DELETE FROM objects").execute(&pool).await.unwrap();
sqlx::query("DELETE FROM buckets").execute(&pool).await.unwrap();
let backend = PostgresBackend::new(pool.clone());
let state = post3_server::state::State { store: backend };
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let cancel_clone = cancel.clone();
let router = post3_server::s3::router::build_router(state);
tokio::spawn(async move {
axum::serve(listener, router.into_make_service())
.with_graceful_shutdown(async move {
cancel_clone.cancelled().await;
})
.await
.unwrap();
});
let creds = Credentials::new("test", "test", None, None, "test");
let config = aws_sdk_s3::Config::builder()
.behavior_version_latest()
.region(aws_types::region::Region::new("us-east-1"))
.endpoint_url(format!("http://{}", addr))
.credentials_provider(creds)
.force_path_style(true)
.build();
let client = Client::from_conf(config);
Self {
addr,
client,
cancel,
pool,
}
}
pub async fn shutdown(self) {
self.cancel.cancel();
// Give the server task a moment to wind down
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
self.pool.close().await;
}
}

View File

@@ -0,0 +1,390 @@
//! Integration tests using FilesystemBackend (no PostgreSQL required).
use std::net::SocketAddr;
use aws_credential_types::Credentials;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::Client;
use post3::FilesystemBackend;
use tokio::net::TcpListener;
use tokio_util::sync::CancellationToken;
struct FsTestServer {
client: Client,
cancel: CancellationToken,
_tmpdir: tempfile::TempDir,
}
impl FsTestServer {
async fn start() -> Self {
let tmpdir = tempfile::tempdir().unwrap();
let backend = FilesystemBackend::new(tmpdir.path());
let state = post3_server::state::State { store: backend };
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr: SocketAddr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let cancel_clone = cancel.clone();
let router = post3_server::s3::router::build_router(state);
tokio::spawn(async move {
axum::serve(listener, router.into_make_service())
.with_graceful_shutdown(async move {
cancel_clone.cancelled().await;
})
.await
.unwrap();
});
let creds = Credentials::new("test", "test", None, None, "test");
let config = aws_sdk_s3::Config::builder()
.behavior_version_latest()
.region(aws_types::region::Region::new("us-east-1"))
.endpoint_url(format!("http://{}", addr))
.credentials_provider(creds)
.force_path_style(true)
.build();
let client = Client::from_conf(config);
Self {
client,
cancel,
_tmpdir: tmpdir,
}
}
async fn shutdown(self) {
self.cancel.cancel();
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
}
// --- Tests ---
#[tokio::test]
async fn test_fs_bucket_crud() {
let server = FsTestServer::start().await;
let c = &server.client;
// Create
c.create_bucket().bucket("my-bucket").send().await.unwrap();
// Head
c.head_bucket().bucket("my-bucket").send().await.unwrap();
// List
let resp = c.list_buckets().send().await.unwrap();
let names: Vec<_> = resp
.buckets()
.iter()
.filter_map(|b| b.name())
.collect();
assert!(names.contains(&"my-bucket"));
// Delete
c.delete_bucket().bucket("my-bucket").send().await.unwrap();
// Verify gone
let result = c.head_bucket().bucket("my-bucket").send().await;
assert!(result.is_err());
server.shutdown().await;
}
#[tokio::test]
async fn test_fs_put_get_delete() {
let server = FsTestServer::start().await;
let c = &server.client;
c.create_bucket().bucket("test").send().await.unwrap();
// Put
c.put_object()
.bucket("test")
.key("hello.txt")
.content_type("text/plain")
.body(aws_sdk_s3::primitives::ByteStream::from_static(
b"hello world",
))
.send()
.await
.unwrap();
// Get
let resp = c
.get_object()
.bucket("test")
.key("hello.txt")
.send()
.await
.unwrap();
let body = resp.body.collect().await.unwrap().into_bytes();
assert_eq!(body.as_ref(), b"hello world");
// Head
let head = c
.head_object()
.bucket("test")
.key("hello.txt")
.send()
.await
.unwrap();
assert_eq!(head.content_length(), Some(11));
assert_eq!(head.content_type(), Some("text/plain"));
// Delete
c.delete_object()
.bucket("test")
.key("hello.txt")
.send()
.await
.unwrap();
// Verify gone
let result = c
.get_object()
.bucket("test")
.key("hello.txt")
.send()
.await;
assert!(result.is_err());
// Cleanup
c.delete_bucket().bucket("test").send().await.unwrap();
server.shutdown().await;
}
#[tokio::test]
async fn test_fs_list_objects() {
let server = FsTestServer::start().await;
let c = &server.client;
c.create_bucket().bucket("test").send().await.unwrap();
for i in 0..5 {
c.put_object()
.bucket("test")
.key(format!("item-{i:02}"))
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"data"))
.send()
.await
.unwrap();
}
// List all
let resp = c
.list_objects_v2()
.bucket("test")
.send()
.await
.unwrap();
assert_eq!(resp.key_count(), Some(5));
// List with prefix
let resp = c
.list_objects_v2()
.bucket("test")
.prefix("item-03")
.send()
.await
.unwrap();
assert_eq!(resp.key_count(), Some(1));
// Cleanup
for i in 0..5 {
c.delete_object()
.bucket("test")
.key(format!("item-{i:02}"))
.send()
.await
.unwrap();
}
c.delete_bucket().bucket("test").send().await.unwrap();
server.shutdown().await;
}
#[tokio::test]
async fn test_fs_user_metadata() {
let server = FsTestServer::start().await;
let c = &server.client;
c.create_bucket().bucket("test").send().await.unwrap();
c.put_object()
.bucket("test")
.key("meta.txt")
.metadata("author", "test-user")
.metadata("version", "1")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"data"))
.send()
.await
.unwrap();
let head = c
.head_object()
.bucket("test")
.key("meta.txt")
.send()
.await
.unwrap();
let meta = head.metadata().unwrap();
assert_eq!(meta.get("author").unwrap(), "test-user");
assert_eq!(meta.get("version").unwrap(), "1");
// Cleanup
c.delete_object()
.bucket("test")
.key("meta.txt")
.send()
.await
.unwrap();
c.delete_bucket().bucket("test").send().await.unwrap();
server.shutdown().await;
}
#[tokio::test]
async fn test_fs_multipart_upload() {
let server = FsTestServer::start().await;
let c = &server.client;
c.create_bucket().bucket("test").send().await.unwrap();
// Create multipart upload
let create = c
.create_multipart_upload()
.bucket("test")
.key("big.bin")
.send()
.await
.unwrap();
let upload_id = create.upload_id().unwrap();
// Upload parts (non-last parts must be >= 5 MB per S3 spec)
let min_part = 5 * 1024 * 1024;
let part1 = c
.upload_part()
.bucket("test")
.key("big.bin")
.upload_id(upload_id)
.part_number(1)
.body(aws_sdk_s3::primitives::ByteStream::from(vec![0xAAu8; min_part]))
.send()
.await
.unwrap();
let part2 = c
.upload_part()
.bucket("test")
.key("big.bin")
.upload_id(upload_id)
.part_number(2)
.body(aws_sdk_s3::primitives::ByteStream::from(vec![0xBBu8; 1024]))
.send()
.await
.unwrap();
// Complete
let completed = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(part1.e_tag().unwrap())
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(part2.e_tag().unwrap())
.build(),
)
.build();
let complete_resp = c
.complete_multipart_upload()
.bucket("test")
.key("big.bin")
.upload_id(upload_id)
.multipart_upload(completed)
.send()
.await
.unwrap();
// Verify compound ETag
let etag = complete_resp.e_tag().unwrap();
assert!(etag.contains("-2"), "Expected compound ETag, got: {etag}");
// Verify data
let resp = c
.get_object()
.bucket("test")
.key("big.bin")
.send()
.await
.unwrap();
let body = resp.body.collect().await.unwrap().into_bytes();
assert_eq!(body.len(), min_part + 1024);
assert!(body[..min_part].iter().all(|b| *b == 0xAA));
assert!(body[min_part..].iter().all(|b| *b == 0xBB));
// Cleanup
c.delete_object()
.bucket("test")
.key("big.bin")
.send()
.await
.unwrap();
c.delete_bucket().bucket("test").send().await.unwrap();
server.shutdown().await;
}
#[tokio::test]
async fn test_fs_abort_multipart() {
let server = FsTestServer::start().await;
let c = &server.client;
c.create_bucket().bucket("test").send().await.unwrap();
let create = c
.create_multipart_upload()
.bucket("test")
.key("aborted.bin")
.send()
.await
.unwrap();
let upload_id = create.upload_id().unwrap();
// Upload a part
c.upload_part()
.bucket("test")
.key("aborted.bin")
.upload_id(upload_id)
.part_number(1)
.body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; 100]))
.send()
.await
.unwrap();
// Abort
c.abort_multipart_upload()
.bucket("test")
.key("aborted.bin")
.upload_id(upload_id)
.send()
.await
.unwrap();
// Verify no object was created
let result = c
.get_object()
.bucket("test")
.key("aborted.bin")
.send()
.await;
assert!(result.is_err());
c.delete_bucket().bucket("test").send().await.unwrap();
server.shutdown().await;
}

View File

@@ -0,0 +1,871 @@
mod common;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use common::TestServer;
#[tokio::test]
async fn test_create_and_list_buckets() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("test-bucket")
.send()
.await
.unwrap();
let resp = server.client.list_buckets().send().await.unwrap();
let names: Vec<_> = resp
.buckets()
.iter()
.filter_map(|b| b.name())
.collect();
assert!(names.contains(&"test-bucket"));
server.shutdown().await;
}
#[tokio::test]
async fn test_head_bucket() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("hb-test")
.send()
.await
.unwrap();
server
.client
.head_bucket()
.bucket("hb-test")
.send()
.await
.unwrap();
let err = server
.client
.head_bucket()
.bucket("no-such-bucket")
.send()
.await;
assert!(err.is_err());
server.shutdown().await;
}
#[tokio::test]
async fn test_delete_bucket() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("to-delete")
.send()
.await
.unwrap();
server
.client
.delete_bucket()
.bucket("to-delete")
.send()
.await
.unwrap();
let err = server
.client
.head_bucket()
.bucket("to-delete")
.send()
.await;
assert!(err.is_err());
server.shutdown().await;
}
#[tokio::test]
async fn test_put_and_get_object() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("data")
.send()
.await
.unwrap();
let body = ByteStream::from_static(b"hello world");
server
.client
.put_object()
.bucket("data")
.key("greeting.txt")
.content_type("text/plain")
.body(body)
.send()
.await
.unwrap();
let resp = server
.client
.get_object()
.bucket("data")
.key("greeting.txt")
.send()
.await
.unwrap();
let content_type = resp.content_type().map(|s| s.to_string());
let bytes = resp.body.collect().await.unwrap().into_bytes();
assert_eq!(bytes.as_ref(), b"hello world");
assert_eq!(content_type.as_deref(), Some("text/plain"));
server.shutdown().await;
}
#[tokio::test]
async fn test_put_large_object_chunked() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("large")
.send()
.await
.unwrap();
// 3 MiB object => should be split into 3 blocks at 1 MiB each
let data = vec![0x42u8; 3 * 1024 * 1024];
let body = ByteStream::from(data.clone());
server
.client
.put_object()
.bucket("large")
.key("big-file.bin")
.body(body)
.send()
.await
.unwrap();
let resp = server
.client
.get_object()
.bucket("large")
.key("big-file.bin")
.send()
.await
.unwrap();
let bytes = resp.body.collect().await.unwrap().into_bytes();
assert_eq!(bytes.len(), 3 * 1024 * 1024);
assert_eq!(bytes.as_ref(), data.as_slice());
server.shutdown().await;
}
#[tokio::test]
async fn test_head_object() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("meta")
.send()
.await
.unwrap();
let body = ByteStream::from_static(b"test");
server
.client
.put_object()
.bucket("meta")
.key("file.txt")
.body(body)
.send()
.await
.unwrap();
let resp = server
.client
.head_object()
.bucket("meta")
.key("file.txt")
.send()
.await
.unwrap();
assert_eq!(resp.content_length(), Some(4));
server.shutdown().await;
}
#[tokio::test]
async fn test_delete_object() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("del")
.send()
.await
.unwrap();
let body = ByteStream::from_static(b"bye");
server
.client
.put_object()
.bucket("del")
.key("gone.txt")
.body(body)
.send()
.await
.unwrap();
server
.client
.delete_object()
.bucket("del")
.key("gone.txt")
.send()
.await
.unwrap();
let err = server
.client
.get_object()
.bucket("del")
.key("gone.txt")
.send()
.await;
assert!(err.is_err());
server.shutdown().await;
}
#[tokio::test]
async fn test_list_objects_v2() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("list-test")
.send()
.await
.unwrap();
for i in 0..5 {
let body = ByteStream::from_static(b"x");
server
.client
.put_object()
.bucket("list-test")
.key(format!("prefix/file-{i}.txt"))
.body(body)
.send()
.await
.unwrap();
}
let resp = server
.client
.list_objects_v2()
.bucket("list-test")
.prefix("prefix/")
.send()
.await
.unwrap();
assert_eq!(resp.key_count(), Some(5));
server.shutdown().await;
}
#[tokio::test]
async fn test_overwrite_object() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("ow")
.send()
.await
.unwrap();
let body1 = ByteStream::from_static(b"version1");
server
.client
.put_object()
.bucket("ow")
.key("file.txt")
.body(body1)
.send()
.await
.unwrap();
let body2 = ByteStream::from_static(b"version2-longer");
server
.client
.put_object()
.bucket("ow")
.key("file.txt")
.body(body2)
.send()
.await
.unwrap();
let resp = server
.client
.get_object()
.bucket("ow")
.key("file.txt")
.send()
.await
.unwrap();
let bytes = resp.body.collect().await.unwrap().into_bytes();
assert_eq!(bytes.as_ref(), b"version2-longer");
server.shutdown().await;
}
#[tokio::test]
async fn test_user_metadata_roundtrip() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("meta-test")
.send()
.await
.unwrap();
let body = ByteStream::from_static(b"with metadata");
server
.client
.put_object()
.bucket("meta-test")
.key("doc.txt")
.body(body)
.metadata("author", "test-user")
.metadata("version", "42")
.send()
.await
.unwrap();
let resp = server
.client
.head_object()
.bucket("meta-test")
.key("doc.txt")
.send()
.await
.unwrap();
let meta = resp.metadata().unwrap();
assert_eq!(meta.get("author").map(|s| s.as_str()), Some("test-user"));
assert_eq!(meta.get("version").map(|s| s.as_str()), Some("42"));
server.shutdown().await;
}
// --- Multipart upload tests ---
#[tokio::test]
async fn test_multipart_upload_basic() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("mp-basic")
.send()
.await
.unwrap();
// Create multipart upload
let create_resp = server
.client
.create_multipart_upload()
.bucket("mp-basic")
.key("large-file.bin")
.send()
.await
.unwrap();
let upload_id = create_resp.upload_id().unwrap().to_string();
// Upload 3 parts (non-last parts must be >= 5 MB per S3 spec)
let min_part = 5 * 1024 * 1024;
let part1_data = vec![0x11u8; min_part];
let part2_data = vec![0x22u8; min_part];
let part3_data = vec![0x33u8; 1024 * 1024];
let p1 = server
.client
.upload_part()
.bucket("mp-basic")
.key("large-file.bin")
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from(part1_data.clone()))
.send()
.await
.unwrap();
let p2 = server
.client
.upload_part()
.bucket("mp-basic")
.key("large-file.bin")
.upload_id(&upload_id)
.part_number(2)
.body(ByteStream::from(part2_data.clone()))
.send()
.await
.unwrap();
let p3 = server
.client
.upload_part()
.bucket("mp-basic")
.key("large-file.bin")
.upload_id(&upload_id)
.part_number(3)
.body(ByteStream::from(part3_data.clone()))
.send()
.await
.unwrap();
// Complete multipart upload
let completed = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(p1.e_tag().unwrap())
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(p2.e_tag().unwrap())
.build(),
)
.parts(
CompletedPart::builder()
.part_number(3)
.e_tag(p3.e_tag().unwrap())
.build(),
)
.build();
let complete_resp = server
.client
.complete_multipart_upload()
.bucket("mp-basic")
.key("large-file.bin")
.upload_id(&upload_id)
.multipart_upload(completed)
.send()
.await
.unwrap();
// Verify ETag is compound format (hex-3)
let etag = complete_resp.e_tag().unwrap();
assert!(etag.contains("-3"), "Expected compound ETag, got: {etag}");
// Get and verify assembled data
let get_resp = server
.client
.get_object()
.bucket("mp-basic")
.key("large-file.bin")
.send()
.await
.unwrap();
let body = get_resp.body.collect().await.unwrap().into_bytes();
assert_eq!(body.len(), min_part * 2 + 1024 * 1024);
let mut expected = Vec::new();
expected.extend_from_slice(&part1_data);
expected.extend_from_slice(&part2_data);
expected.extend_from_slice(&part3_data);
assert_eq!(body.as_ref(), expected.as_slice());
server.shutdown().await;
}
#[tokio::test]
async fn test_abort_multipart_upload() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("mp-abort")
.send()
.await
.unwrap();
let create_resp = server
.client
.create_multipart_upload()
.bucket("mp-abort")
.key("aborted.bin")
.send()
.await
.unwrap();
let upload_id = create_resp.upload_id().unwrap().to_string();
// Upload a part
server
.client
.upload_part()
.bucket("mp-abort")
.key("aborted.bin")
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from(vec![0xAAu8; 1024]))
.send()
.await
.unwrap();
// Abort
server
.client
.abort_multipart_upload()
.bucket("mp-abort")
.key("aborted.bin")
.upload_id(&upload_id)
.send()
.await
.unwrap();
// Verify object doesn't exist
let err = server
.client
.get_object()
.bucket("mp-abort")
.key("aborted.bin")
.send()
.await;
assert!(err.is_err());
server.shutdown().await;
}
#[tokio::test]
async fn test_list_parts() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("mp-list-parts")
.send()
.await
.unwrap();
let create_resp = server
.client
.create_multipart_upload()
.bucket("mp-list-parts")
.key("parts.bin")
.send()
.await
.unwrap();
let upload_id = create_resp.upload_id().unwrap().to_string();
// Upload 3 parts
for i in 1..=3 {
server
.client
.upload_part()
.bucket("mp-list-parts")
.key("parts.bin")
.upload_id(&upload_id)
.part_number(i)
.body(ByteStream::from(vec![i as u8; 1024 * 100]))
.send()
.await
.unwrap();
}
// List parts
let list_resp = server
.client
.list_parts()
.bucket("mp-list-parts")
.key("parts.bin")
.upload_id(&upload_id)
.send()
.await
.unwrap();
let parts = list_resp.parts();
assert_eq!(parts.len(), 3);
assert_eq!(parts[0].part_number(), Some(1));
assert_eq!(parts[1].part_number(), Some(2));
assert_eq!(parts[2].part_number(), Some(3));
for p in parts {
assert_eq!(p.size(), Some(1024 * 100));
}
// Cleanup
server
.client
.abort_multipart_upload()
.bucket("mp-list-parts")
.key("parts.bin")
.upload_id(&upload_id)
.send()
.await
.unwrap();
server.shutdown().await;
}
#[tokio::test]
async fn test_list_multipart_uploads() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("mp-list-uploads")
.send()
.await
.unwrap();
// Create two uploads
let u1 = server
.client
.create_multipart_upload()
.bucket("mp-list-uploads")
.key("file-a.bin")
.send()
.await
.unwrap();
let u1_id = u1.upload_id().unwrap().to_string();
let u2 = server
.client
.create_multipart_upload()
.bucket("mp-list-uploads")
.key("file-b.bin")
.send()
.await
.unwrap();
let u2_id = u2.upload_id().unwrap().to_string();
// List multipart uploads
let list_resp = server
.client
.list_multipart_uploads()
.bucket("mp-list-uploads")
.send()
.await
.unwrap();
let uploads = list_resp.uploads();
assert_eq!(uploads.len(), 2);
let keys: Vec<&str> = uploads.iter().filter_map(|u| u.key()).collect();
assert!(keys.contains(&"file-a.bin"));
assert!(keys.contains(&"file-b.bin"));
// Cleanup
server
.client
.abort_multipart_upload()
.bucket("mp-list-uploads")
.key("file-a.bin")
.upload_id(&u1_id)
.send()
.await
.unwrap();
server
.client
.abort_multipart_upload()
.bucket("mp-list-uploads")
.key("file-b.bin")
.upload_id(&u2_id)
.send()
.await
.unwrap();
server.shutdown().await;
}
#[tokio::test]
async fn test_overwrite_part() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("mp-overwrite")
.send()
.await
.unwrap();
let create_resp = server
.client
.create_multipart_upload()
.bucket("mp-overwrite")
.key("ow.bin")
.send()
.await
.unwrap();
let upload_id = create_resp.upload_id().unwrap().to_string();
// Upload part 1 with data A
server
.client
.upload_part()
.bucket("mp-overwrite")
.key("ow.bin")
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from(vec![0xAAu8; 1024]))
.send()
.await
.unwrap();
// Re-upload part 1 with data B
let p1 = server
.client
.upload_part()
.bucket("mp-overwrite")
.key("ow.bin")
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from(vec![0xBBu8; 1024]))
.send()
.await
.unwrap();
// Complete with the latest etag
let completed = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(p1.e_tag().unwrap())
.build(),
)
.build();
server
.client
.complete_multipart_upload()
.bucket("mp-overwrite")
.key("ow.bin")
.upload_id(&upload_id)
.multipart_upload(completed)
.send()
.await
.unwrap();
// Verify data B
let get_resp = server
.client
.get_object()
.bucket("mp-overwrite")
.key("ow.bin")
.send()
.await
.unwrap();
let body = get_resp.body.collect().await.unwrap().into_bytes();
assert_eq!(body.as_ref(), vec![0xBBu8; 1024].as_slice());
server.shutdown().await;
}
#[tokio::test]
async fn test_multipart_with_metadata() {
let server = TestServer::start().await;
server
.client
.create_bucket()
.bucket("mp-meta")
.send()
.await
.unwrap();
// Create multipart upload with metadata
let create_resp = server
.client
.create_multipart_upload()
.bucket("mp-meta")
.key("meta-file.bin")
.metadata("author", "test-user")
.metadata("version", "7")
.send()
.await
.unwrap();
let upload_id = create_resp.upload_id().unwrap().to_string();
// Upload one part
let p1 = server
.client
.upload_part()
.bucket("mp-meta")
.key("meta-file.bin")
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from(vec![0xFFu8; 512]))
.send()
.await
.unwrap();
// Complete
let completed = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(p1.e_tag().unwrap())
.build(),
)
.build();
server
.client
.complete_multipart_upload()
.bucket("mp-meta")
.key("meta-file.bin")
.upload_id(&upload_id)
.multipart_upload(completed)
.send()
.await
.unwrap();
// Head object — verify metadata came through
let head = server
.client
.head_object()
.bucket("mp-meta")
.key("meta-file.bin")
.send()
.await
.unwrap();
let meta = head.metadata().unwrap();
assert_eq!(meta.get("author").map(|s| s.as_str()), Some("test-user"));
assert_eq!(meta.get("version").map(|s| s.as_str()), Some("7"));
server.shutdown().await;
}

22
crates/post3/Cargo.toml Normal file
View File

@@ -0,0 +1,22 @@
[package]
name = "post3"
version.workspace = true
edition.workspace = true
[dependencies]
anyhow.workspace = true
tokio.workspace = true
tracing.workspace = true
sqlx.workspace = true
uuid.workspace = true
bytes.workspace = true
chrono.workspace = true
md-5.workspace = true
hex.workspace = true
thiserror.workspace = true
serde.workspace = true
serde_json.workspace = true
percent-encoding.workspace = true
[dev-dependencies]
tempfile.workspace = true

View File

@@ -0,0 +1,37 @@
CREATE TABLE buckets (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX idx_buckets_name ON buckets (name);
CREATE TABLE objects (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
bucket_id UUID NOT NULL REFERENCES buckets(id) ON DELETE CASCADE,
key TEXT NOT NULL,
size BIGINT NOT NULL,
etag TEXT NOT NULL,
content_type TEXT NOT NULL DEFAULT 'application/octet-stream',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX idx_objects_bucket_key ON objects (bucket_id, key);
CREATE INDEX idx_objects_key_prefix ON objects (bucket_id, key text_pattern_ops);
CREATE TABLE object_metadata (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
object_id UUID NOT NULL REFERENCES objects(id) ON DELETE CASCADE,
meta_key TEXT NOT NULL,
meta_value TEXT NOT NULL
);
CREATE UNIQUE INDEX idx_metadata_object_key ON object_metadata (object_id, meta_key);
CREATE INDEX idx_metadata_object_id ON object_metadata (object_id);
CREATE TABLE blocks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
object_id UUID NOT NULL REFERENCES objects(id) ON DELETE CASCADE,
block_index INT NOT NULL,
data BYTEA NOT NULL,
block_size INT NOT NULL
);
CREATE UNIQUE INDEX idx_blocks_object_index ON blocks (object_id, block_index);
CREATE INDEX idx_blocks_object_id ON blocks (object_id);

View File

@@ -0,0 +1,29 @@
CREATE TABLE multipart_uploads (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
bucket_id UUID NOT NULL REFERENCES buckets(id) ON DELETE CASCADE,
key TEXT NOT NULL,
upload_id TEXT NOT NULL,
content_type TEXT NOT NULL DEFAULT 'application/octet-stream',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX idx_multipart_upload_id ON multipart_uploads (upload_id);
CREATE INDEX idx_multipart_bucket ON multipart_uploads (bucket_id);
CREATE TABLE multipart_upload_metadata (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
upload_id UUID NOT NULL REFERENCES multipart_uploads(id) ON DELETE CASCADE,
meta_key TEXT NOT NULL,
meta_value TEXT NOT NULL
);
CREATE UNIQUE INDEX idx_mp_meta_key ON multipart_upload_metadata (upload_id, meta_key);
CREATE TABLE upload_parts (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
upload_id UUID NOT NULL REFERENCES multipart_uploads(id) ON DELETE CASCADE,
part_number INT NOT NULL,
data BYTEA NOT NULL,
size BIGINT NOT NULL,
etag TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX idx_upload_parts_num ON upload_parts (upload_id, part_number);

123
crates/post3/src/backend.rs Normal file
View File

@@ -0,0 +1,123 @@
use std::collections::HashMap;
use std::future::Future;
use bytes::Bytes;
use crate::error::Post3Error;
use crate::models::{
BucketInfo, CompleteMultipartUploadResult, CreateMultipartUploadResult, GetObjectResult,
HeadObjectResult, ListMultipartUploadsResult, ListObjectsResult, ListPartsResult,
PutObjectResult, UploadPartResult,
};
/// Trait abstracting storage operations. Implemented by `PostgresBackend` and `FilesystemBackend`.
pub trait StorageBackend: Clone + Send + Sync + 'static {
// --- Bucket operations ---
fn create_bucket(
&self,
name: &str,
) -> impl Future<Output = Result<BucketInfo, Post3Error>> + Send;
fn head_bucket(
&self,
name: &str,
) -> impl Future<Output = Result<Option<BucketInfo>, Post3Error>> + Send;
fn delete_bucket(
&self,
name: &str,
) -> impl Future<Output = Result<(), Post3Error>> + Send;
fn list_buckets(&self) -> impl Future<Output = Result<Vec<BucketInfo>, Post3Error>> + Send;
// --- Object operations ---
fn put_object(
&self,
bucket: &str,
key: &str,
content_type: Option<&str>,
metadata: HashMap<String, String>,
body: Bytes,
) -> impl Future<Output = Result<PutObjectResult, Post3Error>> + Send;
fn get_object(
&self,
bucket: &str,
key: &str,
) -> impl Future<Output = Result<GetObjectResult, Post3Error>> + Send;
fn head_object(
&self,
bucket: &str,
key: &str,
) -> impl Future<Output = Result<Option<HeadObjectResult>, Post3Error>> + Send;
fn delete_object(
&self,
bucket: &str,
key: &str,
) -> impl Future<Output = Result<(), Post3Error>> + Send;
fn list_objects_v2(
&self,
bucket: &str,
prefix: Option<&str>,
continuation_token: Option<&str>,
max_keys: Option<i64>,
delimiter: Option<&str>,
) -> impl Future<Output = Result<ListObjectsResult, Post3Error>> + Send;
// --- Multipart upload operations ---
fn create_multipart_upload(
&self,
bucket: &str,
key: &str,
content_type: Option<&str>,
metadata: HashMap<String, String>,
) -> impl Future<Output = Result<CreateMultipartUploadResult, Post3Error>> + Send;
fn upload_part(
&self,
bucket: &str,
key: &str,
upload_id: &str,
part_number: i32,
body: Bytes,
) -> impl Future<Output = Result<UploadPartResult, Post3Error>> + Send;
fn complete_multipart_upload(
&self,
bucket: &str,
key: &str,
upload_id: &str,
part_etags: Vec<(i32, String)>,
) -> impl Future<Output = Result<CompleteMultipartUploadResult, Post3Error>> + Send;
fn abort_multipart_upload(
&self,
bucket: &str,
key: &str,
upload_id: &str,
) -> impl Future<Output = Result<(), Post3Error>> + Send;
fn list_parts(
&self,
bucket: &str,
key: &str,
upload_id: &str,
max_parts: Option<i32>,
part_number_marker: Option<i32>,
) -> impl Future<Output = Result<ListPartsResult, Post3Error>> + Send;
fn list_multipart_uploads(
&self,
bucket: &str,
prefix: Option<&str>,
key_marker: Option<&str>,
upload_id_marker: Option<&str>,
max_uploads: Option<i32>,
) -> impl Future<Output = Result<ListMultipartUploadsResult, Post3Error>> + Send;
}

45
crates/post3/src/error.rs Normal file
View File

@@ -0,0 +1,45 @@
#[derive(Debug, thiserror::Error)]
pub enum Post3Error {
#[error("bucket not found: {0}")]
BucketNotFound(String),
#[error("bucket already exists: {0}")]
BucketAlreadyExists(String),
#[error("object not found: bucket={bucket}, key={key}")]
ObjectNotFound { bucket: String, key: String },
#[error("bucket not empty: {0}")]
BucketNotEmpty(String),
#[error("multipart upload not found: {0}")]
UploadNotFound(String),
#[error("invalid part: upload_id={upload_id}, part_number={part_number}")]
InvalidPart { upload_id: String, part_number: i32 },
#[error("etag mismatch for part {part_number}: expected={expected}, got={got}")]
ETagMismatch {
part_number: i32,
expected: String,
got: String,
},
#[error("invalid part order in complete request")]
InvalidPartOrder,
#[error("part {part_number} is too small: size={size}, minimum=5242880")]
EntityTooSmall { part_number: i32, size: i64 },
#[error("io error: {0}")]
Io(#[from] std::io::Error),
#[error("serialization error: {0}")]
Serialization(String),
#[error(transparent)]
Database(#[from] sqlx::Error),
#[error(transparent)]
Other(#[from] anyhow::Error),
}

2173
crates/post3/src/fs.rs Normal file

File diff suppressed because it is too large Load Diff

11
crates/post3/src/lib.rs Normal file
View File

@@ -0,0 +1,11 @@
pub mod backend;
pub mod error;
pub mod fs;
pub mod models;
pub mod repositories;
pub mod store;
pub use backend::StorageBackend;
pub use error::Post3Error;
pub use fs::FilesystemBackend;
pub use store::{PostgresBackend, Store};

170
crates/post3/src/models.rs Normal file
View File

@@ -0,0 +1,170 @@
use chrono::{DateTime, Utc};
use uuid::Uuid;
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct BucketRow {
pub id: Uuid,
pub name: String,
pub created_at: DateTime<Utc>,
}
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct ObjectRow {
pub id: Uuid,
pub bucket_id: Uuid,
pub key: String,
pub size: i64,
pub etag: String,
pub content_type: String,
pub created_at: DateTime<Utc>,
}
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct BlockRow {
pub id: Uuid,
pub object_id: Uuid,
pub block_index: i32,
pub data: Vec<u8>,
pub block_size: i32,
}
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct MetadataEntry {
pub id: Uuid,
pub object_id: Uuid,
pub meta_key: String,
pub meta_value: String,
}
/// Backend-neutral bucket summary.
#[derive(Debug, Clone)]
pub struct BucketInfo {
pub name: String,
pub created_at: DateTime<Utc>,
}
/// Backend-neutral object metadata (no internal IDs).
#[derive(Debug, Clone)]
pub struct ObjectMeta {
pub key: String,
pub size: i64,
pub etag: String,
pub content_type: String,
pub last_modified: DateTime<Utc>,
}
#[derive(Debug, Clone)]
pub struct ObjectInfo {
pub key: String,
pub size: i64,
pub etag: String,
pub last_modified: DateTime<Utc>,
}
#[derive(Debug, Clone)]
pub struct ListObjectsResult {
pub objects: Vec<ObjectInfo>,
pub is_truncated: bool,
pub next_continuation_token: Option<String>,
pub prefix: Option<String>,
pub delimiter: Option<String>,
pub common_prefixes: Vec<String>,
pub key_count: usize,
}
#[derive(Debug)]
pub struct PutObjectResult {
pub etag: String,
pub size: i64,
}
#[derive(Debug)]
pub struct GetObjectResult {
pub metadata: ObjectMeta,
pub user_metadata: std::collections::HashMap<String, String>,
pub body: bytes::Bytes,
}
#[derive(Debug)]
pub struct HeadObjectResult {
pub object: ObjectMeta,
pub user_metadata: std::collections::HashMap<String, String>,
}
// --- Multipart upload models ---
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct MultipartUploadRow {
pub id: Uuid,
pub bucket_id: Uuid,
pub key: String,
pub upload_id: String,
pub content_type: String,
pub created_at: DateTime<Utc>,
}
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct UploadPartRow {
pub id: Uuid,
pub upload_id: Uuid,
pub part_number: i32,
pub data: Vec<u8>,
pub size: i64,
pub etag: String,
pub created_at: DateTime<Utc>,
}
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct UploadPartInfo {
pub part_number: i32,
pub size: i64,
pub etag: String,
pub created_at: DateTime<Utc>,
}
#[derive(Debug)]
pub struct CreateMultipartUploadResult {
pub bucket: String,
pub key: String,
pub upload_id: String,
}
#[derive(Debug)]
pub struct UploadPartResult {
pub etag: String,
}
#[derive(Debug)]
pub struct CompleteMultipartUploadResult {
pub bucket: String,
pub key: String,
pub etag: String,
pub size: i64,
}
#[derive(Debug)]
pub struct ListPartsResult {
pub bucket: String,
pub key: String,
pub upload_id: String,
pub parts: Vec<UploadPartInfo>,
pub is_truncated: bool,
pub next_part_number_marker: Option<i32>,
}
#[derive(Debug)]
pub struct MultipartUploadInfo {
pub key: String,
pub upload_id: String,
pub initiated: DateTime<Utc>,
}
#[derive(Debug)]
pub struct ListMultipartUploadsResult {
pub bucket: String,
pub uploads: Vec<MultipartUploadInfo>,
pub is_truncated: bool,
pub next_key_marker: Option<String>,
pub next_upload_id_marker: Option<String>,
pub prefix: Option<String>,
}

View File

@@ -0,0 +1,44 @@
use sqlx::{Postgres, Transaction};
use uuid::Uuid;
use crate::error::Post3Error;
use crate::models::BlockRow;
pub struct BlocksRepository;
impl BlocksRepository {
pub async fn insert_in_tx(
tx: &mut Transaction<'_, Postgres>,
object_id: Uuid,
block_index: i32,
data: &[u8],
) -> Result<(), Post3Error> {
let block_size = data.len() as i32;
sqlx::query(
"INSERT INTO blocks (object_id, block_index, data, block_size) \
VALUES ($1, $2, $3, $4)",
)
.bind(object_id)
.bind(block_index)
.bind(data)
.bind(block_size)
.execute(&mut **tx)
.await?;
Ok(())
}
pub async fn get_all(
db: &sqlx::PgPool,
object_id: Uuid,
) -> Result<Vec<BlockRow>, Post3Error> {
let rows = sqlx::query_as::<_, BlockRow>(
"SELECT * FROM blocks WHERE object_id = $1 ORDER BY block_index ASC",
)
.bind(object_id)
.fetch_all(db)
.await?;
Ok(rows)
}
}

View File

@@ -0,0 +1,80 @@
use sqlx::PgPool;
use uuid::Uuid;
use crate::error::Post3Error;
use crate::models::BucketRow;
pub struct BucketsRepository<'a> {
db: &'a PgPool,
}
impl<'a> BucketsRepository<'a> {
pub fn new(db: &'a PgPool) -> Self {
Self { db }
}
pub async fn create(&self, name: &str) -> Result<BucketRow, Post3Error> {
let existing = self.get_by_name(name).await?;
if existing.is_some() {
return Err(Post3Error::BucketAlreadyExists(name.to_string()));
}
let row = sqlx::query_as::<_, BucketRow>(
"INSERT INTO buckets (name) VALUES ($1) RETURNING *",
)
.bind(name)
.fetch_one(self.db)
.await?;
Ok(row)
}
pub async fn get_by_name(&self, name: &str) -> Result<Option<BucketRow>, Post3Error> {
let row =
sqlx::query_as::<_, BucketRow>("SELECT * FROM buckets WHERE name = $1")
.bind(name)
.fetch_optional(self.db)
.await?;
Ok(row)
}
pub async fn list(&self) -> Result<Vec<BucketRow>, Post3Error> {
let rows = sqlx::query_as::<_, BucketRow>(
"SELECT * FROM buckets ORDER BY created_at ASC",
)
.fetch_all(self.db)
.await?;
Ok(rows)
}
pub async fn delete(&self, name: &str) -> Result<(), Post3Error> {
let bucket = self
.get_by_name(name)
.await?
.ok_or_else(|| Post3Error::BucketNotFound(name.to_string()))?;
if !self.is_empty(bucket.id).await? {
return Err(Post3Error::BucketNotEmpty(name.to_string()));
}
sqlx::query("DELETE FROM buckets WHERE id = $1")
.bind(bucket.id)
.execute(self.db)
.await?;
Ok(())
}
pub async fn is_empty(&self, bucket_id: Uuid) -> Result<bool, Post3Error> {
let count: (i64,) = sqlx::query_as(
"SELECT COUNT(*) FROM objects WHERE bucket_id = $1",
)
.bind(bucket_id)
.fetch_one(self.db)
.await?;
Ok(count.0 == 0)
}
}

View File

@@ -0,0 +1,49 @@
use sqlx::{PgPool, Postgres, Transaction};
use std::collections::HashMap;
use uuid::Uuid;
use crate::error::Post3Error;
use crate::models::MetadataEntry;
pub struct MetadataRepository;
impl MetadataRepository {
pub async fn insert_batch_in_tx(
tx: &mut Transaction<'_, Postgres>,
object_id: Uuid,
metadata: &HashMap<String, String>,
) -> Result<(), Post3Error> {
for (key, value) in metadata {
sqlx::query(
"INSERT INTO object_metadata (object_id, meta_key, meta_value) \
VALUES ($1, $2, $3)",
)
.bind(object_id)
.bind(key)
.bind(value)
.execute(&mut **tx)
.await?;
}
Ok(())
}
pub async fn get_all(
db: &PgPool,
object_id: Uuid,
) -> Result<HashMap<String, String>, Post3Error> {
let rows = sqlx::query_as::<_, MetadataEntry>(
"SELECT * FROM object_metadata WHERE object_id = $1",
)
.bind(object_id)
.fetch_all(db)
.await?;
let map = rows
.into_iter()
.map(|e| (e.meta_key, e.meta_value))
.collect();
Ok(map)
}
}

View File

@@ -0,0 +1,7 @@
pub mod blocks;
pub mod buckets;
pub mod metadata;
pub mod multipart_metadata;
pub mod multipart_uploads;
pub mod objects;
pub mod upload_parts;

View File

@@ -0,0 +1,50 @@
use sqlx::{PgPool, Postgres, Transaction};
use std::collections::HashMap;
use uuid::Uuid;
use crate::error::Post3Error;
use crate::models::MetadataEntry;
pub struct MultipartMetadataRepository;
impl MultipartMetadataRepository {
pub async fn insert_batch_in_tx(
tx: &mut Transaction<'_, Postgres>,
upload_id: Uuid,
metadata: &HashMap<String, String>,
) -> Result<(), Post3Error> {
for (key, value) in metadata {
sqlx::query(
"INSERT INTO multipart_upload_metadata (upload_id, meta_key, meta_value) \
VALUES ($1, $2, $3)",
)
.bind(upload_id)
.bind(key)
.bind(value)
.execute(&mut **tx)
.await?;
}
Ok(())
}
pub async fn get_all(
db: &PgPool,
upload_id: Uuid,
) -> Result<HashMap<String, String>, Post3Error> {
let rows = sqlx::query_as::<_, MetadataEntry>(
"SELECT id, upload_id AS object_id, meta_key, meta_value \
FROM multipart_upload_metadata WHERE upload_id = $1",
)
.bind(upload_id)
.fetch_all(db)
.await?;
let map = rows
.into_iter()
.map(|e| (e.meta_key, e.meta_value))
.collect();
Ok(map)
}
}

View File

@@ -0,0 +1,163 @@
use sqlx::{PgPool, Postgres, Transaction};
use uuid::Uuid;
use crate::error::Post3Error;
use crate::models::MultipartUploadRow;
pub struct MultipartUploadsRepository;
impl MultipartUploadsRepository {
pub async fn create_in_tx(
tx: &mut Transaction<'_, Postgres>,
bucket_id: Uuid,
key: &str,
upload_id: &str,
content_type: &str,
) -> Result<MultipartUploadRow, Post3Error> {
let row = sqlx::query_as::<_, MultipartUploadRow>(
"INSERT INTO multipart_uploads (bucket_id, key, upload_id, content_type) \
VALUES ($1, $2, $3, $4) RETURNING *",
)
.bind(bucket_id)
.bind(key)
.bind(upload_id)
.bind(content_type)
.fetch_one(&mut **tx)
.await?;
Ok(row)
}
pub async fn get_by_upload_id(
db: &PgPool,
upload_id: &str,
) -> Result<Option<MultipartUploadRow>, Post3Error> {
let row = sqlx::query_as::<_, MultipartUploadRow>(
"SELECT * FROM multipart_uploads WHERE upload_id = $1",
)
.bind(upload_id)
.fetch_optional(db)
.await?;
Ok(row)
}
pub async fn delete_in_tx(
tx: &mut Transaction<'_, Postgres>,
id: Uuid,
) -> Result<(), Post3Error> {
sqlx::query("DELETE FROM multipart_uploads WHERE id = $1")
.bind(id)
.execute(&mut **tx)
.await?;
Ok(())
}
pub async fn delete_by_upload_id(
db: &PgPool,
upload_id: &str,
) -> Result<bool, Post3Error> {
let result = sqlx::query("DELETE FROM multipart_uploads WHERE upload_id = $1")
.bind(upload_id)
.execute(db)
.await?;
Ok(result.rows_affected() > 0)
}
pub async fn list(
db: &PgPool,
bucket_id: Uuid,
prefix: Option<&str>,
key_marker: Option<&str>,
upload_id_marker: Option<&str>,
max_uploads: i64,
) -> Result<Vec<MultipartUploadRow>, Post3Error> {
let rows = match (prefix, key_marker) {
(Some(pfx), Some(marker)) => {
let pattern = format!("{pfx}%");
// When key_marker is provided, return uploads with key > marker,
// or same key but upload_id > upload_id_marker
if let Some(uid_marker) = upload_id_marker {
sqlx::query_as::<_, MultipartUploadRow>(
"SELECT * FROM multipart_uploads \
WHERE bucket_id = $1 AND key LIKE $2 \
AND (key > $3 OR (key = $3 AND upload_id > $4)) \
ORDER BY key ASC, upload_id ASC LIMIT $5",
)
.bind(bucket_id)
.bind(pattern)
.bind(marker)
.bind(uid_marker)
.bind(max_uploads)
.fetch_all(db)
.await?
} else {
sqlx::query_as::<_, MultipartUploadRow>(
"SELECT * FROM multipart_uploads \
WHERE bucket_id = $1 AND key LIKE $2 AND key > $3 \
ORDER BY key ASC, upload_id ASC LIMIT $4",
)
.bind(bucket_id)
.bind(pattern)
.bind(marker)
.bind(max_uploads)
.fetch_all(db)
.await?
}
}
(Some(pfx), None) => {
let pattern = format!("{pfx}%");
sqlx::query_as::<_, MultipartUploadRow>(
"SELECT * FROM multipart_uploads \
WHERE bucket_id = $1 AND key LIKE $2 \
ORDER BY key ASC, upload_id ASC LIMIT $3",
)
.bind(bucket_id)
.bind(pattern)
.bind(max_uploads)
.fetch_all(db)
.await?
}
(None, Some(marker)) => {
if let Some(uid_marker) = upload_id_marker {
sqlx::query_as::<_, MultipartUploadRow>(
"SELECT * FROM multipart_uploads \
WHERE bucket_id = $1 \
AND (key > $2 OR (key = $2 AND upload_id > $3)) \
ORDER BY key ASC, upload_id ASC LIMIT $4",
)
.bind(bucket_id)
.bind(marker)
.bind(uid_marker)
.bind(max_uploads)
.fetch_all(db)
.await?
} else {
sqlx::query_as::<_, MultipartUploadRow>(
"SELECT * FROM multipart_uploads \
WHERE bucket_id = $1 AND key > $2 \
ORDER BY key ASC, upload_id ASC LIMIT $3",
)
.bind(bucket_id)
.bind(marker)
.bind(max_uploads)
.fetch_all(db)
.await?
}
}
(None, None) => {
sqlx::query_as::<_, MultipartUploadRow>(
"SELECT * FROM multipart_uploads \
WHERE bucket_id = $1 \
ORDER BY key ASC, upload_id ASC LIMIT $2",
)
.bind(bucket_id)
.bind(max_uploads)
.fetch_all(db)
.await?
}
};
Ok(rows)
}
}

View File

@@ -0,0 +1,139 @@
use sqlx::{PgPool, Postgres, Transaction};
use uuid::Uuid;
use crate::error::Post3Error;
use crate::models::ObjectRow;
pub struct ObjectsRepository<'a> {
db: &'a PgPool,
}
impl<'a> ObjectsRepository<'a> {
pub fn new(db: &'a PgPool) -> Self {
Self { db }
}
pub async fn insert_in_tx(
tx: &mut Transaction<'_, Postgres>,
bucket_id: Uuid,
key: &str,
size: i64,
etag: &str,
content_type: &str,
) -> Result<ObjectRow, Post3Error> {
// Delete existing (cascades to blocks + metadata)
sqlx::query("DELETE FROM objects WHERE bucket_id = $1 AND key = $2")
.bind(bucket_id)
.bind(key)
.execute(&mut **tx)
.await?;
let row = sqlx::query_as::<_, ObjectRow>(
"INSERT INTO objects (bucket_id, key, size, etag, content_type) \
VALUES ($1, $2, $3, $4, $5) RETURNING *",
)
.bind(bucket_id)
.bind(key)
.bind(size)
.bind(etag)
.bind(content_type)
.fetch_one(&mut **tx)
.await?;
Ok(row)
}
pub async fn get(
&self,
bucket_id: Uuid,
key: &str,
) -> Result<Option<ObjectRow>, Post3Error> {
let row = sqlx::query_as::<_, ObjectRow>(
"SELECT * FROM objects WHERE bucket_id = $1 AND key = $2",
)
.bind(bucket_id)
.bind(key)
.fetch_optional(self.db)
.await?;
Ok(row)
}
pub async fn delete(
&self,
bucket_id: Uuid,
key: &str,
) -> Result<bool, Post3Error> {
let result =
sqlx::query("DELETE FROM objects WHERE bucket_id = $1 AND key = $2")
.bind(bucket_id)
.bind(key)
.execute(self.db)
.await?;
Ok(result.rows_affected() > 0)
}
pub async fn list(
&self,
bucket_id: Uuid,
prefix: Option<&str>,
start_after: Option<&str>,
max_keys: i64,
) -> Result<Vec<ObjectRow>, Post3Error> {
let rows = match (prefix, start_after) {
(Some(pfx), Some(after)) => {
let pattern = format!("{pfx}%");
sqlx::query_as::<_, ObjectRow>(
"SELECT * FROM objects \
WHERE bucket_id = $1 AND key LIKE $2 AND key > $3 \
ORDER BY key ASC LIMIT $4",
)
.bind(bucket_id)
.bind(pattern)
.bind(after)
.bind(max_keys)
.fetch_all(self.db)
.await?
}
(Some(pfx), None) => {
let pattern = format!("{pfx}%");
sqlx::query_as::<_, ObjectRow>(
"SELECT * FROM objects \
WHERE bucket_id = $1 AND key LIKE $2 \
ORDER BY key ASC LIMIT $3",
)
.bind(bucket_id)
.bind(pattern)
.bind(max_keys)
.fetch_all(self.db)
.await?
}
(None, Some(after)) => {
sqlx::query_as::<_, ObjectRow>(
"SELECT * FROM objects \
WHERE bucket_id = $1 AND key > $2 \
ORDER BY key ASC LIMIT $3",
)
.bind(bucket_id)
.bind(after)
.bind(max_keys)
.fetch_all(self.db)
.await?
}
(None, None) => {
sqlx::query_as::<_, ObjectRow>(
"SELECT * FROM objects \
WHERE bucket_id = $1 \
ORDER BY key ASC LIMIT $2",
)
.bind(bucket_id)
.bind(max_keys)
.fetch_all(self.db)
.await?
}
};
Ok(rows)
}
}

View File

@@ -0,0 +1,88 @@
use sqlx::PgPool;
use uuid::Uuid;
use crate::error::Post3Error;
use crate::models::{UploadPartInfo, UploadPartRow};
pub struct UploadPartsRepository;
impl UploadPartsRepository {
pub async fn upsert(
db: &PgPool,
upload_id: Uuid,
part_number: i32,
data: &[u8],
size: i64,
etag: &str,
) -> Result<(), Post3Error> {
sqlx::query(
"INSERT INTO upload_parts (upload_id, part_number, data, size, etag) \
VALUES ($1, $2, $3, $4, $5) \
ON CONFLICT (upload_id, part_number) DO UPDATE \
SET data = EXCLUDED.data, size = EXCLUDED.size, \
etag = EXCLUDED.etag, created_at = NOW()",
)
.bind(upload_id)
.bind(part_number)
.bind(data)
.bind(size)
.bind(etag)
.execute(db)
.await?;
Ok(())
}
pub async fn list_info(
db: &PgPool,
upload_id: Uuid,
part_number_marker: Option<i32>,
max_parts: i64,
) -> Result<Vec<UploadPartInfo>, Post3Error> {
let rows = if let Some(marker) = part_number_marker {
sqlx::query_as::<_, UploadPartInfo>(
"SELECT part_number, size, etag, created_at \
FROM upload_parts \
WHERE upload_id = $1 AND part_number > $2 \
ORDER BY part_number ASC LIMIT $3",
)
.bind(upload_id)
.bind(marker)
.bind(max_parts)
.fetch_all(db)
.await?
} else {
sqlx::query_as::<_, UploadPartInfo>(
"SELECT part_number, size, etag, created_at \
FROM upload_parts \
WHERE upload_id = $1 \
ORDER BY part_number ASC LIMIT $2",
)
.bind(upload_id)
.bind(max_parts)
.fetch_all(db)
.await?
};
Ok(rows)
}
pub async fn get_ordered_by_numbers(
db: &PgPool,
upload_id: Uuid,
part_numbers: &[i32],
) -> Result<Vec<UploadPartRow>, Post3Error> {
// Fetch the requested parts in order
let rows = sqlx::query_as::<_, UploadPartRow>(
"SELECT * FROM upload_parts \
WHERE upload_id = $1 AND part_number = ANY($2) \
ORDER BY part_number ASC",
)
.bind(upload_id)
.bind(part_numbers)
.fetch_all(db)
.await?;
Ok(rows)
}
}

705
crates/post3/src/store.rs Normal file
View File

@@ -0,0 +1,705 @@
use std::collections::HashMap;
use bytes::Bytes;
use md5::{Digest, Md5};
use sqlx::PgPool;
use crate::backend::StorageBackend;
use crate::error::Post3Error;
use crate::models::{
BucketInfo, BucketRow, CompleteMultipartUploadResult, CreateMultipartUploadResult,
GetObjectResult, HeadObjectResult, ListMultipartUploadsResult, ListObjectsResult,
ListPartsResult, MultipartUploadInfo, MultipartUploadRow, ObjectInfo, ObjectMeta,
PutObjectResult, UploadPartResult,
};
use crate::repositories::blocks::BlocksRepository;
use crate::repositories::buckets::BucketsRepository;
use crate::repositories::metadata::MetadataRepository;
use crate::repositories::multipart_metadata::MultipartMetadataRepository;
use crate::repositories::multipart_uploads::MultipartUploadsRepository;
use crate::repositories::objects::ObjectsRepository;
use crate::repositories::upload_parts::UploadPartsRepository;
pub const DEFAULT_BLOCK_SIZE: usize = 1024 * 1024; // 1 MiB
/// PostgreSQL-backed storage. Also exported as `PostgresBackend`.
#[derive(Clone)]
pub struct Store {
db: PgPool,
block_size: usize,
}
/// Alias for `Store` — the PostgreSQL-backed storage backend.
pub type PostgresBackend = Store;
impl Store {
pub fn new(db: PgPool) -> Self {
Self {
db,
block_size: DEFAULT_BLOCK_SIZE,
}
}
pub fn with_block_size(mut self, block_size: usize) -> Self {
self.block_size = block_size;
self
}
pub fn pool(&self) -> &PgPool {
&self.db
}
// --- Private helpers ---
async fn require_bucket(&self, name: &str) -> Result<BucketRow, Post3Error> {
BucketsRepository::new(&self.db)
.get_by_name(name)
.await?
.ok_or_else(|| Post3Error::BucketNotFound(name.to_string()))
}
async fn require_upload(
&self,
upload_id: &str,
expected_bucket_id: uuid::Uuid,
expected_key: &str,
) -> Result<MultipartUploadRow, Post3Error> {
let upload = MultipartUploadsRepository::get_by_upload_id(&self.db, upload_id)
.await?
.ok_or_else(|| Post3Error::UploadNotFound(upload_id.to_string()))?;
if upload.bucket_id != expected_bucket_id || upload.key != expected_key {
return Err(Post3Error::UploadNotFound(upload_id.to_string()));
}
Ok(upload)
}
}
impl StorageBackend for Store {
// --- Bucket operations ---
async fn create_bucket(&self, name: &str) -> Result<BucketInfo, Post3Error> {
let row = BucketsRepository::new(&self.db).create(name).await?;
Ok(BucketInfo {
name: row.name,
created_at: row.created_at,
})
}
async fn head_bucket(&self, name: &str) -> Result<Option<BucketInfo>, Post3Error> {
Ok(BucketsRepository::new(&self.db)
.get_by_name(name)
.await?
.map(|row| BucketInfo {
name: row.name,
created_at: row.created_at,
}))
}
async fn delete_bucket(&self, name: &str) -> Result<(), Post3Error> {
BucketsRepository::new(&self.db).delete(name).await
}
async fn list_buckets(&self) -> Result<Vec<BucketInfo>, Post3Error> {
Ok(BucketsRepository::new(&self.db)
.list()
.await?
.into_iter()
.map(|row| BucketInfo {
name: row.name,
created_at: row.created_at,
})
.collect())
}
// --- Object operations ---
async fn put_object(
&self,
bucket: &str,
key: &str,
content_type: Option<&str>,
metadata: HashMap<String, String>,
body: Bytes,
) -> Result<PutObjectResult, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let content_type = content_type.unwrap_or("application/octet-stream");
let mut hasher = Md5::new();
hasher.update(&body);
let etag = format!("\"{}\"", hex::encode(hasher.finalize()));
let size = body.len() as i64;
let mut tx = self.db.begin().await?;
let object_row = ObjectsRepository::insert_in_tx(
&mut tx,
bucket_row.id,
key,
size,
&etag,
content_type,
)
.await?;
for (chunk_index, chunk) in body.chunks(self.block_size).enumerate() {
BlocksRepository::insert_in_tx(
&mut tx,
object_row.id,
chunk_index as i32,
chunk,
)
.await?;
}
if !metadata.is_empty() {
MetadataRepository::insert_batch_in_tx(
&mut tx,
object_row.id,
&metadata,
)
.await?;
}
tx.commit().await?;
Ok(PutObjectResult { etag, size })
}
async fn get_object(
&self,
bucket: &str,
key: &str,
) -> Result<GetObjectResult, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let object = ObjectsRepository::new(&self.db)
.get(bucket_row.id, key)
.await?
.ok_or_else(|| Post3Error::ObjectNotFound {
bucket: bucket.to_string(),
key: key.to_string(),
})?;
let blocks = BlocksRepository::get_all(&self.db, object.id).await?;
let mut body = Vec::with_capacity(object.size as usize);
for block in blocks {
body.extend_from_slice(&block.data);
}
let user_metadata =
MetadataRepository::get_all(&self.db, object.id).await?;
Ok(GetObjectResult {
metadata: ObjectMeta {
key: object.key,
size: object.size,
etag: object.etag,
content_type: object.content_type,
last_modified: object.created_at,
},
user_metadata,
body: Bytes::from(body),
})
}
async fn head_object(
&self,
bucket: &str,
key: &str,
) -> Result<Option<HeadObjectResult>, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let object = ObjectsRepository::new(&self.db)
.get(bucket_row.id, key)
.await?;
match object {
Some(obj) => {
let user_metadata =
MetadataRepository::get_all(&self.db, obj.id).await?;
Ok(Some(HeadObjectResult {
object: ObjectMeta {
key: obj.key,
size: obj.size,
etag: obj.etag,
content_type: obj.content_type,
last_modified: obj.created_at,
},
user_metadata,
}))
}
None => Ok(None),
}
}
async fn delete_object(
&self,
bucket: &str,
key: &str,
) -> Result<(), Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
ObjectsRepository::new(&self.db)
.delete(bucket_row.id, key)
.await?;
Ok(())
}
async fn list_objects_v2(
&self,
bucket: &str,
prefix: Option<&str>,
continuation_token: Option<&str>,
max_keys: Option<i64>,
delimiter: Option<&str>,
) -> Result<ListObjectsResult, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let max_keys = max_keys.unwrap_or(1000);
// MaxKeys=0 is valid: return empty result
if max_keys == 0 {
return Ok(ListObjectsResult {
objects: Vec::new(),
is_truncated: false,
next_continuation_token: None,
prefix: prefix.map(|s| s.to_string()),
delimiter: delimiter.map(|s| s.to_string()),
common_prefixes: Vec::new(),
key_count: 0,
});
}
// Fetch a generous batch for delimiter grouping (need enough to fill max_keys
// after rolling up common prefixes). For non-delimiter case, fetch max_keys+1.
let fetch_limit = if delimiter.is_some() {
// Fetch more to account for prefix rollups — worst case all keys share prefixes
(max_keys + 1) * 10
} else {
max_keys + 1
};
let rows = ObjectsRepository::new(&self.db)
.list(bucket_row.id, prefix, continuation_token, fetch_limit)
.await?;
let all_objects: Vec<ObjectInfo> = rows
.into_iter()
.map(|o| ObjectInfo {
key: o.key,
size: o.size,
etag: o.etag,
last_modified: o.created_at,
})
.collect();
let prefix_str = prefix.unwrap_or("");
if let Some(delim) = delimiter {
// Separate into direct objects and rolled-up common prefixes
let mut seen_prefixes = std::collections::BTreeSet::new();
let mut direct_objects = Vec::new();
for obj in &all_objects {
let after_prefix = &obj.key[prefix_str.len()..];
if let Some(pos) = after_prefix.find(delim) {
let cp = format!("{}{}", prefix_str, &after_prefix[..pos + delim.len()]);
seen_prefixes.insert(cp);
} else {
direct_objects.push(obj.clone());
}
}
// Filter out common prefixes that are <= continuation token
let all_prefixes: Vec<String> = if let Some(token) = continuation_token {
seen_prefixes
.into_iter()
.filter(|cp| cp.as_str() > token)
.collect()
} else {
seen_prefixes.into_iter().collect()
};
// Merge objects and common_prefixes in sorted order, limited to max_keys
let mut result_objects = Vec::new();
let mut result_prefixes = Vec::new();
let mut oi = 0usize;
let mut pi = 0usize;
let mut count = 0i64;
let mut last_key: Option<String> = None;
while count < max_keys && (oi < direct_objects.len() || pi < all_prefixes.len()) {
let take_object = match (direct_objects.get(oi), all_prefixes.get(pi)) {
(Some(obj), Some(pfx)) => obj.key.as_str() < pfx.as_str(),
(Some(_), None) => true,
(None, Some(_)) => false,
(None, None) => break,
};
if take_object {
last_key = Some(direct_objects[oi].key.clone());
result_objects.push(direct_objects[oi].clone());
oi += 1;
} else {
last_key = Some(all_prefixes[pi].clone());
result_prefixes.push(all_prefixes[pi].clone());
pi += 1;
}
count += 1;
}
let is_truncated = oi < direct_objects.len() || pi < all_prefixes.len();
let next_token = if is_truncated { last_key } else { None };
let key_count = result_objects.len() + result_prefixes.len();
Ok(ListObjectsResult {
objects: result_objects,
is_truncated,
next_continuation_token: next_token,
prefix: prefix.map(|s| s.to_string()),
delimiter: Some(delim.to_string()),
common_prefixes: result_prefixes,
key_count,
})
} else {
let is_truncated = all_objects.len() as i64 > max_keys;
let items: Vec<_> = all_objects.into_iter().take(max_keys as usize).collect();
let next_token = if is_truncated {
items.last().map(|o| o.key.clone())
} else {
None
};
let key_count = items.len();
Ok(ListObjectsResult {
objects: items,
is_truncated,
next_continuation_token: next_token,
prefix: prefix.map(|s| s.to_string()),
delimiter: None,
common_prefixes: Vec::new(),
key_count,
})
}
}
// --- Multipart upload operations ---
async fn create_multipart_upload(
&self,
bucket: &str,
key: &str,
content_type: Option<&str>,
metadata: HashMap<String, String>,
) -> Result<CreateMultipartUploadResult, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let content_type = content_type.unwrap_or("application/octet-stream");
let upload_id = uuid::Uuid::new_v4().to_string();
let mut tx = self.db.begin().await?;
let upload_row = MultipartUploadsRepository::create_in_tx(
&mut tx,
bucket_row.id,
key,
&upload_id,
content_type,
)
.await?;
if !metadata.is_empty() {
MultipartMetadataRepository::insert_batch_in_tx(
&mut tx,
upload_row.id,
&metadata,
)
.await?;
}
tx.commit().await?;
Ok(CreateMultipartUploadResult {
bucket: bucket.to_string(),
key: key.to_string(),
upload_id,
})
}
async fn upload_part(
&self,
bucket: &str,
key: &str,
upload_id: &str,
part_number: i32,
body: Bytes,
) -> Result<UploadPartResult, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let upload = self
.require_upload(upload_id, bucket_row.id, key)
.await?;
let mut hasher = Md5::new();
hasher.update(&body);
let etag = format!("\"{}\"", hex::encode(hasher.finalize()));
let size = body.len() as i64;
UploadPartsRepository::upsert(
&self.db,
upload.id,
part_number,
&body,
size,
&etag,
)
.await?;
Ok(UploadPartResult { etag })
}
async fn complete_multipart_upload(
&self,
bucket: &str,
key: &str,
upload_id: &str,
part_etags: Vec<(i32, String)>,
) -> Result<CompleteMultipartUploadResult, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let upload = self
.require_upload(upload_id, bucket_row.id, key)
.await?;
// Validate part numbers are in ascending order
for window in part_etags.windows(2) {
if window[0].0 >= window[1].0 {
return Err(Post3Error::InvalidPartOrder);
}
}
// Fetch the requested parts
let part_numbers: Vec<i32> = part_etags.iter().map(|(n, _)| *n).collect();
let parts = UploadPartsRepository::get_ordered_by_numbers(
&self.db,
upload.id,
&part_numbers,
)
.await?;
// Validate all parts exist and ETags match
for (expected_num, expected_etag) in &part_etags {
let part = parts
.iter()
.find(|p| p.part_number == *expected_num)
.ok_or_else(|| Post3Error::InvalidPart {
upload_id: upload_id.to_string(),
part_number: *expected_num,
})?;
// Normalize ETags by stripping quotes for comparison
let stored = part.etag.trim_matches('"');
let expected = expected_etag.trim_matches('"');
if stored != expected {
return Err(Post3Error::ETagMismatch {
part_number: *expected_num,
expected: expected_etag.clone(),
got: part.etag.clone(),
});
}
}
// Validate minimum part size (5 MB) for all parts except the last
const MIN_PART_SIZE: i64 = 5 * 1024 * 1024;
for (i, part) in parts.iter().enumerate() {
if i < parts.len() - 1 && part.size < MIN_PART_SIZE {
return Err(Post3Error::EntityTooSmall {
part_number: part.part_number,
size: part.size,
});
}
}
// Compute compound ETag: MD5(concat of raw MD5 bytes of each part) + "-N"
let mut etag_hasher = Md5::new();
let part_count = parts.len();
for part in &parts {
// Part etag is quoted hex, e.g. "\"abcdef...\""
let hex_str = part.etag.trim_matches('"');
if let Ok(raw_md5) = hex::decode(hex_str) {
etag_hasher.update(&raw_md5);
}
}
let compound_etag = format!(
"\"{}-{}\"",
hex::encode(etag_hasher.finalize()),
part_count
);
// Concatenate all part data
let total_size: i64 = parts.iter().map(|p| p.size).sum();
let mut assembled = Vec::with_capacity(total_size as usize);
for part in &parts {
assembled.extend_from_slice(&part.data);
}
// Get upload metadata
let user_metadata =
MultipartMetadataRepository::get_all(&self.db, upload.id).await?;
// Begin transaction for the final object assembly
let mut tx = self.db.begin().await?;
// Insert the final object (deletes any existing object with same key)
let object_row = ObjectsRepository::insert_in_tx(
&mut tx,
bucket_row.id,
key,
total_size,
&compound_etag,
&upload.content_type,
)
.await?;
// Chunk into 1 MiB blocks
for (chunk_index, chunk) in assembled.chunks(self.block_size).enumerate() {
BlocksRepository::insert_in_tx(
&mut tx,
object_row.id,
chunk_index as i32,
chunk,
)
.await?;
}
// Transfer metadata
if !user_metadata.is_empty() {
MetadataRepository::insert_batch_in_tx(
&mut tx,
object_row.id,
&user_metadata,
)
.await?;
}
// Delete the multipart upload (cascades to parts + upload metadata)
MultipartUploadsRepository::delete_in_tx(&mut tx, upload.id).await?;
tx.commit().await?;
Ok(CompleteMultipartUploadResult {
bucket: bucket.to_string(),
key: key.to_string(),
etag: compound_etag,
size: total_size,
})
}
async fn abort_multipart_upload(
&self,
bucket: &str,
key: &str,
upload_id: &str,
) -> Result<(), Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let upload = self
.require_upload(upload_id, bucket_row.id, key)
.await?;
// CASCADE deletes parts + metadata
MultipartUploadsRepository::delete_by_upload_id(&self.db, &upload.upload_id)
.await?;
Ok(())
}
async fn list_parts(
&self,
bucket: &str,
key: &str,
upload_id: &str,
max_parts: Option<i32>,
part_number_marker: Option<i32>,
) -> Result<ListPartsResult, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let upload = self
.require_upload(upload_id, bucket_row.id, key)
.await?;
let max_parts = max_parts.unwrap_or(1000) as i64;
// Fetch one extra to detect truncation
let parts = UploadPartsRepository::list_info(
&self.db,
upload.id,
part_number_marker,
max_parts + 1,
)
.await?;
let is_truncated = parts.len() as i64 > max_parts;
let items: Vec<_> = parts.into_iter().take(max_parts as usize).collect();
let next_marker = if is_truncated {
items.last().map(|p| p.part_number)
} else {
None
};
Ok(ListPartsResult {
bucket: bucket.to_string(),
key: key.to_string(),
upload_id: upload_id.to_string(),
parts: items,
is_truncated,
next_part_number_marker: next_marker,
})
}
async fn list_multipart_uploads(
&self,
bucket: &str,
prefix: Option<&str>,
key_marker: Option<&str>,
upload_id_marker: Option<&str>,
max_uploads: Option<i32>,
) -> Result<ListMultipartUploadsResult, Post3Error> {
let bucket_row = self.require_bucket(bucket).await?;
let max_uploads = max_uploads.unwrap_or(1000) as i64;
// Fetch one extra to detect truncation
let rows = MultipartUploadsRepository::list(
&self.db,
bucket_row.id,
prefix,
key_marker,
upload_id_marker,
max_uploads + 1,
)
.await?;
let is_truncated = rows.len() as i64 > max_uploads;
let items: Vec<_> = rows.into_iter().take(max_uploads as usize).collect();
let (next_key_marker, next_upload_id_marker) = if is_truncated {
items
.last()
.map(|u| (Some(u.key.clone()), Some(u.upload_id.clone())))
.unwrap_or((None, None))
} else {
(None, None)
};
let uploads = items
.into_iter()
.map(|u| MultipartUploadInfo {
key: u.key,
upload_id: u.upload_id,
initiated: u.created_at,
})
.collect();
Ok(ListMultipartUploadsResult {
bucket: bucket.to_string(),
uploads,
is_truncated,
next_key_marker,
next_upload_id_marker,
prefix: prefix.map(|s| s.to_string()),
})
}
}