feat: add a very high amount of parallel processes

The library is not intended to handle this many tasks created so fast, and as such it is not optimized for. Generally we only allow a single modifier to the process manager at once. However, even then it can start and run a lot of tasks at once. Note that these processes, are comparable to pods / wasm containers / etc. And as such it wouldn't be realistic to run so many in parallel.

Signed-off-by: kjuulh <contact@kjuulh.io>

examples/parallel/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "parallel"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+noprocess.workspace = true
+
+tokio.workspace = true
+tokio-util.workspace = true
+tracing.workspace = true
+anyhow.workspace = true
+rand.workspace = true
+tracing-subscriber = "0.3.22"

examples/parallel/src/main.rs

@@ -0,0 +1,208 @@
+//! Example: Running 10,000 parallel processes with a reconciler
+//!
+//! This example demonstrates running many concurrent processes that:
+//! - Randomly wait for varying durations
+//! - Randomly complete successfully or encounter errors
+//! - Automatically restart on success
+//! - Get restarted by a reconciler when they error
+//!
+//! Run with: cargo run -p parallel
+
+use std::{
+    future::Future,
+    sync::atomic::{AtomicUsize, Ordering},
+    time::Duration,
+};
+
+use noprocess::{
+    Process, ProcessHandler, ProcessManager, ProcessResult, ProcessState, ShutdownConfig,
+};
+use rand::{Rng, SeedableRng};
+use tokio_util::sync::CancellationToken;
+
+/// Number of parallel processes to spawn
+const NUM_PROCESSES: usize = 100_000;
+
+/// Shared counters for tracking process activity
+static RUNNING: AtomicUsize = AtomicUsize::new(0);
+static COMPLETED: AtomicUsize = AtomicUsize::new(0);
+static ERRORS: AtomicUsize = AtomicUsize::new(0);
+static TOTAL_RUNS: AtomicUsize = AtomicUsize::new(0);
+static RECONCILER_RESTARTS: AtomicUsize = AtomicUsize::new(0);
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt::init();
+
+    println!("Starting {} parallel processes...", NUM_PROCESSES);
+
+    let process_manager = ProcessManager::new();
+    let mut handle_ids = Vec::with_capacity(NUM_PROCESSES);
+
+    // Create and add all processes
+    // Note: restart_on_success is false so processes don't auto-restart.
+    // The reconciler handles restarting errored processes instead.
+    for i in 0..NUM_PROCESSES {
+        let process = Process::builder(RandomWorker { id: i })
+            .handle_id(format!("worker-{i}"))
+            .shutdown_config(ShutdownConfig {
+                graceful_timeout: Duration::from_secs(1),
+                restart_on_success: false,
+                restart_delay: Duration::from_millis(100),
+            })
+            .build();
+
+        let handle_id = process_manager.add_process(process).await;
+        handle_ids.push(handle_id);
+    }
+
+    println!("All processes added. Starting them...");
+
+    // Start all processes (serialized due to inner mutex)
+    for id in &handle_ids {
+        process_manager.start_process(id).await?;
+    }
+
+    println!("All processes started!");
+    println!();
+
+    // Monitor loop - print stats every second
+    let monitor_handle = tokio::spawn(async {
+        loop {
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            let total_runs = TOTAL_RUNS.load(Ordering::Relaxed);
+            let restarts = total_runs.saturating_sub(NUM_PROCESSES);
+            println!(
+                "Stats: running={}, completed={}, errors={}, restarts={}, reconciler_restarts={}",
+                RUNNING.load(Ordering::Relaxed),
+                COMPLETED.load(Ordering::Relaxed),
+                ERRORS.load(Ordering::Relaxed),
+                restarts,
+                RECONCILER_RESTARTS.load(Ordering::Relaxed),
+            );
+        }
+    });
+
+    // Reconciler loop - check for errored processes and restart them
+    let reconciler_manager = process_manager.clone();
+    let reconciler_handle = tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(Duration::from_millis(500)).await;
+
+            let pending = reconciler_manager
+                .get_all_process_states()
+                .await
+                .into_iter()
+                .filter(|(_, p)| p == &ProcessState::Stopped || p == &ProcessState::Errored)
+                .collect::<Vec<_>>();
+
+            for (handle_id, _) in &pending {
+                // Check state again to avoid race conditions
+                if let Some(ProcessState::Stopped) =
+                    reconciler_manager.get_process_state(handle_id).await
+                    && reconciler_manager.start_process(handle_id).await.is_ok()
+                {
+                    RECONCILER_RESTARTS.fetch_add(1, Ordering::Relaxed);
+                }
+
+                if let Some(ProcessState::Errored) =
+                    reconciler_manager.get_process_state(handle_id).await
+                    && reconciler_manager.start_process(handle_id).await.is_ok()
+                {
+                    RECONCILER_RESTARTS.fetch_add(1, Ordering::Relaxed);
+                }
+            }
+        }
+    });
+
+    // Let it run for 10 seconds
+    tokio::time::sleep(Duration::from_secs(10)).await;
+
+    // Stop reconciler first to prevent it from restarting processes during shutdown
+    reconciler_handle.abort();
+
+    println!();
+    println!("Stopping all processes...");
+
+    // Stop all processes (serialized due to inner mutex)
+    for id in &handle_ids {
+        let _ = process_manager.stop_process(id).await;
+    }
+
+    monitor_handle.abort();
+
+    println!();
+    println!("Final stats:");
+    println!("  Total completed: {}", COMPLETED.load(Ordering::Relaxed));
+    println!("  Total errors: {}", ERRORS.load(Ordering::Relaxed));
+    let total_runs = TOTAL_RUNS.load(Ordering::Relaxed);
+    println!(
+        "  Total restarts (auto): {}",
+        total_runs.saturating_sub(NUM_PROCESSES)
+    );
+    println!(
+        "  Reconciler restarts: {}",
+        RECONCILER_RESTARTS.load(Ordering::Relaxed)
+    );
+
+    // Demonstrate state inspection API
+    println!();
+    println!("Final process states:");
+    let running = process_manager.get_running_processes().await;
+    let stopped = process_manager.get_stopped_processes().await;
+    let errored = process_manager.get_errored_processes().await;
+    println!("  Running: {}", running.len());
+    println!("  Stopped: {}", stopped.len());
+    println!("  Errored: {}", errored.len());
+
+    Ok(())
+}
+
+/// A worker process that randomly waits, completes, or errors
+pub struct RandomWorker {
+    id: usize,
+}
+
+impl ProcessHandler for RandomWorker {
+    fn call(&self, cancellation: CancellationToken) -> impl Future<Output = ProcessResult> + Send {
+        let id = self.id;
+
+        async move {
+            RUNNING.fetch_add(1, Ordering::Relaxed);
+            TOTAL_RUNS.fetch_add(1, Ordering::Relaxed);
+
+            // Use StdRng which is Send-safe
+            let mut rng = rand::rngs::StdRng::from_entropy();
+
+            loop {
+                // Random wait between 10ms and 500ms
+                let wait_ms = rng.gen_range(10..500);
+
+                tokio::select! {
+                    _ = tokio::time::sleep(Duration::from_millis(wait_ms)) => {
+                        // Decide what to do next
+                        let action: u8 = rng.gen_range(0..100);
+
+                        if action < 5 {
+                            // 5% chance: error out
+                            RUNNING.fetch_sub(1, Ordering::Relaxed);
+                            ERRORS.fetch_add(1, Ordering::Relaxed);
+                            return Err(format!("Worker {id} encountered an error").into());
+                        } else if action < 15 {
+                            // 10% chance: complete successfully (will restart due to config)
+                            RUNNING.fetch_sub(1, Ordering::Relaxed);
+                            COMPLETED.fetch_add(1, Ordering::Relaxed);
+                            return Ok(());
+                        }
+                        // 85% chance: continue working
+                    }
+                    _ = cancellation.cancelled() => {
+                        RUNNING.fetch_sub(1, Ordering::Relaxed);
+                        COMPLETED.fetch_add(1, Ordering::Relaxed);
+                        return Ok(());
+                    }
+                }
+            }
+        }
+    }
+}