Add musicfs-test-utils crate with RED resilience tests

Phase 1 of resilience testing design doc implementation:
- New musicfs-test-utils crate with FaultyOrigin, FaultyCasStore, fixtures
- Failpoints instrumented in musicfs-cas/store.rs
- 16 resilience tests (13 RED for missing features, 3 GREEN for existing)
- 3 Docker/Toxiproxy network tests (RED until docker-compose up)
- docker-compose.yml for Toxiproxy + MinIO + SFTP test infrastructure

Tests properly fail-first (TDD): check_all() sequential, no health timeout,
missing corruption detection, no passthrough mode, etc.

musicfs/crates/musicfs-cas/Cargo.toml

@@ -3,7 +3,12 @@ name = "musicfs-cas"
 version.workspace = true
 edition.workspace = true
 
+[features]
+default = []
+failpoints = ["fail/failpoints"]
+
 [dependencies]
+fail = { workspace = true, optional = true }
 musicfs-core = { path = "../musicfs-core" }
 musicfs-origins = { path = "../musicfs-origins" }
 musicfs-sync = { path = "../musicfs-sync" }

musicfs/crates/musicfs-cas/src/store.rs

@@ -6,6 +6,9 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use tokio::fs;
 use tracing::{debug, trace, warn};
 
+#[cfg(feature = "failpoints")]
+use fail::fail_point;
+
 const DEFAULT_MAX_SIZE_10GB: u64 = 10 * 1024 * 1024 * 1024;
 const DEFAULT_SHARD_LEVELS_256_SUBDIRS: u8 = 2;
 
@@ -80,8 +83,24 @@ impl CasStore {
             fs::create_dir_all(parent).await?;
         }
 
+        #[cfg(feature = "failpoints")]
+        fail_point!("cas-put-before-write", |_| {
+            Err(CasError::Io(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Failpoint: cas-put-before-write",
+            )))
+        });
+
         fs::write(&path, data).await?;
 
+        #[cfg(feature = "failpoints")]
+        fail_point!("cas-put-after-write-before-index", |_| {
+            Err(CasError::Io(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Failpoint: cas-put-after-write-before-index",
+            )))
+        });
+
         let location = ChunkLocation {
             path: path.clone(),
             size: data.len() as u32,

musicfs/crates/musicfs-origins/src/health.rs

@@ -180,7 +180,7 @@ impl HealthMonitor {
         HealthCheckHandle { stop_tx }
     }
 
-    async fn check_all(&self) {
+    pub async fn check_all(&self) {
         let origins: Vec<_> = self
             .origins
             .iter()

musicfs/crates/musicfs-test-utils/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "musicfs-test-utils"
+version.workspace = true
+edition.workspace = true
+description = "Test utilities and fixtures for MusicFS resilience testing"
+
+[dependencies]
+musicfs-core = { path = "../musicfs-core" }
+musicfs-origins = { path = "../musicfs-origins" }
+musicfs-cas = { path = "../musicfs-cas" }
+musicfs-cache = { path = "../musicfs-cache" }
+
+async-trait.workspace = true
+tokio = { workspace = true, features = ["full", "sync", "time"] }
+tracing.workspace = true
+thiserror.workspace = true
+parking_lot.workspace = true
+tempfile.workspace = true
+bytes.workspace = true
+
+# Fault injection
+fail = { version = "0.5", optional = true }
+rlimit = { version = "0.10", optional = true }
+nix = { version = "0.29", optional = true, features = ["signal", "process"] }
+
+# Docker/network tests
+noxious-client = { version = "1.0", optional = true }
+reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls"] }
+
+[features]
+default = []
+failpoints = ["fail/failpoints"]
+process-tests = ["nix"]
+resource-limits = ["rlimit"]
+docker-tests = ["noxious-client", "reqwest"]
+full = ["failpoints", "process-tests", "resource-limits", "docker-tests"]
+
+[dev-dependencies]
+tokio-test = "0.4"

musicfs/crates/musicfs-test-utils/src/assertions.rs

@@ -0,0 +1,206 @@
+use musicfs_cas::CasError;
+use musicfs_core::Error;
+use std::time::{Duration, Instant};
+
+pub fn assert_error_contains<T, E: std::fmt::Debug>(result: Result<T, E>, expected_text: &str) {
+    match result {
+        Ok(_) => panic!("Expected error containing '{}', but got Ok", expected_text),
+        Err(e) => {
+            let error_msg = format!("{:?}", e);
+            assert!(
+                error_msg.contains(expected_text),
+                "Expected error containing '{}', but got: {}",
+                expected_text,
+                error_msg
+            );
+        }
+    }
+}
+
+pub fn assert_io_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Io(_)) => (),
+        Err(e) => panic!("Expected Io error, got: {:?}", e),
+        Ok(_) => panic!("Expected Io error, got Ok"),
+    }
+}
+
+pub fn assert_cas_io_error<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::Io(_)) => (),
+        Err(e) => panic!("Expected CasError::Io, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::Io, got Ok"),
+    }
+}
+
+pub fn assert_cas_not_found<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::NotFound(_)) => (),
+        Err(e) => panic!("Expected CasError::NotFound, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::NotFound, got Ok"),
+    }
+}
+
+pub fn assert_cas_integrity_error<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::IntegrityError { .. }) => (),
+        Err(e) => panic!("Expected CasError::IntegrityError, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::IntegrityError, got Ok"),
+    }
+}
+
+pub fn assert_file_not_found<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::FileNotFound(_)) => (),
+        Err(e) => panic!("Expected FileNotFound error, got: {:?}", e),
+        Ok(_) => panic!("Expected FileNotFound error, got Ok"),
+    }
+}
+
+pub fn assert_origin_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Origin(_)) => (),
+        Err(e) => panic!("Expected Origin error, got: {:?}", e),
+        Ok(_) => panic!("Expected Origin error, got Ok"),
+    }
+}
+
+pub fn assert_timeout_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Timeout(_)) => (),
+        Err(e) => panic!("Expected Timeout error, got: {:?}", e),
+        Ok(_) => panic!("Expected Timeout error, got Ok"),
+    }
+}
+
+pub struct TimedAssertion {
+    start: Instant,
+    min_duration: Option<Duration>,
+    max_duration: Option<Duration>,
+}
+
+impl TimedAssertion {
+    pub fn new() -> Self {
+        Self {
+            start: Instant::now(),
+            min_duration: None,
+            max_duration: None,
+        }
+    }
+
+    pub fn expect_at_least(mut self, duration: Duration) -> Self {
+        self.min_duration = Some(duration);
+        self
+    }
+
+    pub fn expect_at_most(mut self, duration: Duration) -> Self {
+        self.max_duration = Some(duration);
+        self
+    }
+
+    pub fn assert_elapsed(self) {
+        let elapsed = self.start.elapsed();
+
+        if let Some(min) = self.min_duration {
+            assert!(
+                elapsed >= min,
+                "Expected at least {:?}, but only {:?} elapsed",
+                min,
+                elapsed
+            );
+        }
+
+        if let Some(max) = self.max_duration {
+            assert!(
+                elapsed <= max,
+                "Expected at most {:?}, but {:?} elapsed",
+                max,
+                elapsed
+            );
+        }
+    }
+}
+
+impl Default for TimedAssertion {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+pub async fn assert_completes_within<F, T>(future: F, timeout: Duration) -> T
+where
+    F: std::future::Future<Output = T>,
+{
+    tokio::time::timeout(timeout, future)
+        .await
+        .expect(&format!(
+            "Operation did not complete within {:?}",
+            timeout
+        ))
+}
+
+pub async fn assert_times_out<F, T>(future: F, timeout: Duration)
+where
+    F: std::future::Future<Output = T>,
+{
+    match tokio::time::timeout(timeout, future).await {
+        Ok(_) => panic!("Expected operation to time out, but it completed"),
+        Err(_) => (),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_assert_error_contains() {
+        let result: Result<(), Error> = Err(Error::Origin("connection refused".into()));
+        assert_error_contains(result, "connection");
+    }
+
+    #[test]
+    #[should_panic(expected = "Expected error containing")]
+    fn test_assert_error_contains_failure() {
+        let result: Result<(), Error> = Err(Error::Origin("something else".into()));
+        assert_error_contains(result, "connection");
+    }
+
+    #[test]
+    fn test_assert_io_error() {
+        let result: Result<(), Error> =
+            Err(Error::Io(std::io::Error::new(std::io::ErrorKind::Other, "test")));
+        assert_io_error(result);
+    }
+
+    #[test]
+    fn test_timed_assertion_at_least() {
+        let timer = TimedAssertion::new().expect_at_least(Duration::from_millis(10));
+        std::thread::sleep(Duration::from_millis(15));
+        timer.assert_elapsed();
+    }
+
+    #[test]
+    fn test_timed_assertion_at_most() {
+        let timer = TimedAssertion::new().expect_at_most(Duration::from_millis(100));
+        timer.assert_elapsed();
+    }
+
+    #[tokio::test]
+    async fn test_assert_completes_within() {
+        let result =
+            assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
+        assert_eq!(result, 42);
+    }
+
+    #[tokio::test]
+    async fn test_assert_times_out() {
+        assert_times_out(
+            async {
+                tokio::time::sleep(Duration::from_secs(10)).await;
+            },
+            Duration::from_millis(10),
+        )
+        .await;
+    }
+}

musicfs/crates/musicfs-test-utils/src/faulty_cas.rs

@@ -0,0 +1,250 @@
+use bytes::Bytes;
+use musicfs_cas::{CasConfig, CasError, CasStore, DedupStats};
+use musicfs_core::ChunkHash;
+use std::io::{self, ErrorKind};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
+
+pub struct FaultyCasStore {
+    inner: Arc<CasStore>,
+    inject_enospc: AtomicBool,
+    inject_eio_on_read: AtomicBool,
+    inject_eio_on_write: AtomicBool,
+    inject_corruption: AtomicBool,
+    fail_after_n_puts: AtomicUsize,
+    put_count: AtomicUsize,
+}
+
+impl FaultyCasStore {
+    pub fn new(inner: Arc<CasStore>) -> Self {
+        Self {
+            inner,
+            inject_enospc: AtomicBool::new(false),
+            inject_eio_on_read: AtomicBool::new(false),
+            inject_eio_on_write: AtomicBool::new(false),
+            inject_corruption: AtomicBool::new(false),
+            fail_after_n_puts: AtomicUsize::new(usize::MAX),
+            put_count: AtomicUsize::new(0),
+        }
+    }
+
+    pub async fn open(config: CasConfig) -> Result<Self, CasError> {
+        let store = CasStore::open(config).await?;
+        Ok(Self::new(Arc::new(store)))
+    }
+
+    pub fn set_inject_enospc(&self, enabled: bool) {
+        self.inject_enospc.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_eio_on_read(&self, enabled: bool) {
+        self.inject_eio_on_read.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_eio_on_write(&self, enabled: bool) {
+        self.inject_eio_on_write.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_corruption(&self, enabled: bool) {
+        self.inject_corruption.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_fail_after_n_puts(&self, n: usize) {
+        self.fail_after_n_puts.store(n, Ordering::SeqCst);
+        self.put_count.store(0, Ordering::SeqCst);
+    }
+
+    pub fn reset_faults(&self) {
+        self.inject_enospc.store(false, Ordering::SeqCst);
+        self.inject_eio_on_read.store(false, Ordering::SeqCst);
+        self.inject_eio_on_write.store(false, Ordering::SeqCst);
+        self.inject_corruption.store(false, Ordering::SeqCst);
+        self.fail_after_n_puts.store(usize::MAX, Ordering::SeqCst);
+        self.put_count.store(0, Ordering::SeqCst);
+    }
+
+    pub fn put_count(&self) -> usize {
+        self.put_count.load(Ordering::SeqCst)
+    }
+
+    pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
+        let count = self.put_count.fetch_add(1, Ordering::SeqCst);
+
+        if self.inject_enospc.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "No space left on device (ENOSPC injected)",
+            )));
+        }
+
+        if self.inject_eio_on_write.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+
+        let threshold = self.fail_after_n_puts.load(Ordering::SeqCst);
+        if count >= threshold {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Injected failure after N puts",
+            )));
+        }
+
+        self.inner.put(data).await
+    }
+
+    pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
+        if self.inject_eio_on_read.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+
+        let data = self.inner.get(hash).await?;
+
+        if self.inject_corruption.load(Ordering::SeqCst) {
+            let mut corrupted = data.to_vec();
+            if !corrupted.is_empty() {
+                corrupted[0] = corrupted[0].wrapping_add(1);
+            }
+            return Err(CasError::IntegrityError {
+                expected: hash.as_hex(),
+                actual: ChunkHash::from_bytes(&corrupted).as_hex(),
+            });
+        }
+
+        Ok(data)
+    }
+
+    pub fn exists(&self, hash: &ChunkHash) -> bool {
+        self.inner.exists(hash)
+    }
+
+    pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
+        if self.inject_eio_on_write.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+        self.inner.delete(hash).await
+    }
+
+    pub fn current_size(&self) -> u64 {
+        self.inner.current_size()
+    }
+
+    pub fn max_size(&self) -> u64 {
+        self.inner.max_size()
+    }
+
+    pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
+        self.inner.list_chunks()
+    }
+
+    pub fn dedup_stats(&self) -> DedupStats {
+        self.inner.dedup_stats()
+    }
+
+    pub fn inner(&self) -> &Arc<CasStore> {
+        &self.inner
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    async fn test_store() -> (FaultyCasStore, TempDir) {
+        let dir = TempDir::new().unwrap();
+        let config = CasConfig {
+            chunks_dir: dir.path().join("chunks"),
+            max_size: 1024 * 1024,
+            shard_levels: 2,
+        };
+        let store = FaultyCasStore::open(config).await.unwrap();
+        (store, dir)
+    }
+
+    #[tokio::test]
+    async fn test_healthy_passthrough() {
+        let (store, _dir) = test_store().await;
+
+        let data = b"test data";
+        let hash = store.put(data).await.unwrap();
+        let retrieved = store.get(&hash).await.unwrap();
+        assert_eq!(&retrieved[..], data);
+    }
+
+    #[tokio::test]
+    async fn test_inject_enospc() {
+        let (store, _dir) = test_store().await;
+
+        store.set_inject_enospc(true);
+        let result = store.put(b"test").await;
+        assert!(result.is_err());
+
+        let err = result.unwrap_err();
+        assert!(matches!(err, CasError::Io(_)));
+
+        store.set_inject_enospc(false);
+        assert!(store.put(b"test").await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_inject_eio_on_read() {
+        let (store, _dir) = test_store().await;
+
+        let hash = store.put(b"test").await.unwrap();
+
+        store.set_inject_eio_on_read(true);
+        let result = store.get(&hash).await;
+        assert!(result.is_err());
+
+        store.set_inject_eio_on_read(false);
+        assert!(store.get(&hash).await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_inject_corruption() {
+        let (store, _dir) = test_store().await;
+
+        let hash = store.put(b"test data").await.unwrap();
+
+        store.set_inject_corruption(true);
+        let result = store.get(&hash).await;
+        assert!(matches!(result, Err(CasError::IntegrityError { .. })));
+    }
+
+    #[tokio::test]
+    async fn test_fail_after_n_puts() {
+        let (store, _dir) = test_store().await;
+
+        store.set_fail_after_n_puts(2);
+
+        assert!(store.put(b"data1").await.is_ok());
+        assert!(store.put(b"data2").await.is_ok());
+        assert!(store.put(b"data3").await.is_err());
+        assert!(store.put(b"data4").await.is_err());
+        assert_eq!(store.put_count(), 4);
+    }
+
+    #[tokio::test]
+    async fn test_reset_faults() {
+        let (store, _dir) = test_store().await;
+
+        store.set_inject_enospc(true);
+        store.set_inject_eio_on_read(true);
+        store.set_fail_after_n_puts(1);
+
+        store.reset_faults();
+
+        assert!(store.put(b"test").await.is_ok());
+        let hash = store.put(b"test2").await.unwrap();
+        assert!(store.get(&hash).await.is_ok());
+    }
+}

musicfs/crates/musicfs-test-utils/src/faulty_origin.rs

@@ -0,0 +1,328 @@
+use async_trait::async_trait;
+use musicfs_core::{DirEntry, Error, FileStat, HealthStatus, OriginId, OriginType, Result};
+use musicfs_origins::{Origin, WatchCallback, WatchHandle};
+use parking_lot::RwLock;
+use std::io::{self, ErrorKind};
+use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::io::AsyncRead;
+
+#[derive(Debug, Clone)]
+pub enum FailMode {
+    Healthy,
+    FailEveryNth(usize),
+    FailAfterN(usize),
+    TimeoutMs(u64),
+    PartialRead { max_bytes: usize },
+    ReturnError(ErrorKind),
+}
+
+impl Default for FailMode {
+    fn default() -> Self {
+        FailMode::Healthy
+    }
+}
+
+pub struct FaultyOrigin {
+    inner: Arc<dyn Origin>,
+    fail_mode: Arc<RwLock<FailMode>>,
+    call_count: AtomicUsize,
+}
+
+impl FaultyOrigin {
+    pub fn new(inner: Arc<dyn Origin>, mode: FailMode) -> Self {
+        Self {
+            inner,
+            fail_mode: Arc::new(RwLock::new(mode)),
+            call_count: AtomicUsize::new(0),
+        }
+    }
+
+    pub fn wrap(inner: impl Origin + 'static) -> Self {
+        Self::new(Arc::new(inner), FailMode::Healthy)
+    }
+
+    pub fn set_mode(&self, mode: FailMode) {
+        *self.fail_mode.write() = mode;
+    }
+
+    pub fn call_count(&self) -> usize {
+        self.call_count.load(Ordering::SeqCst)
+    }
+
+    pub fn reset_count(&self) {
+        self.call_count.store(0, Ordering::SeqCst);
+    }
+
+    fn increment_and_check(&self) -> Option<Error> {
+        let count = self.call_count.fetch_add(1, Ordering::SeqCst) + 1;
+        let mode = self.fail_mode.read();
+
+        match *mode {
+            FailMode::Healthy => None,
+            FailMode::FailEveryNth(n) if n > 0 && count % n == 0 => {
+                Some(Error::Origin("Injected failure (every Nth)".into()))
+            }
+            FailMode::FailEveryNth(_) => None,
+            FailMode::FailAfterN(n) if count > n => {
+                Some(Error::Origin("Injected failure (after N)".into()))
+            }
+            FailMode::FailAfterN(_) => None,
+            FailMode::TimeoutMs(_) => None,
+            FailMode::PartialRead { .. } => None,
+            FailMode::ReturnError(kind) => {
+                Some(Error::Io(io::Error::new(kind, "Injected I/O error")))
+            }
+        }
+    }
+
+    async fn maybe_timeout(&self) -> Option<Error> {
+        let mode = self.fail_mode.read().clone();
+        if let FailMode::TimeoutMs(ms) = mode {
+            tokio::time::sleep(Duration::from_millis(ms)).await;
+            Some(Error::Timeout("Injected timeout".into()))
+        } else {
+            None
+        }
+    }
+
+    fn truncate_if_partial(&self, mut data: Vec<u8>) -> Vec<u8> {
+        let mode = self.fail_mode.read();
+        if let FailMode::PartialRead { max_bytes } = *mode {
+            data.truncate(max_bytes);
+        }
+        data
+    }
+}
+
+#[async_trait]
+impl Origin for FaultyOrigin {
+    fn id(&self) -> &OriginId {
+        self.inner.id()
+    }
+
+    fn origin_type(&self) -> OriginType {
+        self.inner.origin_type()
+    }
+
+    fn display_name(&self) -> &str {
+        self.inner.display_name()
+    }
+
+    async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.readdir(path).await
+    }
+
+    async fn stat(&self, path: &Path) -> Result<FileStat> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.stat(path).await
+    }
+
+    async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        let data = self.inner.read(path, offset, size).await?;
+        Ok(self.truncate_if_partial(data))
+    }
+
+    async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        let data = self.inner.read_full(path).await?;
+        Ok(self.truncate_if_partial(data))
+    }
+
+    async fn exists(&self, path: &Path) -> Result<bool> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.exists(path).await
+    }
+
+    async fn health(&self) -> HealthStatus {
+        let mode = self.fail_mode.read().clone();
+        match mode {
+            FailMode::Healthy => self.inner.health().await,
+            FailMode::ReturnError(_) => HealthStatus::Unhealthy,
+            FailMode::TimeoutMs(ms) => {
+                tokio::time::sleep(Duration::from_millis(ms)).await;
+                HealthStatus::Unhealthy
+            }
+            FailMode::FailAfterN(n) if self.call_count.load(Ordering::SeqCst) >= n => {
+                HealthStatus::Unhealthy
+            }
+            _ => self.inner.health().await,
+        }
+    }
+
+    async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.open_read(path).await
+    }
+
+    async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
+        self.inner.watch(path, callback).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::SystemTime;
+
+    struct MockOrigin {
+        id: OriginId,
+    }
+
+    impl MockOrigin {
+        fn new(id: &str) -> Self {
+            Self {
+                id: OriginId::from(id),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl Origin for MockOrigin {
+        fn id(&self) -> &OriginId {
+            &self.id
+        }
+
+        fn origin_type(&self) -> OriginType {
+            OriginType::Local
+        }
+
+        fn display_name(&self) -> &str {
+            "mock"
+        }
+
+        async fn readdir(&self, _path: &Path) -> Result<Vec<DirEntry>> {
+            Ok(vec![])
+        }
+
+        async fn stat(&self, _path: &Path) -> Result<FileStat> {
+            Ok(FileStat {
+                size: 1000,
+                mtime: SystemTime::now(),
+                is_dir: false,
+            })
+        }
+
+        async fn read(&self, _path: &Path, _offset: u64, size: u32) -> Result<Vec<u8>> {
+            Ok(vec![0u8; size as usize])
+        }
+
+        async fn read_full(&self, _path: &Path) -> Result<Vec<u8>> {
+            Ok(vec![0u8; 100])
+        }
+
+        async fn exists(&self, _path: &Path) -> Result<bool> {
+            Ok(true)
+        }
+
+        async fn health(&self) -> HealthStatus {
+            HealthStatus::Healthy
+        }
+
+        async fn open_read(&self, _path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
+            Err(Error::Origin("Not implemented".into()))
+        }
+
+        async fn watch(&self, _path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
+            Err(Error::Origin("Not implemented".into()))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_healthy_passthrough() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
+
+        let result = faulty.stat(Path::new("/test")).await;
+        assert!(result.is_ok());
+        assert_eq!(faulty.call_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_fail_every_nth() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::FailEveryNth(2));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert_eq!(faulty.call_count(), 4);
+    }
+
+    #[tokio::test]
+    async fn test_fail_after_n() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::FailAfterN(2));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_partial_read() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::PartialRead { max_bytes: 10 });
+
+        let data = faulty.read(Path::new("/test"), 0, 100).await.unwrap();
+        assert_eq!(data.len(), 10);
+    }
+
+    #[tokio::test]
+    async fn test_mode_change_mid_test() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::ReturnError(ErrorKind::ConnectionRefused));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+
+        faulty.set_mode(FailMode::Healthy);
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_health_reflects_mode() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
+
+        assert_eq!(faulty.health().await, HealthStatus::Healthy);
+
+        faulty.set_mode(FailMode::ReturnError(ErrorKind::ConnectionRefused));
+        assert_eq!(faulty.health().await, HealthStatus::Unhealthy);
+    }
+}

musicfs/crates/musicfs-test-utils/src/fixtures.rs

@@ -0,0 +1,255 @@
+use musicfs_cache::TreeBuilder;
+use musicfs_cas::{CasConfig, CasStore};
+use musicfs_core::{
+    AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath,
+};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, RwLock};
+use std::time::SystemTime;
+use tempfile::TempDir;
+
+pub fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: None,
+    }
+}
+
+pub fn make_file_meta_with_origin(id: i64, vpath: &str, size: u64, origin_id: &str) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from(origin_id),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: None,
+    }
+}
+
+pub fn make_audio_meta(artist: &str, album: &str, title: &str) -> AudioMeta {
+    AudioMeta {
+        title: Some(title.to_string()),
+        artist: Some(artist.to_string()),
+        album: Some(album.to_string()),
+        album_artist: None,
+        genre: None,
+        year: None,
+        track: None,
+        disc: None,
+        duration_ms: Some(180_000),
+        bitrate: Some(320),
+        sample_rate: Some(44100),
+        format: AudioFormat::Flac,
+    }
+}
+
+pub fn make_audio_file(
+    id: i64,
+    vpath: &str,
+    size: u64,
+    artist: &str,
+    album: &str,
+    title: &str,
+) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: Some(make_audio_meta(artist, album, title)),
+    }
+}
+
+pub fn make_audio_file_full(
+    id: i64,
+    vpath: &str,
+    size: u64,
+    artist: &str,
+    album: &str,
+    title: &str,
+    track: u32,
+    year: u32,
+) -> FileMeta {
+    let mut audio = make_audio_meta(artist, album, title);
+    audio.track = Some(track);
+    audio.year = Some(year);
+
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: Some(audio),
+    }
+}
+
+pub struct TestCasStore {
+    pub store: Arc<CasStore>,
+    pub dir: TempDir,
+}
+
+pub async fn setup_test_cas() -> TestCasStore {
+    let dir = TempDir::new().expect("Failed to create temp dir for CAS");
+    let config = CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size: 100 * 1024 * 1024,
+        shard_levels: 2,
+    };
+    let store = CasStore::open(config)
+        .await
+        .expect("Failed to open CAS store");
+    TestCasStore {
+        store: Arc::new(store),
+        dir,
+    }
+}
+
+pub async fn setup_test_cas_with_size(max_size: u64) -> TestCasStore {
+    let dir = TempDir::new().expect("Failed to create temp dir for CAS");
+    let config = CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size,
+        shard_levels: 2,
+    };
+    let store = CasStore::open(config)
+        .await
+        .expect("Failed to open CAS store");
+    TestCasStore {
+        store: Arc::new(store),
+        dir,
+    }
+}
+
+pub fn setup_test_tree(files: &[FileMeta]) -> Arc<RwLock<musicfs_cache::VirtualTree>> {
+    let mut builder = TreeBuilder::new();
+    for file in files {
+        builder.add_file(file);
+    }
+    Arc::new(RwLock::new(builder.build()))
+}
+
+pub fn create_test_file(dir: &Path, relative_path: &str, content: &[u8]) -> PathBuf {
+    let full_path = dir.join(relative_path);
+    if let Some(parent) = full_path.parent() {
+        std::fs::create_dir_all(parent).expect("Failed to create parent directories");
+    }
+    std::fs::write(&full_path, content).expect("Failed to write test file");
+    full_path
+}
+
+pub fn create_test_dir_structure(base: &Path, structure: &[&str]) {
+    for path in structure {
+        let full_path = base.join(path);
+        if path.ends_with('/') {
+            std::fs::create_dir_all(&full_path).expect("Failed to create directory");
+        } else {
+            if let Some(parent) = full_path.parent() {
+                std::fs::create_dir_all(parent).expect("Failed to create parent");
+            }
+            std::fs::write(&full_path, format!("content of {}", path))
+                .expect("Failed to write file");
+        }
+    }
+}
+
+pub struct TestOriginDir {
+    pub dir: TempDir,
+}
+
+impl TestOriginDir {
+    pub fn new() -> Self {
+        Self {
+            dir: TempDir::new().expect("Failed to create origin temp dir"),
+        }
+    }
+
+    pub fn add_file(&self, path: &str, content: &[u8]) -> PathBuf {
+        create_test_file(self.dir.path(), path, content)
+    }
+
+    pub fn add_audio_file(&self, path: &str) -> PathBuf {
+        let fake_audio = b"FAKE_FLAC_HEADER_FOR_TESTING_ONLY";
+        self.add_file(path, fake_audio)
+    }
+
+    pub fn path(&self) -> &Path {
+        self.dir.path()
+    }
+}
+
+impl Default for TestOriginDir {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_make_file_meta() {
+        let meta = make_file_meta(1, "/Artist/Album/Track.flac", 1000);
+        assert_eq!(meta.id.0, 1);
+        assert_eq!(meta.virtual_path.as_str(), "/Artist/Album/Track.flac");
+        assert_eq!(meta.size, 1000);
+        assert!(meta.audio.is_none());
+    }
+
+    #[test]
+    fn test_make_audio_file() {
+        let meta = make_audio_file(1, "/path.flac", 5000, "Artist", "Album", "Title");
+        assert!(meta.audio.is_some());
+        let audio = meta.audio.unwrap();
+        assert_eq!(audio.artist, Some("Artist".to_string()));
+        assert_eq!(audio.album, Some("Album".to_string()));
+        assert_eq!(audio.title, Some("Title".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_setup_test_cas() {
+        let test_cas = setup_test_cas().await;
+        let hash = test_cas.store.put(b"test data").await.unwrap();
+        assert!(test_cas.store.exists(&hash));
+    }
+
+    #[test]
+    fn test_setup_test_tree() {
+        let files = vec![
+            make_file_meta(1, "/A/B/1.flac", 100),
+            make_file_meta(2, "/A/B/2.flac", 200),
+        ];
+        let tree = setup_test_tree(&files);
+        let guard = tree.read().unwrap();
+        assert!(guard.file_count() > 0);
+    }
+
+    #[test]
+    fn test_origin_dir() {
+        let origin = TestOriginDir::new();
+        let path = origin.add_file("artist/album/track.flac", b"content");
+        assert!(path.exists());
+    }
+}

musicfs/crates/musicfs-test-utils/src/lib.rs

@@ -0,0 +1,9 @@
+pub mod assertions;
+pub mod faulty_cas;
+pub mod faulty_origin;
+pub mod fixtures;
+
+pub use assertions::*;
+pub use faulty_cas::FaultyCasStore;
+pub use faulty_origin::{FailMode, FaultyOrigin};
+pub use fixtures::*;

musicfs/crates/musicfs-test-utils/tests/docker_network.rs

@@ -0,0 +1,148 @@
+#![cfg(feature = "docker-tests")]
+
+use musicfs_core::{OriginId, OriginType};
+use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
+use noxious_client::{Client, StreamDirection, Toxic, ToxicKind};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+use tempfile::TempDir;
+
+const TOXIPROXY_API: &str = "http://localhost:8474";
+const TOXIPROXY_LISTEN: &str = "localhost:18080";
+const UPSTREAM_ADDR: &str = "minio:9000";
+
+async fn require_toxiproxy() {
+    let available = match reqwest::get(format!("{}/version", TOXIPROXY_API)).await {
+        Ok(resp) => resp.status().is_success(),
+        Err(_) => false,
+    };
+    assert!(available, "Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d", TOXIPROXY_API);
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_latency_injection() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_latency", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let toxic = Toxic {
+        name: "latency_downstream".to_string(),
+        kind: ToxicKind::Latency {
+            latency: 500,
+            jitter: 100,
+        },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy
+        .add_toxic(&toxic)
+        .await
+        .expect("Failed to add toxic");
+
+    let start = std::time::Instant::now();
+    let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    let elapsed = start.elapsed();
+
+    assert!(
+        elapsed >= Duration::from_millis(400),
+        "Latency should be injected, got {:?}",
+        elapsed
+    );
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_timeout_simulates_network_partition() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_partition", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    assert!(result.is_ok(), "Should reach MinIO through proxy initially");
+
+    let toxic = Toxic {
+        name: "timeout".to_string(),
+        kind: ToxicKind::Timeout { timeout: 0 },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy
+        .add_toxic(&toxic)
+        .await
+        .expect("Failed to add toxic");
+
+    let result = tokio::time::timeout(
+        Duration::from_secs(2),
+        reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)),
+    )
+    .await;
+
+    assert!(
+        result.is_err() || result.unwrap().is_err(),
+        "Should timeout during partition"
+    );
+
+    proxy
+        .remove_toxic("timeout")
+        .await
+        .expect("Failed to remove toxic");
+
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    assert!(result.is_ok(), "Should reach MinIO after partition heals");
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_slow_close_throttles_responses() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_slow", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let toxic = Toxic {
+        name: "slow_close".to_string(),
+        kind: ToxicKind::SlowClose { delay: 1000 },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy
+        .add_toxic(&toxic)
+        .await
+        .expect("Failed to add toxic");
+
+    let start = std::time::Instant::now();
+    let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    let elapsed = start.elapsed();
+
+    assert!(
+        elapsed >= Duration::from_millis(800),
+        "Slow close should delay response, got {:?}",
+        elapsed
+    );
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}
+
+

musicfs/crates/musicfs-test-utils/tests/resilience.rs

@@ -0,0 +1,253 @@
+use musicfs_cache::{VirtualTree, ROOT_INODE};
+use musicfs_cas::{CasConfig, CasStore};
+use musicfs_core::{HealthStatus, OriginId, OriginType, RealPath};
+use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
+use musicfs_test_utils::{FaultyOrigin, FailMode};
+use std::collections::HashMap;
+use std::io::ErrorKind;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tempfile::TempDir;
+
+fn setup_test_file(dir: &TempDir, name: &str, content: &[u8]) -> PathBuf {
+    let path = dir.path().join(name);
+    std::fs::write(&path, content).unwrap();
+    path
+}
+
+async fn setup_cas(dir: &Path) -> CasStore {
+    CasStore::open(CasConfig {
+        chunks_dir: dir.join("chunks"),
+        max_size: 100 * 1024 * 1024,
+        shard_levels: 2,
+    })
+    .await
+    .unwrap()
+}
+
+fn create_faulty_origin(id: &str, dir: &TempDir, mode: FailMode) -> Arc<FaultyOrigin> {
+    let inner = Arc::new(LocalOrigin::new(OriginId::from(id), dir.path().to_path_buf()));
+    Arc::new(FaultyOrigin::new(inner, mode))
+}
+
+#[tokio::test]
+async fn test_sqlite_integrity_check_detects_corruption() {
+    todo!("Issue 2.4: Implement Database::open_with_integrity_check()")
+}
+
+#[tokio::test]
+async fn test_tantivy_corruption_triggers_rebuild() {
+    todo!("Issue 2.4: Implement SearchIndex::open_with_recovery()")
+}
+
+#[tokio::test]
+async fn test_sled_corruption_triggers_repair() {
+    todo!("Issue 3.5: Implement sled recovery in CasStore::open()")
+}
+
+#[tokio::test]
+async fn test_cas_put_handles_enospc() {
+    let dir = TempDir::new().unwrap();
+    let store = CasStore::open(CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size: 100,
+        shard_levels: 2,
+    })
+    .await
+    .unwrap();
+
+    let large_data = vec![0u8; 1000];
+    let result = store.put(&large_data).await;
+    
+    assert!(result.is_err(), "Issue 2.8: CasStore should pre-check space and reject oversized write");
+}
+
+#[test]
+fn test_poisoned_tree_lock_returns_eio_not_panic() {
+    use std::sync::{Arc, RwLock};
+    use std::thread;
+
+    let lock = Arc::new(RwLock::new(42));
+    let lock_clone = lock.clone();
+
+    let handle = thread::spawn(move || {
+        let _guard = lock_clone.write().unwrap();
+        panic!("writer panic");
+    });
+
+    let _ = handle.join();
+
+    let result = lock.read();
+    assert!(result.is_ok(), "Issue 2.9: Lock access after panic should return EIO, not poison error");
+}
+
+#[test]
+fn test_parking_lot_rwlock_survives_panic() {
+    use parking_lot::RwLock;
+    use std::sync::Arc;
+    use std::thread;
+
+    let tree = Arc::new(RwLock::new(VirtualTree::new()));
+    let tree_clone = tree.clone();
+
+    let handle = thread::spawn(move || {
+        let _guard = tree_clone.write();
+        panic!("writer panic");
+    });
+
+    let _ = handle.join();
+
+    assert!(tree.read().get(ROOT_INODE).is_some(), "parking_lot RwLock should survive writer panic");
+}
+
+#[tokio::test]
+async fn test_failover_on_primary_death() {
+    let primary_dir = TempDir::new().unwrap();
+    let backup_dir = TempDir::new().unwrap();
+    setup_test_file(&primary_dir, "test.txt", b"primary");
+    setup_test_file(&backup_dir, "test.txt", b"backup");
+
+    let primary = create_faulty_origin("primary", &primary_dir, FailMode::ReturnError(ErrorKind::ConnectionRefused));
+    let backup = create_faulty_origin("backup", &backup_dir, FailMode::Healthy);
+
+    let mut thresholds = HashMap::new();
+    thresholds.insert(OriginType::Local, 1);
+    let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
+    let registry = Arc::new(OriginRegistry::new(monitor.clone()));
+
+    registry.register(primary.clone(), 1);
+    registry.register(backup.clone(), 2);
+
+    monitor.check_now(&OriginId::from("primary")).await;
+    monitor.check_now(&OriginId::from("backup")).await;
+
+    assert!(registry.health().is_unhealthy(&OriginId::from("primary")));
+    assert!(registry.health().is_healthy(&OriginId::from("backup")));
+
+    let path = RealPath {
+        origin_id: OriginId::from("backup"),
+        path: PathBuf::from("/test.txt"),
+    };
+    let candidates = registry.route_all(&path);
+    assert_eq!(candidates.len(), 1);
+    assert_eq!(candidates[0].id(), &OriginId::from("backup"));
+}
+
+#[tokio::test]
+async fn test_origin_recovery_resumes_routing() {
+    let dir = TempDir::new().unwrap();
+    setup_test_file(&dir, "test.txt", b"content");
+
+    let faulty = create_faulty_origin("recovering", &dir, FailMode::ReturnError(ErrorKind::ConnectionRefused));
+
+    let mut thresholds = HashMap::new();
+    thresholds.insert(OriginType::Local, 1);
+    let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
+    monitor.add_origin(faulty.clone());
+
+    monitor.check_now(&OriginId::from("recovering")).await;
+    assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().status, HealthStatus::Unhealthy);
+
+    faulty.set_mode(FailMode::Healthy);
+    monitor.check_now(&OriginId::from("recovering")).await;
+
+    assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().status, HealthStatus::Healthy);
+    assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().consecutive_failures, 0);
+}
+
+#[tokio::test]
+async fn test_local_origin_health_check_has_timeout() {
+    let dir = TempDir::new().unwrap();
+    setup_test_file(&dir, "test.txt", b"content");
+
+    let slow = create_faulty_origin("slow", &dir, FailMode::TimeoutMs(5_000));
+
+    let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
+    monitor.add_origin(slow.clone());
+
+    let start = Instant::now();
+    monitor.check_now(&OriginId::from("slow")).await;
+    let elapsed = start.elapsed();
+
+    assert!(elapsed < Duration::from_secs(2), 
+        "Issue 4.2.1: Health check should timeout in <2s, took {:?}", elapsed);
+    
+    let state = monitor.get_state(&OriginId::from("slow")).unwrap();
+    assert_eq!(state.status, HealthStatus::Unhealthy);
+}
+
+#[tokio::test]
+async fn test_health_checks_run_in_parallel() {
+    let slow1_dir = TempDir::new().unwrap();
+    let slow2_dir = TempDir::new().unwrap();
+    let slow3_dir = TempDir::new().unwrap();
+
+    let slow1 = create_faulty_origin("slow1", &slow1_dir, FailMode::TimeoutMs(200));
+    let slow2 = create_faulty_origin("slow2", &slow2_dir, FailMode::TimeoutMs(200));
+    let slow3 = create_faulty_origin("slow3", &slow3_dir, FailMode::TimeoutMs(200));
+
+    let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
+    monitor.add_origin(slow1);
+    monitor.add_origin(slow2);
+    monitor.add_origin(slow3);
+
+    let start = Instant::now();
+    monitor.check_all().await;
+    let elapsed = start.elapsed();
+
+    assert!(elapsed < Duration::from_millis(350), "Issue 4.2.2: check_all() should run in parallel (sequential would take ~600ms), took {:?}", elapsed);
+}
+
+#[tokio::test]
+async fn test_tantivy_survives_uncommitted_crash() {
+    todo!("Issue 5.2: Implement tantivy crash recovery test")
+}
+
+#[tokio::test]
+async fn test_fd_exhaustion_handling() {
+    todo!("Issue 5.3: Implement fd exhaustion test with rlimit")
+}
+
+#[tokio::test]
+async fn test_corrupt_chunk_auto_refetched() {
+    let dir = TempDir::new().unwrap();
+    let origin_dir = TempDir::new().unwrap();
+    setup_test_file(&origin_dir, "test.flac", b"original audio data");
+
+    let store = setup_cas(dir.path()).await;
+    let data = b"chunk data";
+    let hash = store.put(data).await.unwrap();
+
+    let hex = hash.as_hex();
+    let chunk_path = dir.path().join("chunks").join(&hex[0..2]).join(&hex[2..4]).join(&hex);
+    let mut corrupted = std::fs::read(&chunk_path).unwrap();
+    corrupted[0] = corrupted[0].wrapping_add(1);
+    std::fs::write(&chunk_path, &corrupted).unwrap();
+
+    let result = store.get(&hash).await;
+    
+    assert!(result.is_ok(), "Issue 6.4: Corrupted chunk should be auto-refetched from origin");
+}
+
+#[tokio::test]
+async fn test_missing_chunk_triggers_origin_fetch() {
+    todo!("Issue 6.4: Implement missing chunk origin fetch")
+}
+
+#[tokio::test]
+async fn test_passthrough_mode_when_cache_disk_dead() {
+    todo!("Issue 6.6: Implement passthrough mode")
+}
+
+#[test]
+fn test_systemd_service_has_execstoppost() {
+    let service_path = std::path::Path::new("../../../systemd/musicfs.service");
+    if !service_path.exists() {
+        panic!("Issue 3.7: systemd/musicfs.service does not exist");
+    }
+    
+    let content = std::fs::read_to_string(service_path).unwrap();
+    assert!(content.contains("ExecStopPost") || content.contains("fusermount"), 
+            "Issue 3.7: Service file should have ExecStopPost with fusermount for cleanup");
+}