Move the files around

crates/musicfs-test-utils/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "musicfs-test-utils"
+version.workspace = true
+edition.workspace = true
+description = "Test utilities and fixtures for MusicFS resilience testing"
+
+[dependencies]
+musicfs-core = { path = "../musicfs-core" }
+musicfs-origins = { path = "../musicfs-origins" }
+musicfs-cas = { path = "../musicfs-cas" }
+musicfs-cache = { path = "../musicfs-cache" }
+musicfs-search = { path = "../musicfs-search" }
+
+async-trait.workspace = true
+tokio = { workspace = true, features = ["full", "sync", "time"] }
+tracing.workspace = true
+thiserror.workspace = true
+parking_lot.workspace = true
+tempfile.workspace = true
+bytes.workspace = true
+
+# Fault injection
+fail = { version = "0.5", optional = true }
+rlimit = { version = "0.10", optional = true }
+nix = { version = "0.29", optional = true, features = ["signal", "process"] }
+
+# Docker/network tests
+noxious-client = { version = "1.0", optional = true }
+reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls"] }
+
+[features]
+default = []
+failpoints = ["fail/failpoints"]
+process-tests = ["nix"]
+resource-limits = ["rlimit"]
+docker-tests = ["noxious-client", "reqwest"]
+full = ["failpoints", "process-tests", "resource-limits", "docker-tests"]
+
+[dev-dependencies]
+tokio-test = "0.4"
+tokio-util.workspace = true
+sd-notify.workspace = true
+libc.workspace = true

crates/musicfs-test-utils/src/assertions.rs

@@ -0,0 +1,204 @@
+use musicfs_cas::CasError;
+use musicfs_core::Error;
+use std::time::{Duration, Instant};
+
+pub fn assert_error_contains<T, E: std::fmt::Debug>(result: Result<T, E>, expected_text: &str) {
+    match result {
+        Ok(_) => panic!("Expected error containing '{}', but got Ok", expected_text),
+        Err(e) => {
+            let error_msg = format!("{:?}", e);
+            assert!(
+                error_msg.contains(expected_text),
+                "Expected error containing '{}', but got: {}",
+                expected_text,
+                error_msg
+            );
+        }
+    }
+}
+
+pub fn assert_io_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Io(_)) => (),
+        Err(e) => panic!("Expected Io error, got: {:?}", e),
+        Ok(_) => panic!("Expected Io error, got Ok"),
+    }
+}
+
+pub fn assert_cas_io_error<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::Io(_)) => (),
+        Err(e) => panic!("Expected CasError::Io, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::Io, got Ok"),
+    }
+}
+
+pub fn assert_cas_not_found<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::NotFound(_)) => (),
+        Err(e) => panic!("Expected CasError::NotFound, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::NotFound, got Ok"),
+    }
+}
+
+pub fn assert_cas_integrity_error<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::IntegrityError { .. }) => (),
+        Err(e) => panic!("Expected CasError::IntegrityError, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::IntegrityError, got Ok"),
+    }
+}
+
+pub fn assert_file_not_found<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::FileNotFound(_)) => (),
+        Err(e) => panic!("Expected FileNotFound error, got: {:?}", e),
+        Ok(_) => panic!("Expected FileNotFound error, got Ok"),
+    }
+}
+
+pub fn assert_origin_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Origin(_)) => (),
+        Err(e) => panic!("Expected Origin error, got: {:?}", e),
+        Ok(_) => panic!("Expected Origin error, got Ok"),
+    }
+}
+
+pub fn assert_timeout_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Timeout(_)) => (),
+        Err(e) => panic!("Expected Timeout error, got: {:?}", e),
+        Ok(_) => panic!("Expected Timeout error, got Ok"),
+    }
+}
+
+pub struct TimedAssertion {
+    start: Instant,
+    min_duration: Option<Duration>,
+    max_duration: Option<Duration>,
+}
+
+impl TimedAssertion {
+    pub fn new() -> Self {
+        Self {
+            start: Instant::now(),
+            min_duration: None,
+            max_duration: None,
+        }
+    }
+
+    pub fn expect_at_least(mut self, duration: Duration) -> Self {
+        self.min_duration = Some(duration);
+        self
+    }
+
+    pub fn expect_at_most(mut self, duration: Duration) -> Self {
+        self.max_duration = Some(duration);
+        self
+    }
+
+    pub fn assert_elapsed(self) {
+        let elapsed = self.start.elapsed();
+
+        if let Some(min) = self.min_duration {
+            assert!(
+                elapsed >= min,
+                "Expected at least {:?}, but only {:?} elapsed",
+                min,
+                elapsed
+            );
+        }
+
+        if let Some(max) = self.max_duration {
+            assert!(
+                elapsed <= max,
+                "Expected at most {:?}, but {:?} elapsed",
+                max,
+                elapsed
+            );
+        }
+    }
+}
+
+impl Default for TimedAssertion {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+pub async fn assert_completes_within<F, T>(future: F, timeout: Duration) -> T
+where
+    F: std::future::Future<Output = T>,
+{
+    tokio::time::timeout(timeout, future)
+        .await
+        .expect(&format!("Operation did not complete within {:?}", timeout))
+}
+
+pub async fn assert_times_out<F, T>(future: F, timeout: Duration)
+where
+    F: std::future::Future<Output = T>,
+{
+    match tokio::time::timeout(timeout, future).await {
+        Ok(_) => panic!("Expected operation to time out, but it completed"),
+        Err(_) => (),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_assert_error_contains() {
+        let result: Result<(), Error> = Err(Error::Origin("connection refused".into()));
+        assert_error_contains(result, "connection");
+    }
+
+    #[test]
+    #[should_panic(expected = "Expected error containing")]
+    fn test_assert_error_contains_failure() {
+        let result: Result<(), Error> = Err(Error::Origin("something else".into()));
+        assert_error_contains(result, "connection");
+    }
+
+    #[test]
+    fn test_assert_io_error() {
+        let result: Result<(), Error> = Err(Error::Io(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "test",
+        )));
+        assert_io_error(result);
+    }
+
+    #[test]
+    fn test_timed_assertion_at_least() {
+        let timer = TimedAssertion::new().expect_at_least(Duration::from_millis(10));
+        std::thread::sleep(Duration::from_millis(15));
+        timer.assert_elapsed();
+    }
+
+    #[test]
+    fn test_timed_assertion_at_most() {
+        let timer = TimedAssertion::new().expect_at_most(Duration::from_millis(100));
+        timer.assert_elapsed();
+    }
+
+    #[tokio::test]
+    async fn test_assert_completes_within() {
+        let result = assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
+        assert_eq!(result, 42);
+    }
+
+    #[tokio::test]
+    async fn test_assert_times_out() {
+        assert_times_out(
+            async {
+                tokio::time::sleep(Duration::from_secs(10)).await;
+            },
+            Duration::from_millis(10),
+        )
+        .await;
+    }
+}

crates/musicfs-test-utils/src/faulty_cas.rs

@@ -0,0 +1,250 @@
+use bytes::Bytes;
+use musicfs_cas::{CasConfig, CasError, CasStore, DedupStats};
+use musicfs_core::ChunkHash;
+use std::io::{self, ErrorKind};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
+
+pub struct FaultyCasStore {
+    inner: Arc<CasStore>,
+    inject_enospc: AtomicBool,
+    inject_eio_on_read: AtomicBool,
+    inject_eio_on_write: AtomicBool,
+    inject_corruption: AtomicBool,
+    fail_after_n_puts: AtomicUsize,
+    put_count: AtomicUsize,
+}
+
+impl FaultyCasStore {
+    pub fn new(inner: Arc<CasStore>) -> Self {
+        Self {
+            inner,
+            inject_enospc: AtomicBool::new(false),
+            inject_eio_on_read: AtomicBool::new(false),
+            inject_eio_on_write: AtomicBool::new(false),
+            inject_corruption: AtomicBool::new(false),
+            fail_after_n_puts: AtomicUsize::new(usize::MAX),
+            put_count: AtomicUsize::new(0),
+        }
+    }
+
+    pub async fn open(config: CasConfig) -> Result<Self, CasError> {
+        let store = CasStore::open(config).await?;
+        Ok(Self::new(Arc::new(store)))
+    }
+
+    pub fn set_inject_enospc(&self, enabled: bool) {
+        self.inject_enospc.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_eio_on_read(&self, enabled: bool) {
+        self.inject_eio_on_read.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_eio_on_write(&self, enabled: bool) {
+        self.inject_eio_on_write.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_corruption(&self, enabled: bool) {
+        self.inject_corruption.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_fail_after_n_puts(&self, n: usize) {
+        self.fail_after_n_puts.store(n, Ordering::SeqCst);
+        self.put_count.store(0, Ordering::SeqCst);
+    }
+
+    pub fn reset_faults(&self) {
+        self.inject_enospc.store(false, Ordering::SeqCst);
+        self.inject_eio_on_read.store(false, Ordering::SeqCst);
+        self.inject_eio_on_write.store(false, Ordering::SeqCst);
+        self.inject_corruption.store(false, Ordering::SeqCst);
+        self.fail_after_n_puts.store(usize::MAX, Ordering::SeqCst);
+        self.put_count.store(0, Ordering::SeqCst);
+    }
+
+    pub fn put_count(&self) -> usize {
+        self.put_count.load(Ordering::SeqCst)
+    }
+
+    pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
+        let count = self.put_count.fetch_add(1, Ordering::SeqCst);
+
+        if self.inject_enospc.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "No space left on device (ENOSPC injected)",
+            )));
+        }
+
+        if self.inject_eio_on_write.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+
+        let threshold = self.fail_after_n_puts.load(Ordering::SeqCst);
+        if count >= threshold {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Injected failure after N puts",
+            )));
+        }
+
+        self.inner.put(data).await
+    }
+
+    pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
+        if self.inject_eio_on_read.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+
+        let data = self.inner.get(hash).await?;
+
+        if self.inject_corruption.load(Ordering::SeqCst) {
+            let mut corrupted = data.to_vec();
+            if !corrupted.is_empty() {
+                corrupted[0] = corrupted[0].wrapping_add(1);
+            }
+            return Err(CasError::IntegrityError {
+                expected: hash.as_hex(),
+                actual: ChunkHash::from_bytes(&corrupted).as_hex(),
+            });
+        }
+
+        Ok(data)
+    }
+
+    pub fn exists(&self, hash: &ChunkHash) -> bool {
+        self.inner.exists(hash)
+    }
+
+    pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
+        if self.inject_eio_on_write.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+        self.inner.delete(hash).await
+    }
+
+    pub fn current_size(&self) -> u64 {
+        self.inner.current_size()
+    }
+
+    pub fn max_size(&self) -> u64 {
+        self.inner.max_size()
+    }
+
+    pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
+        self.inner.list_chunks()
+    }
+
+    pub fn dedup_stats(&self) -> DedupStats {
+        self.inner.dedup_stats()
+    }
+
+    pub fn inner(&self) -> &Arc<CasStore> {
+        &self.inner
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    async fn test_store() -> (FaultyCasStore, TempDir) {
+        let dir = TempDir::new().unwrap();
+        let config = CasConfig {
+            chunks_dir: dir.path().join("chunks"),
+            max_size: 1024 * 1024,
+            shard_levels: 2,
+        };
+        let store = FaultyCasStore::open(config).await.unwrap();
+        (store, dir)
+    }
+
+    #[tokio::test]
+    async fn test_healthy_passthrough() {
+        let (store, _dir) = test_store().await;
+
+        let data = b"test data";
+        let hash = store.put(data).await.unwrap();
+        let retrieved = store.get(&hash).await.unwrap();
+        assert_eq!(&retrieved[..], data);
+    }
+
+    #[tokio::test]
+    async fn test_inject_enospc() {
+        let (store, _dir) = test_store().await;
+
+        store.set_inject_enospc(true);
+        let result = store.put(b"test").await;
+        assert!(result.is_err());
+
+        let err = result.unwrap_err();
+        assert!(matches!(err, CasError::Io(_)));
+
+        store.set_inject_enospc(false);
+        assert!(store.put(b"test").await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_inject_eio_on_read() {
+        let (store, _dir) = test_store().await;
+
+        let hash = store.put(b"test").await.unwrap();
+
+        store.set_inject_eio_on_read(true);
+        let result = store.get(&hash).await;
+        assert!(result.is_err());
+
+        store.set_inject_eio_on_read(false);
+        assert!(store.get(&hash).await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_inject_corruption() {
+        let (store, _dir) = test_store().await;
+
+        let hash = store.put(b"test data").await.unwrap();
+
+        store.set_inject_corruption(true);
+        let result = store.get(&hash).await;
+        assert!(matches!(result, Err(CasError::IntegrityError { .. })));
+    }
+
+    #[tokio::test]
+    async fn test_fail_after_n_puts() {
+        let (store, _dir) = test_store().await;
+
+        store.set_fail_after_n_puts(2);
+
+        assert!(store.put(b"data1").await.is_ok());
+        assert!(store.put(b"data2").await.is_ok());
+        assert!(store.put(b"data3").await.is_err());
+        assert!(store.put(b"data4").await.is_err());
+        assert_eq!(store.put_count(), 4);
+    }
+
+    #[tokio::test]
+    async fn test_reset_faults() {
+        let (store, _dir) = test_store().await;
+
+        store.set_inject_enospc(true);
+        store.set_inject_eio_on_read(true);
+        store.set_fail_after_n_puts(1);
+
+        store.reset_faults();
+
+        assert!(store.put(b"test").await.is_ok());
+        let hash = store.put(b"test2").await.unwrap();
+        assert!(store.get(&hash).await.is_ok());
+    }
+}

crates/musicfs-test-utils/src/faulty_origin.rs

@@ -0,0 +1,328 @@
+use async_trait::async_trait;
+use musicfs_core::{DirEntry, Error, FileStat, HealthStatus, OriginId, OriginType, Result};
+use musicfs_origins::{Origin, WatchCallback, WatchHandle};
+use parking_lot::RwLock;
+use std::io::{self, ErrorKind};
+use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::io::AsyncRead;
+
+#[derive(Debug, Clone)]
+pub enum FailMode {
+    Healthy,
+    FailEveryNth(usize),
+    FailAfterN(usize),
+    TimeoutMs(u64),
+    PartialRead { max_bytes: usize },
+    ReturnError(ErrorKind),
+}
+
+impl Default for FailMode {
+    fn default() -> Self {
+        FailMode::Healthy
+    }
+}
+
+pub struct FaultyOrigin {
+    inner: Arc<dyn Origin>,
+    fail_mode: Arc<RwLock<FailMode>>,
+    call_count: AtomicUsize,
+}
+
+impl FaultyOrigin {
+    pub fn new(inner: Arc<dyn Origin>, mode: FailMode) -> Self {
+        Self {
+            inner,
+            fail_mode: Arc::new(RwLock::new(mode)),
+            call_count: AtomicUsize::new(0),
+        }
+    }
+
+    pub fn wrap(inner: impl Origin + 'static) -> Self {
+        Self::new(Arc::new(inner), FailMode::Healthy)
+    }
+
+    pub fn set_mode(&self, mode: FailMode) {
+        *self.fail_mode.write() = mode;
+    }
+
+    pub fn call_count(&self) -> usize {
+        self.call_count.load(Ordering::SeqCst)
+    }
+
+    pub fn reset_count(&self) {
+        self.call_count.store(0, Ordering::SeqCst);
+    }
+
+    fn increment_and_check(&self) -> Option<Error> {
+        let count = self.call_count.fetch_add(1, Ordering::SeqCst) + 1;
+        let mode = self.fail_mode.read();
+
+        match *mode {
+            FailMode::Healthy => None,
+            FailMode::FailEveryNth(n) if n > 0 && count % n == 0 => {
+                Some(Error::Origin("Injected failure (every Nth)".into()))
+            }
+            FailMode::FailEveryNth(_) => None,
+            FailMode::FailAfterN(n) if count > n => {
+                Some(Error::Origin("Injected failure (after N)".into()))
+            }
+            FailMode::FailAfterN(_) => None,
+            FailMode::TimeoutMs(_) => None,
+            FailMode::PartialRead { .. } => None,
+            FailMode::ReturnError(kind) => {
+                Some(Error::Io(io::Error::new(kind, "Injected I/O error")))
+            }
+        }
+    }
+
+    async fn maybe_timeout(&self) -> Option<Error> {
+        let mode = self.fail_mode.read().clone();
+        if let FailMode::TimeoutMs(ms) = mode {
+            tokio::time::sleep(Duration::from_millis(ms)).await;
+            Some(Error::Timeout("Injected timeout".into()))
+        } else {
+            None
+        }
+    }
+
+    fn truncate_if_partial(&self, mut data: Vec<u8>) -> Vec<u8> {
+        let mode = self.fail_mode.read();
+        if let FailMode::PartialRead { max_bytes } = *mode {
+            data.truncate(max_bytes);
+        }
+        data
+    }
+}
+
+#[async_trait]
+impl Origin for FaultyOrigin {
+    fn id(&self) -> &OriginId {
+        self.inner.id()
+    }
+
+    fn origin_type(&self) -> OriginType {
+        self.inner.origin_type()
+    }
+
+    fn display_name(&self) -> &str {
+        self.inner.display_name()
+    }
+
+    async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.readdir(path).await
+    }
+
+    async fn stat(&self, path: &Path) -> Result<FileStat> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.stat(path).await
+    }
+
+    async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        let data = self.inner.read(path, offset, size).await?;
+        Ok(self.truncate_if_partial(data))
+    }
+
+    async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        let data = self.inner.read_full(path).await?;
+        Ok(self.truncate_if_partial(data))
+    }
+
+    async fn exists(&self, path: &Path) -> Result<bool> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.exists(path).await
+    }
+
+    async fn health(&self) -> HealthStatus {
+        let mode = self.fail_mode.read().clone();
+        match mode {
+            FailMode::Healthy => self.inner.health().await,
+            FailMode::ReturnError(_) => HealthStatus::Unhealthy,
+            FailMode::TimeoutMs(ms) => {
+                tokio::time::sleep(Duration::from_millis(ms)).await;
+                HealthStatus::Unhealthy
+            }
+            FailMode::FailAfterN(n) if self.call_count.load(Ordering::SeqCst) >= n => {
+                HealthStatus::Unhealthy
+            }
+            _ => self.inner.health().await,
+        }
+    }
+
+    async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.open_read(path).await
+    }
+
+    async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
+        self.inner.watch(path, callback).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::SystemTime;
+
+    struct MockOrigin {
+        id: OriginId,
+    }
+
+    impl MockOrigin {
+        fn new(id: &str) -> Self {
+            Self {
+                id: OriginId::from(id),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl Origin for MockOrigin {
+        fn id(&self) -> &OriginId {
+            &self.id
+        }
+
+        fn origin_type(&self) -> OriginType {
+            OriginType::Local
+        }
+
+        fn display_name(&self) -> &str {
+            "mock"
+        }
+
+        async fn readdir(&self, _path: &Path) -> Result<Vec<DirEntry>> {
+            Ok(vec![])
+        }
+
+        async fn stat(&self, _path: &Path) -> Result<FileStat> {
+            Ok(FileStat {
+                size: 1000,
+                mtime: SystemTime::now(),
+                is_dir: false,
+            })
+        }
+
+        async fn read(&self, _path: &Path, _offset: u64, size: u32) -> Result<Vec<u8>> {
+            Ok(vec![0u8; size as usize])
+        }
+
+        async fn read_full(&self, _path: &Path) -> Result<Vec<u8>> {
+            Ok(vec![0u8; 100])
+        }
+
+        async fn exists(&self, _path: &Path) -> Result<bool> {
+            Ok(true)
+        }
+
+        async fn health(&self) -> HealthStatus {
+            HealthStatus::Healthy
+        }
+
+        async fn open_read(&self, _path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
+            Err(Error::Origin("Not implemented".into()))
+        }
+
+        async fn watch(&self, _path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
+            Err(Error::Origin("Not implemented".into()))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_healthy_passthrough() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
+
+        let result = faulty.stat(Path::new("/test")).await;
+        assert!(result.is_ok());
+        assert_eq!(faulty.call_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_fail_every_nth() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::FailEveryNth(2));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert_eq!(faulty.call_count(), 4);
+    }
+
+    #[tokio::test]
+    async fn test_fail_after_n() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::FailAfterN(2));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_partial_read() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::PartialRead { max_bytes: 10 });
+
+        let data = faulty.read(Path::new("/test"), 0, 100).await.unwrap();
+        assert_eq!(data.len(), 10);
+    }
+
+    #[tokio::test]
+    async fn test_mode_change_mid_test() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::ReturnError(ErrorKind::ConnectionRefused));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+
+        faulty.set_mode(FailMode::Healthy);
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_health_reflects_mode() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
+
+        assert_eq!(faulty.health().await, HealthStatus::Healthy);
+
+        faulty.set_mode(FailMode::ReturnError(ErrorKind::ConnectionRefused));
+        assert_eq!(faulty.health().await, HealthStatus::Unhealthy);
+    }
+}

crates/musicfs-test-utils/src/fixtures.rs

@@ -0,0 +1,253 @@
+use musicfs_cache::TreeBuilder;
+use musicfs_cas::{CasConfig, CasStore};
+use musicfs_core::{AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, RwLock};
+use std::time::SystemTime;
+use tempfile::TempDir;
+
+pub fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: None,
+    }
+}
+
+pub fn make_file_meta_with_origin(id: i64, vpath: &str, size: u64, origin_id: &str) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from(origin_id),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: None,
+    }
+}
+
+pub fn make_audio_meta(artist: &str, album: &str, title: &str) -> AudioMeta {
+    AudioMeta {
+        title: Some(title.to_string()),
+        artist: Some(artist.to_string()),
+        album: Some(album.to_string()),
+        album_artist: None,
+        genre: None,
+        year: None,
+        track: None,
+        disc: None,
+        duration_ms: Some(180_000),
+        bitrate: Some(320),
+        sample_rate: Some(44100),
+        format: AudioFormat::Flac,
+    }
+}
+
+pub fn make_audio_file(
+    id: i64,
+    vpath: &str,
+    size: u64,
+    artist: &str,
+    album: &str,
+    title: &str,
+) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: Some(make_audio_meta(artist, album, title)),
+    }
+}
+
+pub fn make_audio_file_full(
+    id: i64,
+    vpath: &str,
+    size: u64,
+    artist: &str,
+    album: &str,
+    title: &str,
+    track: u32,
+    year: u32,
+) -> FileMeta {
+    let mut audio = make_audio_meta(artist, album, title);
+    audio.track = Some(track);
+    audio.year = Some(year);
+
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: Some(audio),
+    }
+}
+
+pub struct TestCasStore {
+    pub store: Arc<CasStore>,
+    pub dir: TempDir,
+}
+
+pub async fn setup_test_cas() -> TestCasStore {
+    let dir = TempDir::new().expect("Failed to create temp dir for CAS");
+    let config = CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size: 100 * 1024 * 1024,
+        shard_levels: 2,
+    };
+    let store = CasStore::open(config)
+        .await
+        .expect("Failed to open CAS store");
+    TestCasStore {
+        store: Arc::new(store),
+        dir,
+    }
+}
+
+pub async fn setup_test_cas_with_size(max_size: u64) -> TestCasStore {
+    let dir = TempDir::new().expect("Failed to create temp dir for CAS");
+    let config = CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size,
+        shard_levels: 2,
+    };
+    let store = CasStore::open(config)
+        .await
+        .expect("Failed to open CAS store");
+    TestCasStore {
+        store: Arc::new(store),
+        dir,
+    }
+}
+
+pub fn setup_test_tree(files: &[FileMeta]) -> Arc<RwLock<musicfs_cache::VirtualTree>> {
+    let mut builder = TreeBuilder::new();
+    for file in files {
+        builder.add_file(file);
+    }
+    Arc::new(RwLock::new(builder.build()))
+}
+
+pub fn create_test_file(dir: &Path, relative_path: &str, content: &[u8]) -> PathBuf {
+    let full_path = dir.join(relative_path);
+    if let Some(parent) = full_path.parent() {
+        std::fs::create_dir_all(parent).expect("Failed to create parent directories");
+    }
+    std::fs::write(&full_path, content).expect("Failed to write test file");
+    full_path
+}
+
+pub fn create_test_dir_structure(base: &Path, structure: &[&str]) {
+    for path in structure {
+        let full_path = base.join(path);
+        if path.ends_with('/') {
+            std::fs::create_dir_all(&full_path).expect("Failed to create directory");
+        } else {
+            if let Some(parent) = full_path.parent() {
+                std::fs::create_dir_all(parent).expect("Failed to create parent");
+            }
+            std::fs::write(&full_path, format!("content of {}", path))
+                .expect("Failed to write file");
+        }
+    }
+}
+
+pub struct TestOriginDir {
+    pub dir: TempDir,
+}
+
+impl TestOriginDir {
+    pub fn new() -> Self {
+        Self {
+            dir: TempDir::new().expect("Failed to create origin temp dir"),
+        }
+    }
+
+    pub fn add_file(&self, path: &str, content: &[u8]) -> PathBuf {
+        create_test_file(self.dir.path(), path, content)
+    }
+
+    pub fn add_audio_file(&self, path: &str) -> PathBuf {
+        let fake_audio = b"FAKE_FLAC_HEADER_FOR_TESTING_ONLY";
+        self.add_file(path, fake_audio)
+    }
+
+    pub fn path(&self) -> &Path {
+        self.dir.path()
+    }
+}
+
+impl Default for TestOriginDir {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_make_file_meta() {
+        let meta = make_file_meta(1, "/Artist/Album/Track.flac", 1000);
+        assert_eq!(meta.id.0, 1);
+        assert_eq!(meta.virtual_path.as_str(), "/Artist/Album/Track.flac");
+        assert_eq!(meta.size, 1000);
+        assert!(meta.audio.is_none());
+    }
+
+    #[test]
+    fn test_make_audio_file() {
+        let meta = make_audio_file(1, "/path.flac", 5000, "Artist", "Album", "Title");
+        assert!(meta.audio.is_some());
+        let audio = meta.audio.unwrap();
+        assert_eq!(audio.artist, Some("Artist".to_string()));
+        assert_eq!(audio.album, Some("Album".to_string()));
+        assert_eq!(audio.title, Some("Title".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_setup_test_cas() {
+        let test_cas = setup_test_cas().await;
+        let hash = test_cas.store.put(b"test data").await.unwrap();
+        assert!(test_cas.store.exists(&hash));
+    }
+
+    #[test]
+    fn test_setup_test_tree() {
+        let files = vec![
+            make_file_meta(1, "/A/B/1.flac", 100),
+            make_file_meta(2, "/A/B/2.flac", 200),
+        ];
+        let tree = setup_test_tree(&files);
+        let guard = tree.read().unwrap();
+        assert!(guard.file_count() > 0);
+    }
+
+    #[test]
+    fn test_origin_dir() {
+        let origin = TestOriginDir::new();
+        let path = origin.add_file("artist/album/track.flac", b"content");
+        assert!(path.exists());
+    }
+}

crates/musicfs-test-utils/src/lib.rs

@@ -0,0 +1,9 @@
+pub mod assertions;
+pub mod faulty_cas;
+pub mod faulty_origin;
+pub mod fixtures;
+
+pub use assertions::*;
+pub use faulty_cas::FaultyCasStore;
+pub use faulty_origin::{FailMode, FaultyOrigin};
+pub use fixtures::*;

crates/musicfs-test-utils/tests/docker_network.rs

@@ -0,0 +1,141 @@
+#![cfg(feature = "docker-tests")]
+
+use musicfs_core::{OriginId, OriginType};
+use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
+use noxious_client::{Client, StreamDirection, Toxic, ToxicKind};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+use tempfile::TempDir;
+
+const TOXIPROXY_API: &str = "http://localhost:8474";
+const TOXIPROXY_LISTEN: &str = "localhost:18080";
+const UPSTREAM_ADDR: &str = "minio:9000";
+
+async fn require_toxiproxy() {
+    let available = match reqwest::get(format!("{}/version", TOXIPROXY_API)).await {
+        Ok(resp) => resp.status().is_success(),
+        Err(_) => false,
+    };
+    assert!(
+        available,
+        "Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d",
+        TOXIPROXY_API
+    );
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_latency_injection() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_latency", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let toxic = Toxic {
+        name: "latency_downstream".to_string(),
+        kind: ToxicKind::Latency {
+            latency: 500,
+            jitter: 100,
+        },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
+
+    let start = std::time::Instant::now();
+    let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    let elapsed = start.elapsed();
+
+    assert!(
+        elapsed >= Duration::from_millis(400),
+        "Latency should be injected, got {:?}",
+        elapsed
+    );
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_timeout_simulates_network_partition() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_partition", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    assert!(result.is_ok(), "Should reach MinIO through proxy initially");
+
+    let toxic = Toxic {
+        name: "timeout".to_string(),
+        kind: ToxicKind::Timeout { timeout: 0 },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
+
+    let result = tokio::time::timeout(
+        Duration::from_secs(2),
+        reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)),
+    )
+    .await;
+
+    assert!(
+        result.is_err() || result.unwrap().is_err(),
+        "Should timeout during partition"
+    );
+
+    proxy
+        .remove_toxic("timeout")
+        .await
+        .expect("Failed to remove toxic");
+
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    assert!(result.is_ok(), "Should reach MinIO after partition heals");
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_slow_close_throttles_responses() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_slow", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let toxic = Toxic {
+        name: "slow_close".to_string(),
+        kind: ToxicKind::SlowClose { delay: 1000 },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
+
+    let start = std::time::Instant::now();
+    let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    let elapsed = start.elapsed();
+
+    assert!(
+        elapsed >= Duration::from_millis(800),
+        "Slow close should delay response, got {:?}",
+        elapsed
+    );
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}

crates/musicfs-test-utils/tests/resilience.rs

@@ -0,0 +1,838 @@
+use musicfs_cache::{Database, VirtualTree, ROOT_INODE};
+use musicfs_cas::{CasConfig, CasStore};
+use musicfs_core::supervisor::{TaskStatus, TaskSupervisor};
+use musicfs_core::{
+    AudioMeta, FileId, FileMeta, HealthStatus, OriginId, OriginType, RealPath, VirtualPath,
+};
+use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
+use musicfs_search::SearchIndex;
+use musicfs_test_utils::{FailMode, FaultyOrigin};
+use std::collections::HashMap;
+use std::io::ErrorKind;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant, UNIX_EPOCH};
+use tempfile::TempDir;
+use tokio_util::sync::CancellationToken;
+
+fn setup_test_file(dir: &TempDir, name: &str, content: &[u8]) -> PathBuf {
+    let path = dir.path().join(name);
+    std::fs::write(&path, content).unwrap();
+    path
+}
+
+async fn setup_cas(dir: &Path) -> CasStore {
+    CasStore::open(CasConfig {
+        chunks_dir: dir.join("chunks"),
+        max_size: 100 * 1024 * 1024,
+        shard_levels: 2,
+    })
+    .await
+    .unwrap()
+}
+
+fn create_faulty_origin(id: &str, dir: &TempDir, mode: FailMode) -> Arc<FaultyOrigin> {
+    let inner = Arc::new(LocalOrigin::new(
+        OriginId::from(id),
+        dir.path().to_path_buf(),
+    ));
+    Arc::new(FaultyOrigin::new(inner, mode))
+}
+
+fn make_file_meta(id: i64, path: &str, size: u64) -> FileMeta {
+    let name = Path::new(path)
+        .file_stem()
+        .and_then(|s| s.to_str())
+        .unwrap_or("unknown")
+        .to_string();
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(path),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(path),
+        },
+        size,
+        mtime: UNIX_EPOCH,
+        content_hash: None,
+        audio: Some(AudioMeta {
+            title: Some(name),
+            ..Default::default()
+        }),
+    }
+}
+
+#[tokio::test]
+async fn test_sqlite_integrity_check_detects_corruption() {
+    let dir = TempDir::new().unwrap();
+    let db_path = dir.path().join("test.db");
+
+    {
+        let db = Database::open(&db_path).unwrap();
+        db.upsert_file(
+            &OriginId::from("test"),
+            Path::new("/test.flac"),
+            &VirtualPath::new("/Test.flac"),
+            &AudioMeta::default(),
+            UNIX_EPOCH,
+            1000,
+        )
+        .unwrap();
+    }
+
+    let mut data = std::fs::read(&db_path).unwrap();
+    let mid = data.len() / 2;
+    data[mid..mid + 100].fill(0xFF);
+    std::fs::write(&db_path, &data).unwrap();
+
+    let result = Database::open_with_integrity_check(&db_path);
+    assert!(result.is_err());
+}
+
+#[tokio::test]
+async fn test_tantivy_corruption_triggers_rebuild() {
+    let dir = TempDir::new().unwrap();
+    let index_path = dir.path().join("search_idx");
+
+    {
+        let index = SearchIndex::open(&index_path).unwrap();
+        index
+            .index_file(&make_file_meta(1, "/a.flac", 1000))
+            .unwrap();
+        index.commit().unwrap();
+    }
+
+    std::fs::write(index_path.join("meta.json"), b"corrupted").unwrap();
+
+    let index = SearchIndex::open_with_recovery(&index_path).unwrap();
+    let results = index.search("a", 10).unwrap();
+    assert_eq!(results.len(), 0);
+}
+
+#[tokio::test]
+async fn test_sled_corruption_triggers_repair() {
+    let dir = TempDir::new().unwrap();
+    let chunks_dir = dir.path().join("chunks");
+    let config = CasConfig {
+        chunks_dir: chunks_dir.clone(),
+        max_size: 10_000_000,
+        shard_levels: 2,
+    };
+
+    {
+        let store = CasStore::open(config.clone()).await.unwrap();
+        store.put(b"test data").await.unwrap();
+    }
+
+    let sled_dir = chunks_dir.join("index.sled");
+    if sled_dir.exists() {
+        for entry in std::fs::read_dir(&sled_dir).unwrap() {
+            let entry = entry.unwrap();
+            if entry.metadata().unwrap().is_file() {
+                std::fs::write(entry.path(), b"corrupted").unwrap();
+            }
+        }
+    }
+
+    let result = CasStore::open(config).await;
+    assert!(result.is_ok(), "sled should recover from corruption");
+}
+
+#[tokio::test]
+async fn test_cas_put_handles_enospc() {
+    let dir = TempDir::new().unwrap();
+    let store = CasStore::open(CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size: 100,
+        shard_levels: 2,
+    })
+    .await
+    .unwrap();
+
+    let large_data = vec![0u8; 1000];
+    let result = store.put(&large_data).await;
+
+    assert!(
+        result.is_err(),
+        "Issue 2.8: CasStore should pre-check space and reject oversized write"
+    );
+}
+
+/// Demonstrates the PROBLEM with std::sync::RwLock: after a writer panic,
+/// the lock is poisoned and all subsequent access fails with PoisonError.
+/// This is why we use parking_lot::RwLock instead (see test_parking_lot_rwlock_survives_panic).
+#[test]
+fn test_poisoned_tree_lock_returns_eio_not_panic() {
+    use std::sync::{Arc, RwLock};
+    use std::thread;
+
+    let lock = Arc::new(RwLock::new(42));
+    let lock_clone = lock.clone();
+
+    let handle = thread::spawn(move || {
+        let _guard = lock_clone.write().unwrap();
+        panic!("writer panic");
+    });
+
+    let _ = handle.join();
+
+    let result = lock.read();
+    // std::sync::RwLock poisons after writer panic - this is the problem we fix with parking_lot
+    assert!(result.is_err(), "Issue 2.9: std::sync::RwLock should poison after writer panic (this demonstrates the problem)");
+}
+
+#[test]
+fn test_parking_lot_rwlock_survives_panic() {
+    use parking_lot::RwLock;
+    use std::sync::Arc;
+    use std::thread;
+
+    let tree = Arc::new(RwLock::new(VirtualTree::new()));
+    let tree_clone = tree.clone();
+
+    let handle = thread::spawn(move || {
+        let _guard = tree_clone.write();
+        panic!("writer panic");
+    });
+
+    let _ = handle.join();
+
+    assert!(
+        tree.read().get(ROOT_INODE).is_some(),
+        "parking_lot RwLock should survive writer panic"
+    );
+}
+
+#[tokio::test]
+async fn test_failover_on_primary_death() {
+    let primary_dir = TempDir::new().unwrap();
+    let backup_dir = TempDir::new().unwrap();
+    setup_test_file(&primary_dir, "test.txt", b"primary");
+    setup_test_file(&backup_dir, "test.txt", b"backup");
+
+    let primary = create_faulty_origin(
+        "primary",
+        &primary_dir,
+        FailMode::ReturnError(ErrorKind::ConnectionRefused),
+    );
+    let backup = create_faulty_origin("backup", &backup_dir, FailMode::Healthy);
+
+    let mut thresholds = HashMap::new();
+    thresholds.insert(OriginType::Local, 1);
+    let monitor =
+        Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
+    let registry = Arc::new(OriginRegistry::new(monitor.clone()));
+
+    registry.register(primary.clone(), 1);
+    registry.register(backup.clone(), 2);
+
+    monitor.check_now(&OriginId::from("primary")).await;
+    monitor.check_now(&OriginId::from("backup")).await;
+
+    assert!(registry.health().is_unhealthy(&OriginId::from("primary")));
+    assert!(registry.health().is_healthy(&OriginId::from("backup")));
+
+    let path = RealPath {
+        origin_id: OriginId::from("backup"),
+        path: PathBuf::from("/test.txt"),
+    };
+    let candidates = registry.route_all(&path);
+    assert_eq!(candidates.len(), 1);
+    assert_eq!(candidates[0].id(), &OriginId::from("backup"));
+}
+
+#[tokio::test]
+async fn test_origin_recovery_resumes_routing() {
+    let dir = TempDir::new().unwrap();
+    setup_test_file(&dir, "test.txt", b"content");
+
+    let faulty = create_faulty_origin(
+        "recovering",
+        &dir,
+        FailMode::ReturnError(ErrorKind::ConnectionRefused),
+    );
+
+    let mut thresholds = HashMap::new();
+    thresholds.insert(OriginType::Local, 1);
+    let monitor =
+        Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
+    monitor.add_origin(faulty.clone());
+
+    monitor.check_now(&OriginId::from("recovering")).await;
+    assert_eq!(
+        monitor
+            .get_state(&OriginId::from("recovering"))
+            .unwrap()
+            .status,
+        HealthStatus::Unhealthy
+    );
+
+    faulty.set_mode(FailMode::Healthy);
+    monitor.check_now(&OriginId::from("recovering")).await;
+
+    assert_eq!(
+        monitor
+            .get_state(&OriginId::from("recovering"))
+            .unwrap()
+            .status,
+        HealthStatus::Healthy
+    );
+    assert_eq!(
+        monitor
+            .get_state(&OriginId::from("recovering"))
+            .unwrap()
+            .consecutive_failures,
+        0
+    );
+}
+
+#[tokio::test]
+async fn test_local_origin_health_check_has_timeout() {
+    let dir = TempDir::new().unwrap();
+    setup_test_file(&dir, "test.txt", b"content");
+
+    let slow = create_faulty_origin("slow", &dir, FailMode::TimeoutMs(5_000));
+
+    let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
+    monitor.add_origin(slow.clone());
+
+    let start = Instant::now();
+    monitor.check_now(&OriginId::from("slow")).await;
+    let elapsed = start.elapsed();
+
+    assert!(
+        elapsed < Duration::from_secs(2),
+        "Issue 4.2.1: Health check should timeout in <2s, took {:?}",
+        elapsed
+    );
+
+    let state = monitor.get_state(&OriginId::from("slow")).unwrap();
+    assert_eq!(state.status, HealthStatus::Unhealthy);
+}
+
+#[tokio::test]
+async fn test_health_checks_run_in_parallel() {
+    let slow1_dir = TempDir::new().unwrap();
+    let slow2_dir = TempDir::new().unwrap();
+    let slow3_dir = TempDir::new().unwrap();
+
+    let slow1 = create_faulty_origin("slow1", &slow1_dir, FailMode::TimeoutMs(200));
+    let slow2 = create_faulty_origin("slow2", &slow2_dir, FailMode::TimeoutMs(200));
+    let slow3 = create_faulty_origin("slow3", &slow3_dir, FailMode::TimeoutMs(200));
+
+    let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
+    monitor.add_origin(slow1);
+    monitor.add_origin(slow2);
+    monitor.add_origin(slow3);
+
+    let start = Instant::now();
+    monitor.check_all().await;
+    let elapsed = start.elapsed();
+
+    assert!(
+        elapsed < Duration::from_millis(350),
+        "Issue 4.2.2: check_all() should run in parallel (sequential would take ~600ms), took {:?}",
+        elapsed
+    );
+}
+
+#[test]
+fn test_tantivy_survives_uncommitted_crash() {
+    let dir = TempDir::new().unwrap();
+    let index_path = dir.path().join("search_idx");
+
+    {
+        let index = SearchIndex::open(&index_path).unwrap();
+        index
+            .index_file(&make_file_meta(1, "/a.flac", 1000))
+            .unwrap();
+        index.commit().unwrap();
+        index
+            .index_file(&make_file_meta(2, "/b.flac", 1000))
+            .unwrap();
+    }
+
+    let index = SearchIndex::open(&index_path).unwrap();
+    let results = index.search("a", 10).unwrap();
+    assert_eq!(results.len(), 1);
+}
+
+#[tokio::test]
+#[cfg(feature = "resource-limits")]
+async fn test_fd_exhaustion_handling() {
+    use rlimit::{getrlimit, setrlimit, Resource};
+
+    let (orig_soft, orig_hard) = getrlimit(Resource::NOFILE).unwrap();
+
+    setrlimit(Resource::NOFILE, 64, 64).unwrap();
+
+    let dir = TempDir::new().unwrap();
+    let result = CasStore::open(CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size: 1_000_000,
+        shard_levels: 2,
+    })
+    .await;
+
+    match result {
+        Ok(_store) => {}
+        Err(e) => {
+            let msg = format!("{}", e);
+            assert!(!msg.contains("panic"), "Should not panic on fd exhaustion");
+        }
+    }
+
+    setrlimit(Resource::NOFILE, orig_soft, orig_hard).unwrap();
+}
+
+#[tokio::test]
+#[cfg(not(feature = "resource-limits"))]
+async fn test_fd_exhaustion_handling() {
+    eprintln!("Skipping test_fd_exhaustion_handling: resource-limits feature not enabled");
+}
+
+#[tokio::test]
+async fn test_corrupt_chunk_auto_refetched() {
+    use musicfs_cas::{ContentFetcher, FileReader};
+    use musicfs_origins::LocalOrigin;
+
+    let dir = TempDir::new().unwrap();
+    let origin_dir = TempDir::new().unwrap();
+    let test_content = b"original audio data for chunk test";
+    setup_test_file(&origin_dir, "test.flac", test_content);
+
+    let store = Arc::new(setup_cas(dir.path()).await);
+
+    let origin = Arc::new(LocalOrigin::new(
+        OriginId::from("local"),
+        origin_dir.path().to_path_buf(),
+    ));
+    let fetcher = Arc::new(ContentFetcher::new(store.clone()));
+    fetcher.register_origin(origin);
+
+    let file_meta = FileMeta {
+        id: FileId(1),
+        virtual_path: VirtualPath::new("/test.flac"),
+        real_path: RealPath {
+            origin_id: OriginId::from("local"),
+            path: PathBuf::from("/test.flac"),
+        },
+        size: test_content.len() as u64,
+        mtime: UNIX_EPOCH,
+        content_hash: None,
+        audio: None,
+    };
+    fetcher.register_file(file_meta);
+
+    let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
+    let chunk_hash = manifest.chunks[0].hash;
+    let hex = chunk_hash.as_hex();
+    let chunk_path = dir
+        .path()
+        .join("chunks")
+        .join(&hex[0..2])
+        .join(&hex[2..4])
+        .join(&hex);
+
+    let mut corrupted = std::fs::read(&chunk_path).unwrap();
+    corrupted[0] = corrupted[0].wrapping_add(1);
+    std::fs::write(&chunk_path, &corrupted).unwrap();
+
+    let reader = FileReader::with_fetcher(store, fetcher);
+    reader.register_manifest(manifest);
+
+    let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
+
+    assert!(
+        result.is_ok(),
+        "Issue 6.4: Corrupted chunk should be auto-refetched from origin"
+    );
+    assert_eq!(
+        &result.unwrap()[..],
+        test_content,
+        "Data should match original after re-fetch"
+    );
+}
+
+#[tokio::test]
+async fn test_missing_chunk_triggers_origin_fetch() {
+    use musicfs_cas::{ContentFetcher, FileReader};
+    use musicfs_origins::LocalOrigin;
+
+    let dir = TempDir::new().unwrap();
+    let origin_dir = TempDir::new().unwrap();
+    let test_content = b"test data for missing chunk";
+    setup_test_file(&origin_dir, "test.flac", test_content);
+
+    let store = Arc::new(setup_cas(dir.path()).await);
+
+    let origin = Arc::new(LocalOrigin::new(
+        OriginId::from("local"),
+        origin_dir.path().to_path_buf(),
+    ));
+    let fetcher = Arc::new(ContentFetcher::new(store.clone()));
+    fetcher.register_origin(origin);
+
+    let file_meta = FileMeta {
+        id: FileId(1),
+        virtual_path: VirtualPath::new("/test.flac"),
+        real_path: RealPath {
+            origin_id: OriginId::from("local"),
+            path: PathBuf::from("/test.flac"),
+        },
+        size: test_content.len() as u64,
+        mtime: UNIX_EPOCH,
+        content_hash: None,
+        audio: None,
+    };
+    fetcher.register_file(file_meta);
+
+    let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
+    let chunk_hash = manifest.chunks[0].hash;
+    let hex = chunk_hash.as_hex();
+    let chunk_path = dir
+        .path()
+        .join("chunks")
+        .join(&hex[0..2])
+        .join(&hex[2..4])
+        .join(&hex);
+
+    std::fs::remove_file(&chunk_path).unwrap();
+
+    let reader = FileReader::with_fetcher(store, fetcher);
+    reader.register_manifest(manifest);
+
+    let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
+
+    assert!(
+        result.is_ok(),
+        "Issue 6.4: Missing chunk should be re-fetched from origin"
+    );
+    assert_eq!(
+        &result.unwrap()[..],
+        test_content,
+        "Data should match original after re-fetch"
+    );
+}
+
+#[tokio::test]
+async fn test_passthrough_mode_when_cache_disk_dead() {
+    use musicfs_cas::ContentFetcher;
+    use musicfs_origins::LocalOrigin;
+
+    let dir = TempDir::new().unwrap();
+    let origin_dir = TempDir::new().unwrap();
+    let test_content = b"passthrough test data";
+    setup_test_file(&origin_dir, "test.flac", test_content);
+
+    let store = Arc::new(
+        CasStore::open(CasConfig {
+            chunks_dir: dir.path().join("chunks"),
+            max_size: 10,
+            shard_levels: 2,
+        })
+        .await
+        .unwrap(),
+    );
+
+    let origin = Arc::new(LocalOrigin::new(
+        OriginId::from("local"),
+        origin_dir.path().to_path_buf(),
+    ));
+    let fetcher = Arc::new(ContentFetcher::new(store.clone()));
+    fetcher.register_origin(origin);
+
+    let file_meta = FileMeta {
+        id: FileId(1),
+        virtual_path: VirtualPath::new("/test.flac"),
+        real_path: RealPath {
+            origin_id: OriginId::from("local"),
+            path: PathBuf::from("/test.flac"),
+        },
+        size: test_content.len() as u64,
+        mtime: UNIX_EPOCH,
+        content_hash: None,
+        audio: None,
+    };
+    fetcher.register_file(file_meta);
+
+    let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
+
+    assert!(
+        !manifest.chunks.is_empty(),
+        "Issue 6.6: Fetch should complete even when CAS write fails (passthrough mode)"
+    );
+}
+
+#[tokio::test]
+async fn test_cas_size_tracking_is_correct() {
+    let dir = TempDir::new().unwrap();
+    let config = CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size: 10_000_000,
+        shard_levels: 2,
+    };
+    let store = CasStore::open(config).await.unwrap();
+
+    let data = vec![0u8; 1000];
+    store.put(&data).await.unwrap();
+
+    assert!(
+        store.current_size() >= 1000,
+        "Issue C6: current_size should track chunk data (recursive), got {}",
+        store.current_size()
+    );
+}
+
+#[test]
+fn test_pid_file_prevents_concurrent_mount() {
+    use std::fs::File;
+    use std::os::unix::io::AsRawFd;
+
+    let dir = TempDir::new().unwrap();
+    let lock_path = dir.path().join("musicfs.lock");
+
+    fn try_lock(path: &Path) -> Result<File, std::io::Error> {
+        let file = File::create(path)?;
+        let fd = file.as_raw_fd();
+        let ret = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
+        if ret != 0 {
+            return Err(std::io::Error::last_os_error());
+        }
+        Ok(file)
+    }
+
+    let lock1 = try_lock(&lock_path);
+    assert!(lock1.is_ok(), "Issue C9: First lock should succeed");
+
+    let lock2 = try_lock(&lock_path);
+    assert!(
+        lock2.is_err(),
+        "Issue C9: Second lock should fail (already held)"
+    );
+
+    drop(lock1);
+
+    let lock3 = try_lock(&lock_path);
+    assert!(
+        lock3.is_ok(),
+        "Issue C9: Third lock should succeed after first released"
+    );
+}
+
+#[test]
+fn test_panic_hook_logs_to_tracing() {
+    use std::panic;
+
+    musicfs_core::install_panic_hook();
+
+    let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
+        panic!("test panic message");
+    }));
+
+    assert!(result.is_err(), "Panic should have been caught");
+}
+
+#[test]
+fn test_stale_mount_check_function_exists() {
+    let path = std::path::Path::new("/nonexistent/musicfs/mount");
+    assert!(
+        !path.exists(),
+        "Test path should not exist for this test to be meaningful"
+    );
+}
+
+#[test]
+fn test_systemd_service_has_execstoppost() {
+    let service_path = std::path::Path::new("../../dist/musicfs.service");
+    if !service_path.exists() {
+        panic!(
+            "Issue 3.7: dist/musicfs.service does not exist at {:?}",
+            service_path
+        );
+    }
+
+    let content = std::fs::read_to_string(service_path).unwrap();
+    assert!(
+        content.contains("ExecStopPost") && content.contains("fusermount"),
+        "Issue 3.7: Service file should have ExecStopPost with fusermount for cleanup"
+    );
+}
+
+#[test]
+fn test_sd_notify_ready_sent() {
+    use std::os::unix::net::UnixDatagram;
+    use tempfile::TempDir;
+
+    let dir = TempDir::new().unwrap();
+    let socket_path = dir.path().join("notify.sock");
+    let socket = UnixDatagram::bind(&socket_path).unwrap();
+    socket
+        .set_read_timeout(Some(Duration::from_secs(1)))
+        .unwrap();
+
+    std::env::set_var("NOTIFY_SOCKET", &socket_path);
+
+    let result = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]);
+    assert!(
+        result.is_ok(),
+        "sd_notify should succeed when NOTIFY_SOCKET is set"
+    );
+
+    let mut buf = [0u8; 256];
+    let len = socket.recv(&mut buf).unwrap();
+    let msg = std::str::from_utf8(&buf[..len]).unwrap();
+
+    assert!(
+        msg.contains("READY=1"),
+        "sd_notify should send READY=1, got: {}",
+        msg
+    );
+
+    std::env::remove_var("NOTIFY_SOCKET");
+}
+
+#[tokio::test]
+async fn test_shutdown_cancels_background_tasks() {
+    let token = CancellationToken::new();
+    let stopped = Arc::new(AtomicBool::new(false));
+    let stopped_clone = stopped.clone();
+    let token_clone = token.clone();
+
+    tokio::spawn(async move {
+        token_clone.cancelled().await;
+        stopped_clone.store(true, Ordering::SeqCst);
+    });
+
+    assert!(!stopped.load(Ordering::SeqCst));
+    token.cancel();
+    tokio::time::sleep(Duration::from_millis(50)).await;
+    assert!(stopped.load(Ordering::SeqCst));
+}
+
+#[tokio::test]
+async fn test_shutdown_flushes_tantivy() {
+    let dir = TempDir::new().unwrap();
+    let idx_path = dir.path().join("idx");
+
+    {
+        let index = SearchIndex::open(&idx_path).unwrap();
+        index
+            .index_file(&make_file_meta(1, "/a.flac", 1000))
+            .unwrap();
+        index.commit().unwrap();
+    }
+
+    let index2 = SearchIndex::open(&idx_path).unwrap();
+    assert_eq!(index2.search("a", 10).unwrap().len(), 1);
+}
+
+#[tokio::test]
+async fn test_supervisor_detects_task_completion() {
+    let supervisor = TaskSupervisor::new();
+    supervisor.spawn_supervised("fast", async {});
+    tokio::time::sleep(Duration::from_millis(50)).await;
+}
+
+#[tokio::test]
+async fn test_supervisor_detects_panic() {
+    let supervisor = TaskSupervisor::new();
+    supervisor.spawn_supervised("panicker", async {
+        panic!("boom");
+    });
+    tokio::time::sleep(Duration::from_millis(50)).await;
+    assert!(matches!(
+        supervisor.task_status("panicker"),
+        TaskStatus::Failed { .. }
+    ));
+}
+
+#[tokio::test]
+async fn test_supervisor_restarts_critical_task() {
+    let count = Arc::new(AtomicU32::new(0));
+    let c = count.clone();
+
+    let supervisor = TaskSupervisor::new();
+    supervisor.spawn_critical("restartable", move || {
+        let c = c.clone();
+        async move {
+            let n = c.fetch_add(1, Ordering::SeqCst);
+            if n == 0 {
+                panic!("first run fails");
+            }
+            loop {
+                tokio::time::sleep(Duration::from_secs(60)).await;
+            }
+        }
+    });
+
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    assert_eq!(count.load(Ordering::SeqCst), 2);
+    assert!(matches!(
+        supervisor.task_status("restartable"),
+        TaskStatus::Running
+    ));
+}
+
+#[tokio::test]
+async fn test_sigterm_triggers_shutdown() {
+    use std::process::{Command, Stdio};
+    use std::time::Duration;
+    use tokio::time::timeout;
+
+    let musicfs_bin = std::env::var("CARGO_BIN_EXE_musicfs").ok();
+    if musicfs_bin.is_none() {
+        eprintln!(
+            "Skipping test_sigterm_triggers_shutdown: musicfs binary not available in test context"
+        );
+        return;
+    }
+
+    let bin_path = musicfs_bin.unwrap();
+    let temp_dir = tempfile::TempDir::new().unwrap();
+    let mountpoint = temp_dir.path().join("mount");
+    let origin = temp_dir.path().join("origin");
+    std::fs::create_dir_all(&mountpoint).unwrap();
+    std::fs::create_dir_all(&origin).unwrap();
+
+    let mut child = Command::new(&bin_path)
+        .args([
+            "mount",
+            "--origin",
+            origin.to_str().unwrap(),
+            mountpoint.to_str().unwrap(),
+        ])
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .spawn();
+
+    if child.is_err() {
+        eprintln!("Skipping test_sigterm_triggers_shutdown: failed to spawn musicfs");
+        return;
+    }
+
+    let mut child = child.unwrap();
+    tokio::time::sleep(Duration::from_millis(500)).await;
+
+    unsafe {
+        libc::kill(child.id() as i32, libc::SIGTERM);
+    }
+
+    let exit_result = timeout(Duration::from_secs(10), async {
+        loop {
+            match child.try_wait() {
+                Ok(Some(status)) => return status,
+                Ok(None) => tokio::time::sleep(Duration::from_millis(100)).await,
+                Err(_) => break,
+            }
+        }
+        child.wait().unwrap()
+    })
+    .await;
+
+    assert!(
+        exit_result.is_ok(),
+        "Issue 2.1: Process should exit within 10s after SIGTERM"
+    );
+}