Move the files around

crates/musicfs-test-utils/src/assertions.rs

@@ -0,0 +1,204 @@
+use musicfs_cas::CasError;
+use musicfs_core::Error;
+use std::time::{Duration, Instant};
+
+pub fn assert_error_contains<T, E: std::fmt::Debug>(result: Result<T, E>, expected_text: &str) {
+    match result {
+        Ok(_) => panic!("Expected error containing '{}', but got Ok", expected_text),
+        Err(e) => {
+            let error_msg = format!("{:?}", e);
+            assert!(
+                error_msg.contains(expected_text),
+                "Expected error containing '{}', but got: {}",
+                expected_text,
+                error_msg
+            );
+        }
+    }
+}
+
+pub fn assert_io_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Io(_)) => (),
+        Err(e) => panic!("Expected Io error, got: {:?}", e),
+        Ok(_) => panic!("Expected Io error, got Ok"),
+    }
+}
+
+pub fn assert_cas_io_error<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::Io(_)) => (),
+        Err(e) => panic!("Expected CasError::Io, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::Io, got Ok"),
+    }
+}
+
+pub fn assert_cas_not_found<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::NotFound(_)) => (),
+        Err(e) => panic!("Expected CasError::NotFound, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::NotFound, got Ok"),
+    }
+}
+
+pub fn assert_cas_integrity_error<T>(result: Result<T, CasError>) {
+    match result {
+        Err(CasError::IntegrityError { .. }) => (),
+        Err(e) => panic!("Expected CasError::IntegrityError, got: {:?}", e),
+        Ok(_) => panic!("Expected CasError::IntegrityError, got Ok"),
+    }
+}
+
+pub fn assert_file_not_found<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::FileNotFound(_)) => (),
+        Err(e) => panic!("Expected FileNotFound error, got: {:?}", e),
+        Ok(_) => panic!("Expected FileNotFound error, got Ok"),
+    }
+}
+
+pub fn assert_origin_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Origin(_)) => (),
+        Err(e) => panic!("Expected Origin error, got: {:?}", e),
+        Ok(_) => panic!("Expected Origin error, got Ok"),
+    }
+}
+
+pub fn assert_timeout_error<T>(result: Result<T, Error>) {
+    match result {
+        Err(Error::Timeout(_)) => (),
+        Err(e) => panic!("Expected Timeout error, got: {:?}", e),
+        Ok(_) => panic!("Expected Timeout error, got Ok"),
+    }
+}
+
+pub struct TimedAssertion {
+    start: Instant,
+    min_duration: Option<Duration>,
+    max_duration: Option<Duration>,
+}
+
+impl TimedAssertion {
+    pub fn new() -> Self {
+        Self {
+            start: Instant::now(),
+            min_duration: None,
+            max_duration: None,
+        }
+    }
+
+    pub fn expect_at_least(mut self, duration: Duration) -> Self {
+        self.min_duration = Some(duration);
+        self
+    }
+
+    pub fn expect_at_most(mut self, duration: Duration) -> Self {
+        self.max_duration = Some(duration);
+        self
+    }
+
+    pub fn assert_elapsed(self) {
+        let elapsed = self.start.elapsed();
+
+        if let Some(min) = self.min_duration {
+            assert!(
+                elapsed >= min,
+                "Expected at least {:?}, but only {:?} elapsed",
+                min,
+                elapsed
+            );
+        }
+
+        if let Some(max) = self.max_duration {
+            assert!(
+                elapsed <= max,
+                "Expected at most {:?}, but {:?} elapsed",
+                max,
+                elapsed
+            );
+        }
+    }
+}
+
+impl Default for TimedAssertion {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+pub async fn assert_completes_within<F, T>(future: F, timeout: Duration) -> T
+where
+    F: std::future::Future<Output = T>,
+{
+    tokio::time::timeout(timeout, future)
+        .await
+        .expect(&format!("Operation did not complete within {:?}", timeout))
+}
+
+pub async fn assert_times_out<F, T>(future: F, timeout: Duration)
+where
+    F: std::future::Future<Output = T>,
+{
+    match tokio::time::timeout(timeout, future).await {
+        Ok(_) => panic!("Expected operation to time out, but it completed"),
+        Err(_) => (),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_assert_error_contains() {
+        let result: Result<(), Error> = Err(Error::Origin("connection refused".into()));
+        assert_error_contains(result, "connection");
+    }
+
+    #[test]
+    #[should_panic(expected = "Expected error containing")]
+    fn test_assert_error_contains_failure() {
+        let result: Result<(), Error> = Err(Error::Origin("something else".into()));
+        assert_error_contains(result, "connection");
+    }
+
+    #[test]
+    fn test_assert_io_error() {
+        let result: Result<(), Error> = Err(Error::Io(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "test",
+        )));
+        assert_io_error(result);
+    }
+
+    #[test]
+    fn test_timed_assertion_at_least() {
+        let timer = TimedAssertion::new().expect_at_least(Duration::from_millis(10));
+        std::thread::sleep(Duration::from_millis(15));
+        timer.assert_elapsed();
+    }
+
+    #[test]
+    fn test_timed_assertion_at_most() {
+        let timer = TimedAssertion::new().expect_at_most(Duration::from_millis(100));
+        timer.assert_elapsed();
+    }
+
+    #[tokio::test]
+    async fn test_assert_completes_within() {
+        let result = assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
+        assert_eq!(result, 42);
+    }
+
+    #[tokio::test]
+    async fn test_assert_times_out() {
+        assert_times_out(
+            async {
+                tokio::time::sleep(Duration::from_secs(10)).await;
+            },
+            Duration::from_millis(10),
+        )
+        .await;
+    }
+}

crates/musicfs-test-utils/src/faulty_cas.rs

@@ -0,0 +1,250 @@
+use bytes::Bytes;
+use musicfs_cas::{CasConfig, CasError, CasStore, DedupStats};
+use musicfs_core::ChunkHash;
+use std::io::{self, ErrorKind};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
+
+pub struct FaultyCasStore {
+    inner: Arc<CasStore>,
+    inject_enospc: AtomicBool,
+    inject_eio_on_read: AtomicBool,
+    inject_eio_on_write: AtomicBool,
+    inject_corruption: AtomicBool,
+    fail_after_n_puts: AtomicUsize,
+    put_count: AtomicUsize,
+}
+
+impl FaultyCasStore {
+    pub fn new(inner: Arc<CasStore>) -> Self {
+        Self {
+            inner,
+            inject_enospc: AtomicBool::new(false),
+            inject_eio_on_read: AtomicBool::new(false),
+            inject_eio_on_write: AtomicBool::new(false),
+            inject_corruption: AtomicBool::new(false),
+            fail_after_n_puts: AtomicUsize::new(usize::MAX),
+            put_count: AtomicUsize::new(0),
+        }
+    }
+
+    pub async fn open(config: CasConfig) -> Result<Self, CasError> {
+        let store = CasStore::open(config).await?;
+        Ok(Self::new(Arc::new(store)))
+    }
+
+    pub fn set_inject_enospc(&self, enabled: bool) {
+        self.inject_enospc.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_eio_on_read(&self, enabled: bool) {
+        self.inject_eio_on_read.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_eio_on_write(&self, enabled: bool) {
+        self.inject_eio_on_write.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_inject_corruption(&self, enabled: bool) {
+        self.inject_corruption.store(enabled, Ordering::SeqCst);
+    }
+
+    pub fn set_fail_after_n_puts(&self, n: usize) {
+        self.fail_after_n_puts.store(n, Ordering::SeqCst);
+        self.put_count.store(0, Ordering::SeqCst);
+    }
+
+    pub fn reset_faults(&self) {
+        self.inject_enospc.store(false, Ordering::SeqCst);
+        self.inject_eio_on_read.store(false, Ordering::SeqCst);
+        self.inject_eio_on_write.store(false, Ordering::SeqCst);
+        self.inject_corruption.store(false, Ordering::SeqCst);
+        self.fail_after_n_puts.store(usize::MAX, Ordering::SeqCst);
+        self.put_count.store(0, Ordering::SeqCst);
+    }
+
+    pub fn put_count(&self) -> usize {
+        self.put_count.load(Ordering::SeqCst)
+    }
+
+    pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
+        let count = self.put_count.fetch_add(1, Ordering::SeqCst);
+
+        if self.inject_enospc.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "No space left on device (ENOSPC injected)",
+            )));
+        }
+
+        if self.inject_eio_on_write.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+
+        let threshold = self.fail_after_n_puts.load(Ordering::SeqCst);
+        if count >= threshold {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Injected failure after N puts",
+            )));
+        }
+
+        self.inner.put(data).await
+    }
+
+    pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
+        if self.inject_eio_on_read.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+
+        let data = self.inner.get(hash).await?;
+
+        if self.inject_corruption.load(Ordering::SeqCst) {
+            let mut corrupted = data.to_vec();
+            if !corrupted.is_empty() {
+                corrupted[0] = corrupted[0].wrapping_add(1);
+            }
+            return Err(CasError::IntegrityError {
+                expected: hash.as_hex(),
+                actual: ChunkHash::from_bytes(&corrupted).as_hex(),
+            });
+        }
+
+        Ok(data)
+    }
+
+    pub fn exists(&self, hash: &ChunkHash) -> bool {
+        self.inner.exists(hash)
+    }
+
+    pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
+        if self.inject_eio_on_write.load(Ordering::SeqCst) {
+            return Err(CasError::Io(io::Error::new(
+                ErrorKind::Other,
+                "Input/output error (EIO injected)",
+            )));
+        }
+        self.inner.delete(hash).await
+    }
+
+    pub fn current_size(&self) -> u64 {
+        self.inner.current_size()
+    }
+
+    pub fn max_size(&self) -> u64 {
+        self.inner.max_size()
+    }
+
+    pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
+        self.inner.list_chunks()
+    }
+
+    pub fn dedup_stats(&self) -> DedupStats {
+        self.inner.dedup_stats()
+    }
+
+    pub fn inner(&self) -> &Arc<CasStore> {
+        &self.inner
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    async fn test_store() -> (FaultyCasStore, TempDir) {
+        let dir = TempDir::new().unwrap();
+        let config = CasConfig {
+            chunks_dir: dir.path().join("chunks"),
+            max_size: 1024 * 1024,
+            shard_levels: 2,
+        };
+        let store = FaultyCasStore::open(config).await.unwrap();
+        (store, dir)
+    }
+
+    #[tokio::test]
+    async fn test_healthy_passthrough() {
+        let (store, _dir) = test_store().await;
+
+        let data = b"test data";
+        let hash = store.put(data).await.unwrap();
+        let retrieved = store.get(&hash).await.unwrap();
+        assert_eq!(&retrieved[..], data);
+    }
+
+    #[tokio::test]
+    async fn test_inject_enospc() {
+        let (store, _dir) = test_store().await;
+
+        store.set_inject_enospc(true);
+        let result = store.put(b"test").await;
+        assert!(result.is_err());
+
+        let err = result.unwrap_err();
+        assert!(matches!(err, CasError::Io(_)));
+
+        store.set_inject_enospc(false);
+        assert!(store.put(b"test").await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_inject_eio_on_read() {
+        let (store, _dir) = test_store().await;
+
+        let hash = store.put(b"test").await.unwrap();
+
+        store.set_inject_eio_on_read(true);
+        let result = store.get(&hash).await;
+        assert!(result.is_err());
+
+        store.set_inject_eio_on_read(false);
+        assert!(store.get(&hash).await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_inject_corruption() {
+        let (store, _dir) = test_store().await;
+
+        let hash = store.put(b"test data").await.unwrap();
+
+        store.set_inject_corruption(true);
+        let result = store.get(&hash).await;
+        assert!(matches!(result, Err(CasError::IntegrityError { .. })));
+    }
+
+    #[tokio::test]
+    async fn test_fail_after_n_puts() {
+        let (store, _dir) = test_store().await;
+
+        store.set_fail_after_n_puts(2);
+
+        assert!(store.put(b"data1").await.is_ok());
+        assert!(store.put(b"data2").await.is_ok());
+        assert!(store.put(b"data3").await.is_err());
+        assert!(store.put(b"data4").await.is_err());
+        assert_eq!(store.put_count(), 4);
+    }
+
+    #[tokio::test]
+    async fn test_reset_faults() {
+        let (store, _dir) = test_store().await;
+
+        store.set_inject_enospc(true);
+        store.set_inject_eio_on_read(true);
+        store.set_fail_after_n_puts(1);
+
+        store.reset_faults();
+
+        assert!(store.put(b"test").await.is_ok());
+        let hash = store.put(b"test2").await.unwrap();
+        assert!(store.get(&hash).await.is_ok());
+    }
+}

crates/musicfs-test-utils/src/faulty_origin.rs

@@ -0,0 +1,328 @@
+use async_trait::async_trait;
+use musicfs_core::{DirEntry, Error, FileStat, HealthStatus, OriginId, OriginType, Result};
+use musicfs_origins::{Origin, WatchCallback, WatchHandle};
+use parking_lot::RwLock;
+use std::io::{self, ErrorKind};
+use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::io::AsyncRead;
+
+#[derive(Debug, Clone)]
+pub enum FailMode {
+    Healthy,
+    FailEveryNth(usize),
+    FailAfterN(usize),
+    TimeoutMs(u64),
+    PartialRead { max_bytes: usize },
+    ReturnError(ErrorKind),
+}
+
+impl Default for FailMode {
+    fn default() -> Self {
+        FailMode::Healthy
+    }
+}
+
+pub struct FaultyOrigin {
+    inner: Arc<dyn Origin>,
+    fail_mode: Arc<RwLock<FailMode>>,
+    call_count: AtomicUsize,
+}
+
+impl FaultyOrigin {
+    pub fn new(inner: Arc<dyn Origin>, mode: FailMode) -> Self {
+        Self {
+            inner,
+            fail_mode: Arc::new(RwLock::new(mode)),
+            call_count: AtomicUsize::new(0),
+        }
+    }
+
+    pub fn wrap(inner: impl Origin + 'static) -> Self {
+        Self::new(Arc::new(inner), FailMode::Healthy)
+    }
+
+    pub fn set_mode(&self, mode: FailMode) {
+        *self.fail_mode.write() = mode;
+    }
+
+    pub fn call_count(&self) -> usize {
+        self.call_count.load(Ordering::SeqCst)
+    }
+
+    pub fn reset_count(&self) {
+        self.call_count.store(0, Ordering::SeqCst);
+    }
+
+    fn increment_and_check(&self) -> Option<Error> {
+        let count = self.call_count.fetch_add(1, Ordering::SeqCst) + 1;
+        let mode = self.fail_mode.read();
+
+        match *mode {
+            FailMode::Healthy => None,
+            FailMode::FailEveryNth(n) if n > 0 && count % n == 0 => {
+                Some(Error::Origin("Injected failure (every Nth)".into()))
+            }
+            FailMode::FailEveryNth(_) => None,
+            FailMode::FailAfterN(n) if count > n => {
+                Some(Error::Origin("Injected failure (after N)".into()))
+            }
+            FailMode::FailAfterN(_) => None,
+            FailMode::TimeoutMs(_) => None,
+            FailMode::PartialRead { .. } => None,
+            FailMode::ReturnError(kind) => {
+                Some(Error::Io(io::Error::new(kind, "Injected I/O error")))
+            }
+        }
+    }
+
+    async fn maybe_timeout(&self) -> Option<Error> {
+        let mode = self.fail_mode.read().clone();
+        if let FailMode::TimeoutMs(ms) = mode {
+            tokio::time::sleep(Duration::from_millis(ms)).await;
+            Some(Error::Timeout("Injected timeout".into()))
+        } else {
+            None
+        }
+    }
+
+    fn truncate_if_partial(&self, mut data: Vec<u8>) -> Vec<u8> {
+        let mode = self.fail_mode.read();
+        if let FailMode::PartialRead { max_bytes } = *mode {
+            data.truncate(max_bytes);
+        }
+        data
+    }
+}
+
+#[async_trait]
+impl Origin for FaultyOrigin {
+    fn id(&self) -> &OriginId {
+        self.inner.id()
+    }
+
+    fn origin_type(&self) -> OriginType {
+        self.inner.origin_type()
+    }
+
+    fn display_name(&self) -> &str {
+        self.inner.display_name()
+    }
+
+    async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.readdir(path).await
+    }
+
+    async fn stat(&self, path: &Path) -> Result<FileStat> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.stat(path).await
+    }
+
+    async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        let data = self.inner.read(path, offset, size).await?;
+        Ok(self.truncate_if_partial(data))
+    }
+
+    async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        let data = self.inner.read_full(path).await?;
+        Ok(self.truncate_if_partial(data))
+    }
+
+    async fn exists(&self, path: &Path) -> Result<bool> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.exists(path).await
+    }
+
+    async fn health(&self) -> HealthStatus {
+        let mode = self.fail_mode.read().clone();
+        match mode {
+            FailMode::Healthy => self.inner.health().await,
+            FailMode::ReturnError(_) => HealthStatus::Unhealthy,
+            FailMode::TimeoutMs(ms) => {
+                tokio::time::sleep(Duration::from_millis(ms)).await;
+                HealthStatus::Unhealthy
+            }
+            FailMode::FailAfterN(n) if self.call_count.load(Ordering::SeqCst) >= n => {
+                HealthStatus::Unhealthy
+            }
+            _ => self.inner.health().await,
+        }
+    }
+
+    async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
+        if let Some(err) = self.increment_and_check() {
+            return Err(err);
+        }
+        if let Some(err) = self.maybe_timeout().await {
+            return Err(err);
+        }
+        self.inner.open_read(path).await
+    }
+
+    async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
+        self.inner.watch(path, callback).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::SystemTime;
+
+    struct MockOrigin {
+        id: OriginId,
+    }
+
+    impl MockOrigin {
+        fn new(id: &str) -> Self {
+            Self {
+                id: OriginId::from(id),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl Origin for MockOrigin {
+        fn id(&self) -> &OriginId {
+            &self.id
+        }
+
+        fn origin_type(&self) -> OriginType {
+            OriginType::Local
+        }
+
+        fn display_name(&self) -> &str {
+            "mock"
+        }
+
+        async fn readdir(&self, _path: &Path) -> Result<Vec<DirEntry>> {
+            Ok(vec![])
+        }
+
+        async fn stat(&self, _path: &Path) -> Result<FileStat> {
+            Ok(FileStat {
+                size: 1000,
+                mtime: SystemTime::now(),
+                is_dir: false,
+            })
+        }
+
+        async fn read(&self, _path: &Path, _offset: u64, size: u32) -> Result<Vec<u8>> {
+            Ok(vec![0u8; size as usize])
+        }
+
+        async fn read_full(&self, _path: &Path) -> Result<Vec<u8>> {
+            Ok(vec![0u8; 100])
+        }
+
+        async fn exists(&self, _path: &Path) -> Result<bool> {
+            Ok(true)
+        }
+
+        async fn health(&self) -> HealthStatus {
+            HealthStatus::Healthy
+        }
+
+        async fn open_read(&self, _path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
+            Err(Error::Origin("Not implemented".into()))
+        }
+
+        async fn watch(&self, _path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
+            Err(Error::Origin("Not implemented".into()))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_healthy_passthrough() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
+
+        let result = faulty.stat(Path::new("/test")).await;
+        assert!(result.is_ok());
+        assert_eq!(faulty.call_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_fail_every_nth() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::FailEveryNth(2));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert_eq!(faulty.call_count(), 4);
+    }
+
+    #[tokio::test]
+    async fn test_fail_after_n() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::FailAfterN(2));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_partial_read() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::PartialRead { max_bytes: 10 });
+
+        let data = faulty.read(Path::new("/test"), 0, 100).await.unwrap();
+        assert_eq!(data.len(), 10);
+    }
+
+    #[tokio::test]
+    async fn test_mode_change_mid_test() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::ReturnError(ErrorKind::ConnectionRefused));
+
+        assert!(faulty.stat(Path::new("/test")).await.is_err());
+
+        faulty.set_mode(FailMode::Healthy);
+        assert!(faulty.stat(Path::new("/test")).await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_health_reflects_mode() {
+        let inner = Arc::new(MockOrigin::new("test"));
+        let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
+
+        assert_eq!(faulty.health().await, HealthStatus::Healthy);
+
+        faulty.set_mode(FailMode::ReturnError(ErrorKind::ConnectionRefused));
+        assert_eq!(faulty.health().await, HealthStatus::Unhealthy);
+    }
+}

crates/musicfs-test-utils/src/fixtures.rs

@@ -0,0 +1,253 @@
+use musicfs_cache::TreeBuilder;
+use musicfs_cas::{CasConfig, CasStore};
+use musicfs_core::{AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, RwLock};
+use std::time::SystemTime;
+use tempfile::TempDir;
+
+pub fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: None,
+    }
+}
+
+pub fn make_file_meta_with_origin(id: i64, vpath: &str, size: u64, origin_id: &str) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from(origin_id),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: None,
+    }
+}
+
+pub fn make_audio_meta(artist: &str, album: &str, title: &str) -> AudioMeta {
+    AudioMeta {
+        title: Some(title.to_string()),
+        artist: Some(artist.to_string()),
+        album: Some(album.to_string()),
+        album_artist: None,
+        genre: None,
+        year: None,
+        track: None,
+        disc: None,
+        duration_ms: Some(180_000),
+        bitrate: Some(320),
+        sample_rate: Some(44100),
+        format: AudioFormat::Flac,
+    }
+}
+
+pub fn make_audio_file(
+    id: i64,
+    vpath: &str,
+    size: u64,
+    artist: &str,
+    album: &str,
+    title: &str,
+) -> FileMeta {
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: Some(make_audio_meta(artist, album, title)),
+    }
+}
+
+pub fn make_audio_file_full(
+    id: i64,
+    vpath: &str,
+    size: u64,
+    artist: &str,
+    album: &str,
+    title: &str,
+    track: u32,
+    year: u32,
+) -> FileMeta {
+    let mut audio = make_audio_meta(artist, album, title);
+    audio.track = Some(track);
+    audio.year = Some(year);
+
+    FileMeta {
+        id: FileId(id),
+        virtual_path: VirtualPath::new(vpath),
+        real_path: RealPath {
+            origin_id: OriginId::from("test"),
+            path: PathBuf::from(vpath),
+        },
+        size,
+        mtime: SystemTime::now(),
+        content_hash: None,
+        audio: Some(audio),
+    }
+}
+
+pub struct TestCasStore {
+    pub store: Arc<CasStore>,
+    pub dir: TempDir,
+}
+
+pub async fn setup_test_cas() -> TestCasStore {
+    let dir = TempDir::new().expect("Failed to create temp dir for CAS");
+    let config = CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size: 100 * 1024 * 1024,
+        shard_levels: 2,
+    };
+    let store = CasStore::open(config)
+        .await
+        .expect("Failed to open CAS store");
+    TestCasStore {
+        store: Arc::new(store),
+        dir,
+    }
+}
+
+pub async fn setup_test_cas_with_size(max_size: u64) -> TestCasStore {
+    let dir = TempDir::new().expect("Failed to create temp dir for CAS");
+    let config = CasConfig {
+        chunks_dir: dir.path().join("chunks"),
+        max_size,
+        shard_levels: 2,
+    };
+    let store = CasStore::open(config)
+        .await
+        .expect("Failed to open CAS store");
+    TestCasStore {
+        store: Arc::new(store),
+        dir,
+    }
+}
+
+pub fn setup_test_tree(files: &[FileMeta]) -> Arc<RwLock<musicfs_cache::VirtualTree>> {
+    let mut builder = TreeBuilder::new();
+    for file in files {
+        builder.add_file(file);
+    }
+    Arc::new(RwLock::new(builder.build()))
+}
+
+pub fn create_test_file(dir: &Path, relative_path: &str, content: &[u8]) -> PathBuf {
+    let full_path = dir.join(relative_path);
+    if let Some(parent) = full_path.parent() {
+        std::fs::create_dir_all(parent).expect("Failed to create parent directories");
+    }
+    std::fs::write(&full_path, content).expect("Failed to write test file");
+    full_path
+}
+
+pub fn create_test_dir_structure(base: &Path, structure: &[&str]) {
+    for path in structure {
+        let full_path = base.join(path);
+        if path.ends_with('/') {
+            std::fs::create_dir_all(&full_path).expect("Failed to create directory");
+        } else {
+            if let Some(parent) = full_path.parent() {
+                std::fs::create_dir_all(parent).expect("Failed to create parent");
+            }
+            std::fs::write(&full_path, format!("content of {}", path))
+                .expect("Failed to write file");
+        }
+    }
+}
+
+pub struct TestOriginDir {
+    pub dir: TempDir,
+}
+
+impl TestOriginDir {
+    pub fn new() -> Self {
+        Self {
+            dir: TempDir::new().expect("Failed to create origin temp dir"),
+        }
+    }
+
+    pub fn add_file(&self, path: &str, content: &[u8]) -> PathBuf {
+        create_test_file(self.dir.path(), path, content)
+    }
+
+    pub fn add_audio_file(&self, path: &str) -> PathBuf {
+        let fake_audio = b"FAKE_FLAC_HEADER_FOR_TESTING_ONLY";
+        self.add_file(path, fake_audio)
+    }
+
+    pub fn path(&self) -> &Path {
+        self.dir.path()
+    }
+}
+
+impl Default for TestOriginDir {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_make_file_meta() {
+        let meta = make_file_meta(1, "/Artist/Album/Track.flac", 1000);
+        assert_eq!(meta.id.0, 1);
+        assert_eq!(meta.virtual_path.as_str(), "/Artist/Album/Track.flac");
+        assert_eq!(meta.size, 1000);
+        assert!(meta.audio.is_none());
+    }
+
+    #[test]
+    fn test_make_audio_file() {
+        let meta = make_audio_file(1, "/path.flac", 5000, "Artist", "Album", "Title");
+        assert!(meta.audio.is_some());
+        let audio = meta.audio.unwrap();
+        assert_eq!(audio.artist, Some("Artist".to_string()));
+        assert_eq!(audio.album, Some("Album".to_string()));
+        assert_eq!(audio.title, Some("Title".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_setup_test_cas() {
+        let test_cas = setup_test_cas().await;
+        let hash = test_cas.store.put(b"test data").await.unwrap();
+        assert!(test_cas.store.exists(&hash));
+    }
+
+    #[test]
+    fn test_setup_test_tree() {
+        let files = vec![
+            make_file_meta(1, "/A/B/1.flac", 100),
+            make_file_meta(2, "/A/B/2.flac", 200),
+        ];
+        let tree = setup_test_tree(&files);
+        let guard = tree.read().unwrap();
+        assert!(guard.file_count() > 0);
+    }
+
+    #[test]
+    fn test_origin_dir() {
+        let origin = TestOriginDir::new();
+        let path = origin.add_file("artist/album/track.flac", b"content");
+        assert!(path.exists());
+    }
+}

crates/musicfs-test-utils/src/lib.rs

@@ -0,0 +1,9 @@
+pub mod assertions;
+pub mod faulty_cas;
+pub mod faulty_origin;
+pub mod fixtures;
+
+pub use assertions::*;
+pub use faulty_cas::FaultyCasStore;
+pub use faulty_origin::{FailMode, FaultyOrigin};
+pub use fixtures::*;