Add musicfs-test-utils crate with RED resilience tests

Phase 1 of resilience testing design doc implementation:
- New musicfs-test-utils crate with FaultyOrigin, FaultyCasStore, fixtures
- Failpoints instrumented in musicfs-cas/store.rs
- 16 resilience tests (13 RED for missing features, 3 GREEN for existing)
- 3 Docker/Toxiproxy network tests (RED until docker-compose up)
- docker-compose.yml for Toxiproxy + MinIO + SFTP test infrastructure

Tests properly fail-first (TDD): check_all() sequential, no health timeout,
missing corruption detection, no passthrough mode, etc.
This commit is contained in:
Alexander
2026-05-13 13:49:25 +02:00
parent 00f14930cd
commit e3eeba4650
14 changed files with 1912 additions and 31 deletions
+19
View File
@@ -6,6 +6,9 @@ use std::sync::atomic::{AtomicU64, Ordering};
use tokio::fs;
use tracing::{debug, trace, warn};
#[cfg(feature = "failpoints")]
use fail::fail_point;
const DEFAULT_MAX_SIZE_10GB: u64 = 10 * 1024 * 1024 * 1024;
const DEFAULT_SHARD_LEVELS_256_SUBDIRS: u8 = 2;
@@ -80,8 +83,24 @@ impl CasStore {
fs::create_dir_all(parent).await?;
}
#[cfg(feature = "failpoints")]
fail_point!("cas-put-before-write", |_| {
Err(CasError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"Failpoint: cas-put-before-write",
)))
});
fs::write(&path, data).await?;
#[cfg(feature = "failpoints")]
fail_point!("cas-put-after-write-before-index", |_| {
Err(CasError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"Failpoint: cas-put-after-write-before-index",
)))
});
let location = ChunkLocation {
path: path.clone(),
size: data.len() as u32,