Add musicfs-test-utils crate with RED resilience tests
Phase 1 of resilience testing design doc implementation: - New musicfs-test-utils crate with FaultyOrigin, FaultyCasStore, fixtures - Failpoints instrumented in musicfs-cas/store.rs - 16 resilience tests (13 RED for missing features, 3 GREEN for existing) - 3 Docker/Toxiproxy network tests (RED until docker-compose up) - docker-compose.yml for Toxiproxy + MinIO + SFTP test infrastructure Tests properly fail-first (TDD): check_all() sequential, no health timeout, missing corruption detection, no passthrough mode, etc.
This commit is contained in:
@@ -6,6 +6,9 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::fs;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
use fail::fail_point;
|
||||
|
||||
const DEFAULT_MAX_SIZE_10GB: u64 = 10 * 1024 * 1024 * 1024;
|
||||
const DEFAULT_SHARD_LEVELS_256_SUBDIRS: u8 = 2;
|
||||
|
||||
@@ -80,8 +83,24 @@ impl CasStore {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
fail_point!("cas-put-before-write", |_| {
|
||||
Err(CasError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Failpoint: cas-put-before-write",
|
||||
)))
|
||||
});
|
||||
|
||||
fs::write(&path, data).await?;
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
fail_point!("cas-put-after-write-before-index", |_| {
|
||||
Err(CasError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Failpoint: cas-put-after-write-before-index",
|
||||
)))
|
||||
});
|
||||
|
||||
let location = ChunkLocation {
|
||||
path: path.clone(),
|
||||
size: data.len() as u32,
|
||||
|
||||
Reference in New Issue
Block a user