e3eeba4650
Phase 1 of resilience testing design doc implementation: - New musicfs-test-utils crate with FaultyOrigin, FaultyCasStore, fixtures - Failpoints instrumented in musicfs-cas/store.rs - 16 resilience tests (13 RED for missing features, 3 GREEN for existing) - 3 Docker/Toxiproxy network tests (RED until docker-compose up) - docker-compose.yml for Toxiproxy + MinIO + SFTP test infrastructure Tests properly fail-first (TDD): check_all() sequential, no health timeout, missing corruption detection, no passthrough mode, etc.
149 lines
4.3 KiB
Rust
149 lines
4.3 KiB
Rust
#![cfg(feature = "docker-tests")]
|
|
|
|
use musicfs_core::{OriginId, OriginType};
|
|
use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
|
|
use noxious_client::{Client, StreamDirection, Toxic, ToxicKind};
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
use tempfile::TempDir;
|
|
|
|
const TOXIPROXY_API: &str = "http://localhost:8474";
|
|
const TOXIPROXY_LISTEN: &str = "localhost:18080";
|
|
const UPSTREAM_ADDR: &str = "minio:9000";
|
|
|
|
async fn require_toxiproxy() {
|
|
let available = match reqwest::get(format!("{}/version", TOXIPROXY_API)).await {
|
|
Ok(resp) => resp.status().is_success(),
|
|
Err(_) => false,
|
|
};
|
|
assert!(available, "Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d", TOXIPROXY_API);
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
|
|
async fn test_toxiproxy_latency_injection() {
|
|
require_toxiproxy().await;
|
|
|
|
let client = Client::new(TOXIPROXY_API);
|
|
let proxy = client
|
|
.create_proxy("minio_latency", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
|
|
.await
|
|
.expect("Failed to create proxy");
|
|
|
|
let toxic = Toxic {
|
|
name: "latency_downstream".to_string(),
|
|
kind: ToxicKind::Latency {
|
|
latency: 500,
|
|
jitter: 100,
|
|
},
|
|
direction: StreamDirection::Downstream,
|
|
toxicity: 1.0,
|
|
};
|
|
|
|
proxy
|
|
.add_toxic(&toxic)
|
|
.await
|
|
.expect("Failed to add toxic");
|
|
|
|
let start = std::time::Instant::now();
|
|
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
|
|
let elapsed = start.elapsed();
|
|
|
|
assert!(
|
|
elapsed >= Duration::from_millis(400),
|
|
"Latency should be injected, got {:?}",
|
|
elapsed
|
|
);
|
|
|
|
proxy.delete().await.expect("Failed to cleanup proxy");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
|
|
async fn test_toxiproxy_timeout_simulates_network_partition() {
|
|
require_toxiproxy().await;
|
|
|
|
let client = Client::new(TOXIPROXY_API);
|
|
let proxy = client
|
|
.create_proxy("minio_partition", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
|
|
.await
|
|
.expect("Failed to create proxy");
|
|
|
|
let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
|
|
assert!(result.is_ok(), "Should reach MinIO through proxy initially");
|
|
|
|
let toxic = Toxic {
|
|
name: "timeout".to_string(),
|
|
kind: ToxicKind::Timeout { timeout: 0 },
|
|
direction: StreamDirection::Downstream,
|
|
toxicity: 1.0,
|
|
};
|
|
|
|
proxy
|
|
.add_toxic(&toxic)
|
|
.await
|
|
.expect("Failed to add toxic");
|
|
|
|
let result = tokio::time::timeout(
|
|
Duration::from_secs(2),
|
|
reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)),
|
|
)
|
|
.await;
|
|
|
|
assert!(
|
|
result.is_err() || result.unwrap().is_err(),
|
|
"Should timeout during partition"
|
|
);
|
|
|
|
proxy
|
|
.remove_toxic("timeout")
|
|
.await
|
|
.expect("Failed to remove toxic");
|
|
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
|
|
let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
|
|
assert!(result.is_ok(), "Should reach MinIO after partition heals");
|
|
|
|
proxy.delete().await.expect("Failed to cleanup proxy");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
|
|
async fn test_toxiproxy_slow_close_throttles_responses() {
|
|
require_toxiproxy().await;
|
|
|
|
let client = Client::new(TOXIPROXY_API);
|
|
let proxy = client
|
|
.create_proxy("minio_slow", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
|
|
.await
|
|
.expect("Failed to create proxy");
|
|
|
|
let toxic = Toxic {
|
|
name: "slow_close".to_string(),
|
|
kind: ToxicKind::SlowClose { delay: 1000 },
|
|
direction: StreamDirection::Downstream,
|
|
toxicity: 1.0,
|
|
};
|
|
|
|
proxy
|
|
.add_toxic(&toxic)
|
|
.await
|
|
.expect("Failed to add toxic");
|
|
|
|
let start = std::time::Instant::now();
|
|
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
|
|
let elapsed = start.elapsed();
|
|
|
|
assert!(
|
|
elapsed >= Duration::from_millis(800),
|
|
"Slow close should delay response, got {:?}",
|
|
elapsed
|
|
);
|
|
|
|
proxy.delete().await.expect("Failed to cleanup proxy");
|
|
}
|
|
|
|
|