Files
MusicFS/musicfs/crates/musicfs-test-utils/tests/docker_network.rs
T
Alexander e3eeba4650 Add musicfs-test-utils crate with RED resilience tests
Phase 1 of resilience testing design doc implementation:
- New musicfs-test-utils crate with FaultyOrigin, FaultyCasStore, fixtures
- Failpoints instrumented in musicfs-cas/store.rs
- 16 resilience tests (13 RED for missing features, 3 GREEN for existing)
- 3 Docker/Toxiproxy network tests (RED until docker-compose up)
- docker-compose.yml for Toxiproxy + MinIO + SFTP test infrastructure

Tests properly fail-first (TDD): check_all() sequential, no health timeout,
missing corruption detection, no passthrough mode, etc.
2026-05-13 13:49:25 +02:00

149 lines
4.3 KiB
Rust

#![cfg(feature = "docker-tests")]
use musicfs_core::{OriginId, OriginType};
use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
use noxious_client::{Client, StreamDirection, Toxic, ToxicKind};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use tempfile::TempDir;
const TOXIPROXY_API: &str = "http://localhost:8474";
const TOXIPROXY_LISTEN: &str = "localhost:18080";
const UPSTREAM_ADDR: &str = "minio:9000";
async fn require_toxiproxy() {
let available = match reqwest::get(format!("{}/version", TOXIPROXY_API)).await {
Ok(resp) => resp.status().is_success(),
Err(_) => false,
};
assert!(available, "Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d", TOXIPROXY_API);
}
#[tokio::test]
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
async fn test_toxiproxy_latency_injection() {
require_toxiproxy().await;
let client = Client::new(TOXIPROXY_API);
let proxy = client
.create_proxy("minio_latency", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
.await
.expect("Failed to create proxy");
let toxic = Toxic {
name: "latency_downstream".to_string(),
kind: ToxicKind::Latency {
latency: 500,
jitter: 100,
},
direction: StreamDirection::Downstream,
toxicity: 1.0,
};
proxy
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
let start = std::time::Instant::now();
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(400),
"Latency should be injected, got {:?}",
elapsed
);
proxy.delete().await.expect("Failed to cleanup proxy");
}
#[tokio::test]
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
async fn test_toxiproxy_timeout_simulates_network_partition() {
require_toxiproxy().await;
let client = Client::new(TOXIPROXY_API);
let proxy = client
.create_proxy("minio_partition", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
.await
.expect("Failed to create proxy");
let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
assert!(result.is_ok(), "Should reach MinIO through proxy initially");
let toxic = Toxic {
name: "timeout".to_string(),
kind: ToxicKind::Timeout { timeout: 0 },
direction: StreamDirection::Downstream,
toxicity: 1.0,
};
proxy
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
let result = tokio::time::timeout(
Duration::from_secs(2),
reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)),
)
.await;
assert!(
result.is_err() || result.unwrap().is_err(),
"Should timeout during partition"
);
proxy
.remove_toxic("timeout")
.await
.expect("Failed to remove toxic");
tokio::time::sleep(Duration::from_millis(100)).await;
let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
assert!(result.is_ok(), "Should reach MinIO after partition heals");
proxy.delete().await.expect("Failed to cleanup proxy");
}
#[tokio::test]
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
async fn test_toxiproxy_slow_close_throttles_responses() {
require_toxiproxy().await;
let client = Client::new(TOXIPROXY_API);
let proxy = client
.create_proxy("minio_slow", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
.await
.expect("Failed to create proxy");
let toxic = Toxic {
name: "slow_close".to_string(),
kind: ToxicKind::SlowClose { delay: 1000 },
direction: StreamDirection::Downstream,
toxicity: 1.0,
};
proxy
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
let start = std::time::Instant::now();
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(800),
"Slow close should delay response, got {:?}",
elapsed
);
proxy.delete().await.expect("Failed to cleanup proxy");
}