Add musicfs-test-utils crate with RED resilience tests

Phase 1 of resilience testing design doc implementation: - New musicfs-test-utils crate with FaultyOrigin, FaultyCasStore, fixtures - Failpoints instrumented in musicfs-cas/store.rs - 16 resilience tests (13 RED for missing features, 3 GREEN for existing) - 3 Docker/Toxiproxy network tests (RED until docker-compose up) - docker-compose.yml for Toxiproxy + MinIO + SFTP test infrastructure Tests properly fail-first (TDD): check_all() sequential, no health timeout, missing corruption detection, no passthrough mode, etc.
2026-05-13 13:49:25 +02:00
parent 00f14930cd
commit e3eeba4650
14 changed files with 1912 additions and 31 deletions
@@ -0,0 +1,148 @@
+#![cfg(feature = "docker-tests")]
+
+use musicfs_core::{OriginId, OriginType};
+use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
+use noxious_client::{Client, StreamDirection, Toxic, ToxicKind};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+use tempfile::TempDir;
+
+const TOXIPROXY_API: &str = "http://localhost:8474";
+const TOXIPROXY_LISTEN: &str = "localhost:18080";
+const UPSTREAM_ADDR: &str = "minio:9000";
+
+async fn require_toxiproxy() {
+    let available = match reqwest::get(format!("{}/version", TOXIPROXY_API)).await {
+        Ok(resp) => resp.status().is_success(),
+        Err(_) => false,
+    };
+    assert!(available, "Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d", TOXIPROXY_API);
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_latency_injection() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_latency", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let toxic = Toxic {
+        name: "latency_downstream".to_string(),
+        kind: ToxicKind::Latency {
+            latency: 500,
+            jitter: 100,
+        },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy
+        .add_toxic(&toxic)
+        .await
+        .expect("Failed to add toxic");
+
+    let start = std::time::Instant::now();
+    let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    let elapsed = start.elapsed();
+
+    assert!(
+        elapsed >= Duration::from_millis(400),
+        "Latency should be injected, got {:?}",
+        elapsed
+    );
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_timeout_simulates_network_partition() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_partition", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    assert!(result.is_ok(), "Should reach MinIO through proxy initially");
+
+    let toxic = Toxic {
+        name: "timeout".to_string(),
+        kind: ToxicKind::Timeout { timeout: 0 },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy
+        .add_toxic(&toxic)
+        .await
+        .expect("Failed to add toxic");
+
+    let result = tokio::time::timeout(
+        Duration::from_secs(2),
+        reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)),
+    )
+    .await;
+
+    assert!(
+        result.is_err() || result.unwrap().is_err(),
+        "Should timeout during partition"
+    );
+
+    proxy
+        .remove_toxic("timeout")
+        .await
+        .expect("Failed to remove toxic");
+
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    assert!(result.is_ok(), "Should reach MinIO after partition heals");
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}
+
+#[tokio::test]
+#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
+async fn test_toxiproxy_slow_close_throttles_responses() {
+    require_toxiproxy().await;
+
+    let client = Client::new(TOXIPROXY_API);
+    let proxy = client
+        .create_proxy("minio_slow", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
+        .await
+        .expect("Failed to create proxy");
+
+    let toxic = Toxic {
+        name: "slow_close".to_string(),
+        kind: ToxicKind::SlowClose { delay: 1000 },
+        direction: StreamDirection::Downstream,
+        toxicity: 1.0,
+    };
+
+    proxy
+        .add_toxic(&toxic)
+        .await
+        .expect("Failed to add toxic");
+
+    let start = std::time::Instant::now();
+    let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
+    let elapsed = start.elapsed();
+
+    assert!(
+        elapsed >= Duration::from_millis(800),
+        "Slow close should delay response, got {:?}",
+        elapsed
+    );
+
+    proxy.delete().await.expect("Failed to cleanup proxy");
+}
+
+