fujin/MusicFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add musicfs-test-utils crate with RED resilience tests

Browse Source 
Phase 1 of resilience testing design doc implementation:
- New musicfs-test-utils crate with FaultyOrigin, FaultyCasStore, fixtures
- Failpoints instrumented in musicfs-cas/store.rs
- 16 resilience tests (13 RED for missing features, 3 GREEN for existing)
- 3 Docker/Toxiproxy network tests (RED until docker-compose up)
- docker-compose.yml for Toxiproxy + MinIO + SFTP test infrastructure

Tests properly fail-first (TDD): check_all() sequential, no health timeout,
missing corruption detection, no passthrough mode, etc.
This commit is contained in:
Alexander
2026-05-13 13:49:25 +02:00
parent 00f14930cd
commit e3eeba4650
14 changed files with 1912 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
musicfs/Cargo.lock
Generated
+353 -30
View File
@@ -158,7 +158,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -169,7 +169,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -274,6 +274,17 @@ dependencies = [
"generic-array",
]
[[package]]
name = "bmrng"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9758e48498ae13d49b51a979d553d254e67021b203d9597e82a04ebd81025b2"
dependencies = [
"futures",
"loom",
"tokio",
]
[[package]]
name = "bumpalo"
version = "3.20.2"
@@ -322,6 +333,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.44"
@@ -366,7 +383,7 @@ dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -692,7 +709,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -738,6 +755,17 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
[[package]]
name = "fail"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c"
dependencies = [
"log",
"once_cell",
"rand",
]
[[package]]
name = "fallible-iterator"
version = "0.3.0"
@@ -889,6 +917,21 @@ dependencies = [
"zerocopy 0.7.35",
]
[[package]]
name = "futures"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.32"
@@ -896,6 +939,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
@@ -904,6 +948,34 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
[[package]]
name = "futures-executor"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
[[package]]
name = "futures-macro"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "futures-sink"
version = "0.3.32"
@@ -922,8 +994,13 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"slab",
]
@@ -950,6 +1027,19 @@ dependencies = [
"serde_json",
]
[[package]]
name = "generator"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "061d3be1afec479d56fa3bd182bf966c7999ec175fcfdb87ac14d417241366c6"
dependencies = [
"cc",
"libc",
"log",
"rustversion",
"winapi",
]
[[package]]
name = "generic-array"
version = "0.14.7"
@@ -1022,7 +1112,7 @@ dependencies = [
"indexmap 2.14.0",
"slab",
"tokio",
"tokio-util",
"tokio-util 0.7.18",
"tracing",
]
@@ -1173,6 +1263,20 @@ dependencies = [
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
dependencies = [
"futures-util",
"http",
"hyper",
"rustls",
"tokio",
"tokio-rustls",
]
[[package]]
name = "hyper-timeout"
version = "0.4.1"
@@ -1593,6 +1697,20 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "loom"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27a6650b2f722ae8c0e2ebc46d07f80c9923464fc206d962332f1eff83143530"
dependencies = [
"cfg-if",
"futures-util",
"generator",
"scoped-tls",
"serde",
"serde_json",
]
[[package]]
name = "lru"
version = "0.12.5"
@@ -1720,6 +1838,18 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "mockall_double"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dffc15b97456ecc84d2bde8c1df79145e154f45225828c4361f676e1b82acd6"
dependencies = [
"cfg-if",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "moka"
version = "0.12.15"
@@ -1775,6 +1905,7 @@ version = "0.1.0"
dependencies = [
"bytes",
"dirs",
"fail",
"hex",
"musicfs-cache",
"musicfs-core",
@@ -1942,6 +2073,29 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "musicfs-test-utils"
version = "0.1.0"
dependencies = [
"async-trait",
"bytes",
"fail",
"musicfs-cache",
"musicfs-cas",
"musicfs-core",
"musicfs-origins",
"nix",
"noxious-client",
"parking_lot 0.12.5",
"reqwest",
"rlimit",
"tempfile",
"thiserror 1.0.69",
"tokio",
"tokio-test",
"tracing",
]
[[package]]
name = "native-tls"
version = "0.2.18"
@@ -1959,6 +2113,18 @@ dependencies = [
"tempfile",
]
[[package]]
name = "nix"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
"bitflags 2.11.1",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]]
name = "nom"
version = "7.1.3"
@@ -1988,6 +2154,39 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "noxious"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e68998924150ba54dbf1adf4c3f7f7c10bb5d3c6789ab71af11e34fe4c667970"
dependencies = [
"async-trait",
"bmrng",
"bytes",
"futures",
"mockall_double",
"pin-project-lite",
"rand",
"serde",
"thiserror 1.0.69",
"tokio",
"tokio-util 0.6.10",
"tracing",
]
[[package]]
name = "noxious-client"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b7ab7a9efb5768cd07e2b2455f80b3998d7397be76398c2ac03a52a42b652e7"
dependencies = [
"noxious",
"reqwest",
"serde",
"thiserror 1.0.69",
"tokio",
]
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -2084,7 +2283,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -2217,7 +2416,7 @@ checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -2282,7 +2481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -2321,7 +2520,7 @@ dependencies = [
"prost",
"prost-types",
"regex",
"syn",
"syn 2.0.117",
"tempfile",
]
@@ -2335,7 +2534,7 @@ dependencies = [
"itertools",
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -2524,6 +2723,7 @@ dependencies = [
"http",
"http-body",
"hyper",
"hyper-rustls",
"hyper-tls",
"ipnet",
"js-sys",
@@ -2533,6 +2733,7 @@ dependencies = [
"once_cell",
"percent-encoding",
"pin-project-lite",
"rustls",
"rustls-pemfile",
"serde",
"serde_json",
@@ -2541,14 +2742,39 @@ dependencies = [
"system-configuration",
"tokio",
"tokio-native-tls",
"tokio-rustls",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webpki-roots",
"winreg",
]
[[package]]
name = "ring"
version = "0.17.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
"getrandom 0.2.17",
"libc",
"untrusted",
"windows-sys 0.52.0",
]
[[package]]
name = "rlimit"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a"
dependencies = [
"libc",
]
[[package]]
name = "rmp"
version = "0.8.15"
@@ -2630,6 +2856,18 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "rustls"
version = "0.21.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
dependencies = [
"log",
"ring",
"rustls-webpki",
"sct",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.4"
@@ -2639,6 +2877,16 @@ dependencies = [
"base64 0.21.7",
]
[[package]]
name = "rustls-webpki"
version = "0.101.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "rustversion"
version = "1.0.22"
@@ -2669,12 +2917,28 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "sct"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "security-framework"
version = "3.7.0"
@@ -2731,7 +2995,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -3036,6 +3300,17 @@ dependencies = [
"symphonia-metadata",
]
[[package]]
name = "syn"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.117"
@@ -3061,7 +3336,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -3277,7 +3552,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -3288,7 +3563,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -3376,7 +3651,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -3389,6 +3664,16 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
"rustls",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.18"
@@ -3400,6 +3685,31 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-test"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545"
dependencies = [
"futures-core",
"tokio",
"tokio-stream",
]
[[package]]
name = "tokio-util"
version = "0.6.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
"log",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.18"
@@ -3491,7 +3801,7 @@ dependencies = [
"proc-macro2",
"prost-build",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -3508,7 +3818,7 @@ dependencies = [
"rand",
"slab",
"tokio",
"tokio-util",
"tokio-util 0.7.18",
"tower-layer",
"tower-service",
"tracing",
@@ -3532,6 +3842,7 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -3558,7 +3869,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -3654,6 +3965,12 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.8"
@@ -3799,7 +4116,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
"wasm-bindgen-shared",
]
@@ -3981,7 +4298,7 @@ dependencies = [
"anyhow",
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
"wasmtime-component-util",
"wasmtime-wit-bindgen",
"wit-parser 0.201.0",
@@ -4155,7 +4472,7 @@ checksum = "ffaafa5c12355b1a9ee068e9295d50c4ca0a400c721950cdae4f5b54391a2da5"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -4225,6 +4542,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.25.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
[[package]]
name = "winapi"
version = "0.3.9"
@@ -4293,7 +4616,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -4304,7 +4627,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -4543,7 +4866,7 @@ dependencies = [
"heck 0.5.0",
"indexmap 2.14.0",
"prettyplease",
"syn",
"syn 2.0.117",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
@@ -4559,7 +4882,7 @@ dependencies = [
"prettyplease",
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
"wit-bindgen-core",
"wit-bindgen-rust",
]
@@ -4650,7 +4973,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
"synstructure",
]
@@ -4681,7 +5004,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -4692,7 +5015,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]
@@ -4712,7 +5035,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
"synstructure",
]
@@ -4746,7 +5069,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.117",
]
[[package]]