fujin/MusicFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement Phase A: Stop Dying resilience fixes

Browse Source 
Implements all 6 critical resilience fixes from phase-a-stop-dying.md:

- Issue 2.9: Migrate std::sync::RwLock → parking_lot::RwLock (7 files)
  Prevents lock poisoning cascade on writer panic

- Issue 2.2: Add install_panic_hook() to log panics via tracing
  Ensures panics are captured in logs/journald before process death

- Issue 3.7: Add ExecStopPost to systemd service
  Cleans up stale FUSE mounts on service stop

- Issue 2.7: Add check_stale_mount() detection on startup
  Auto-cleans leftover mounts from previous crashes

- Issue 2.10: Integrate sd_notify for systemd lifecycle
  Sends READY=1 after mount, STOPPING on shutdown

- Issue 2.1: Add signal handling with spawn_mount
  Catches SIGTERM/SIGINT for clean shutdown instead of instant death

All 7 Phase A tests pass:
- test_poisoned_tree_lock_returns_eio_not_panic
- test_parking_lot_rwlock_survives_panic
- test_panic_hook_logs_to_tracing
- test_systemd_service_has_execstoppost
- test_stale_mount_check_function_exists
- test_sd_notify_ready_sent
- test_sigterm_triggers_shutdown
This commit is contained in:
Alexander
2026-05-13 14:48:32 +02:00
parent 24086cc744
commit 6285eeb6c0
18 changed files with 301 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
musicfs/Cargo.lock
Generated
+16
View File
@@ -1911,6 +1911,7 @@ dependencies = [
"musicfs-core",
"musicfs-origins",
"musicfs-sync",
"parking_lot 0.12.5",
"rmp-serde",
"serde",
"sled",
@@ -1934,6 +1935,8 @@ dependencies = [
"musicfs-fuse",
"musicfs-metadata",
"musicfs-origins",
"parking_lot 0.12.5",
"sd-notify",
"tokio",
"tracing",
"tracing-appender",
@@ -1946,6 +1949,7 @@ name = "musicfs-core"
version = "0.1.0"
dependencies = [
"hex",
"parking_lot 0.12.5",
"serde",
"serde_json",
"tempfile",
@@ -2015,6 +2019,7 @@ dependencies = [
"dashmap",
"libc",
"musicfs-core",
"parking_lot 0.12.5",
"tempfile",
"thiserror 1.0.69",
"tokio",
@@ -2080,6 +2085,7 @@ dependencies = [
"async-trait",
"bytes",
"fail",
"libc",
"musicfs-cache",
"musicfs-cas",
"musicfs-core",
@@ -2089,6 +2095,7 @@ dependencies = [
"parking_lot 0.12.5",
"reqwest",
"rlimit",
"sd-notify",
"tempfile",
"thiserror 1.0.69",
"tokio",
@@ -2939,6 +2946,15 @@ dependencies = [
"untrusted",
]
[[package]]
name = "sd-notify"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4"
dependencies = [
"libc",
]
[[package]]
name = "security-framework"
version = "3.7.0"
musicfs/Cargo.toml
+2
View File
@@ -87,5 +87,7 @@ tokio-stream = "0.1"
image = { version = "0.24", default-features = false, features = ["jpeg", "png"] }
chrono = "0.4"
sd-notify = "0.4"
[workspace.dependencies.tonic-build]
version = "0.11"
musicfs/crates/musicfs-cache/src/eviction.rs
+6 -6
View File
@@ -1,7 +1,7 @@
use musicfs_cas::CasStore;
use musicfs_core::ChunkHash;
use parking_lot::RwLock;
use std::collections::BTreeMap;
use std::sync::RwLock;
use std::time::Instant;
use tracing::info;
@@ -64,8 +64,8 @@ impl Default for LruEviction {
impl EvictionPolicy for LruEviction {
fn record_access(&self, hash: ChunkHash) {
let now = Instant::now();
let mut times = self.access_times.write().unwrap();
let mut h2t = self.hash_to_time.write().unwrap();
let mut times = self.access_times.write();
let mut h2t = self.hash_to_time.write();
if let Some(old_time) = h2t.remove(&hash) {
times.remove(&old_time);
@@ -76,13 +76,13 @@ impl EvictionPolicy for LruEviction {
}
fn select_victims(&self, count: usize) -> Vec<ChunkHash> {
let times = self.access_times.read().unwrap();
let times = self.access_times.read();
times.values().take(count).copied().collect()
}
fn remove(&self, hash: &ChunkHash) {
let mut times = self.access_times.write().unwrap();
let mut h2t = self.hash_to_time.write().unwrap();
let mut times = self.access_times.write();
let mut h2t = self.hash_to_time.write();
if let Some(time) = h2t.remove(hash) {
times.remove(&time);
musicfs/crates/musicfs-cache/src/tree.rs
+4 -4
View File
@@ -1,8 +1,8 @@
use musicfs_core::{FileId, FileMeta, VirtualPath};
use parking_lot::RwLock;
use std::collections::{BTreeMap, HashMap};
use std::ffi::{OsStr, OsString};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::RwLock;
use std::time::{Duration, SystemTime};
use tracing::{debug, trace};
@@ -291,7 +291,7 @@ impl VirtualTree {
}
pub fn needs_refresh(&self) -> bool {
let last = *self.last_refresh.read().unwrap();
let last = *self.last_refresh.read();
last.elapsed().unwrap_or(Duration::MAX) > self.refresh_policy.ttl
}
@@ -303,11 +303,11 @@ impl VirtualTree {
root.children.clear();
}
*self.last_refresh.write().unwrap() = SystemTime::now();
*self.last_refresh.write() = SystemTime::now();
}
pub fn mark_refreshed(&self) {
*self.last_refresh.write().unwrap() = SystemTime::now();
*self.last_refresh.write() = SystemTime::now();
}
pub fn refresh_policy(&self) -> &RefreshPolicy {
musicfs/crates/musicfs-cas/Cargo.toml
+1
View File
@@ -22,6 +22,7 @@ rmp-serde.workspace = true
hex.workspace = true
dirs.workspace = true
thiserror.workspace = true
parking_lot.workspace = true
[dev-dependencies]
tempfile.workspace = true
musicfs/crates/musicfs-cas/src/fetcher.rs
+8 -7
View File
@@ -2,8 +2,9 @@ use crate::{CasStore, ChunkManifest, ChunkRef};
use musicfs_core::{Event, EventBus, FileId, FileMeta, OriginId};
use musicfs_origins::Origin;
use musicfs_sync::CdcChunker;
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use tracing::{debug, info};
pub struct ContentFetcher {
@@ -37,15 +38,15 @@ impl ContentFetcher {
pub fn register_origin(&self, origin: Arc<dyn Origin>) {
let id = origin.id().clone();
self.origins.write().unwrap().insert(id, origin);
self.origins.write().insert(id, origin);
}
pub fn register_file(&self, meta: FileMeta) {
self.file_meta.write().unwrap().insert(meta.id, meta);
self.file_meta.write().insert(meta.id, meta);
}
pub fn register_files(&self, files: impl IntoIterator<Item = FileMeta>) {
let mut map = self.file_meta.write().unwrap();
let mut map = self.file_meta.write();
for meta in files {
map.insert(meta.id, meta);
}
@@ -53,7 +54,7 @@ impl ContentFetcher {
pub async fn fetch_file(&self, file_id: FileId) -> Result<ChunkManifest, FetchError> {
let meta = {
let files = self.file_meta.read().unwrap();
let files = self.file_meta.read();
files
.get(&file_id)
.cloned()
@@ -61,7 +62,7 @@ impl ContentFetcher {
};
let origin = {
let origins = self.origins.read().unwrap();
let origins = self.origins.read();
origins
.get(&meta.real_path.origin_id)
.cloned()
@@ -123,7 +124,7 @@ impl ContentFetcher {
}
pub fn get_file_meta(&self, file_id: FileId) -> Option<FileMeta> {
self.file_meta.read().unwrap().get(&file_id).cloned()
self.file_meta.read().get(&file_id).cloned()
}
pub fn emit_access_event(&self, meta: &FileMeta, offset: u64, size: u32) {
musicfs/crates/musicfs-cas/src/reader.rs
+4 -4
View File
@@ -3,9 +3,10 @@ use crate::fetcher::{ContentFetcher, FetchError};
use crate::store::CasStore;
use bytes::{Bytes, BytesMut};
use musicfs_core::FileId;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use tracing::{debug, trace};
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -60,13 +61,13 @@ impl FileReader {
}
pub fn register_manifest(&self, manifest: ChunkManifest) {
let mut manifests = self.manifests.write().unwrap();
let mut manifests = self.manifests.write();
manifests.insert(manifest.file_id, manifest);
}
async fn get_or_fetch_manifest(&self, file_id: FileId) -> Result<ChunkManifest, ReaderError> {
{
let manifests = self.manifests.read().unwrap();
let manifests = self.manifests.read();
if let Some(m) = manifests.get(&file_id) {
trace!(file_id = ?file_id, "manifest cache hit");
return Ok(m.clone());
@@ -81,7 +82,6 @@ impl FileReader {
let manifest = fetcher.ensure_cached(file_id).await?;
self.manifests
.write()
.unwrap()
.insert(file_id, manifest.clone());
Ok(manifest)
}
musicfs/crates/musicfs-cli/Cargo.toml
+2
View File
@@ -22,6 +22,8 @@ tracing-subscriber.workspace = true
tracing-appender.workspace = true
anyhow.workspace = true
dirs.workspace = true
parking_lot.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
tracing-journald.workspace = true
sd-notify.workspace = true
musicfs/crates/musicfs-cli/src/main.rs
+65 -5
View File
@@ -6,10 +6,11 @@ use musicfs_core::{FileId, FileMeta, LoggingConfig, OriginId, RealPath, VirtualP
use musicfs_fuse::MusicFs;
use musicfs_metadata::MetadataParser;
use musicfs_origins::{LocalOrigin, Origin};
use parking_lot::RwLock;
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use std::time::SystemTime;
use tracing::{debug, info};
use tracing::{debug, info, warn};
use tracing_appender::non_blocking::WorkerGuard;
use tracing_subscriber::{fmt, prelude::*, EnvFilter, Layer};
@@ -87,6 +88,7 @@ enum OriginCommands {
}
fn main() -> Result<()> {
musicfs_core::install_panic_hook();
let cli = Cli::parse();
match cli.command {
@@ -188,14 +190,50 @@ fn run_mount(
Ok::<_, anyhow::Error>((tree, reader))
})?;
let fs = MusicFs::with_reader(tree, reader, handle);
check_stale_mount(&mountpoint)?;
let fs = MusicFs::with_reader(tree, reader, handle.clone());
info!("Mounting filesystem at {:?}", mountpoint);
info!("Press Ctrl+C to unmount");
fs.mount(&mountpoint)
let session = fs
.spawn_mount(&mountpoint)
.context("Failed to mount filesystem")?;
#[cfg(target_os = "linux")]
{
if let Err(e) = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) {
debug!("sd_notify not available (not running under systemd): {}", e);
}
}
info!("MusicFS ready, PID {}", std::process::id());
runtime.block_on(async {
let mut sigterm =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
let mut sigint =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
tokio::select! {
_ = sigterm.recv() => {
info!("Received SIGTERM, shutting down");
}
_ = sigint.recv() => {
info!("Received SIGINT, shutting down");
}
}
Ok::<_, anyhow::Error>(())
})?;
#[cfg(target_os = "linux")]
{
let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]);
}
info!("Unmounting filesystem");
drop(session);
info!("Shutdown complete");
Ok(())
}
@@ -437,3 +475,25 @@ fn sanitize(s: &str) -> String {
})
.collect()
}
fn check_stale_mount(mountpoint: &Path) -> Result<()> {
if let Ok(mounts) = std::fs::read_to_string("/proc/mounts") {
for line in mounts.lines() {
if line.contains(mountpoint.to_string_lossy().as_ref()) && line.contains("fuse") {
warn!(
"Stale FUSE mount detected at {:?}, attempting cleanup",
mountpoint
);
let status = std::process::Command::new("fusermount")
.args(["-uz", &mountpoint.to_string_lossy()])
.status();
match status {
Ok(s) if s.success() => info!("Stale mount cleaned up"),
Ok(s) => warn!("fusermount exited with: {}", s),
Err(e) => warn!("Failed to run fusermount: {}", e),
}
}
}
}
Ok(())
}
musicfs/crates/musicfs-core/Cargo.toml
+1
View File
@@ -12,6 +12,7 @@ tokio = { workspace = true, features = ["sync"] }
tracing.workspace = true
xxhash-rust.workspace = true
hex.workspace = true
parking_lot.workspace = true
[dev-dependencies]
tempfile.workspace = true
musicfs/crates/musicfs-core/src/lib.rs
+32
View File
@@ -19,6 +19,38 @@ pub fn sanitize_path(path: &Path) -> String {
path.to_string_lossy().to_string()
}
}
/// Install a custom panic hook that logs panics via tracing before the default behavior.
/// This ensures panics are captured in log files and journald.
pub fn install_panic_hook() {
let default_hook = std::panic::take_hook();
std::panic::set_hook(Box::new(move |info| {
let thread = std::thread::current();
let thread_name = thread.name().unwrap_or("<unnamed>");
let message = if let Some(s) = info.payload().downcast_ref::<&str>() {
(*s).to_string()
} else if let Some(s) = info.payload().downcast_ref::<String>() {
s.clone()
} else {
"unknown panic".to_string()
};
let location = info
.location()
.map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
.unwrap_or_else(|| "unknown location".to_string());
tracing::error!(
thread = thread_name,
location = %location,
"PANIC: {}",
message
);
default_hook(info);
}));
}
pub use credentials::{Credential, CredentialConfig, CredentialError, CredentialStore};
pub use error::{Error, Result};
pub use events::{Event, EventBus};
musicfs/crates/musicfs-core/src/metrics.rs
+4 -5
View File
@@ -1,6 +1,6 @@
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::RwLock;
use std::time::Instant;
#[derive(Default)]
@@ -45,7 +45,7 @@ impl Metrics {
self.fuse_ops.open.load(Ordering::Relaxed),
));
for (op, histogram) in self.fuse_latency.histograms.read().unwrap().iter() {
for (op, histogram) in self.fuse_latency.histograms.read().iter() {
let quantiles = histogram.quantiles();
output.push_str(&format!(
"# HELP musicfs_fuse_latency_seconds FUSE operation latency\n\
@@ -95,7 +95,7 @@ impl Metrics {
"# HELP musicfs_origin_health Origin health status (1=healthy, 0=unhealthy)\n\
# TYPE musicfs_origin_health gauge\n",
);
for (origin_id, healthy) in self.origin_health.status.read().unwrap().iter() {
for (origin_id, healthy) in self.origin_health.status.read().iter() {
output.push_str(&format!(
"musicfs_origin_health{{origin=\"{}\"}} {}\n",
origin_id,
@@ -203,7 +203,7 @@ pub struct FuseLatencyMetrics {
impl FuseLatencyMetrics {
pub fn record(&self, op: &str, latency_secs: f64) {
let mut histograms = self.histograms.write().unwrap();
let mut histograms = self.histograms.write();
histograms
.entry(op.to_string())
.or_default()
@@ -268,7 +268,6 @@ impl OriginHealthMetrics {
pub fn set_health(&self, origin_id: &str, healthy: bool) {
self.status
.write()
.unwrap()
.insert(origin_id.to_string(), healthy);
}
}
musicfs/crates/musicfs-fuse/src/filesystem.rs
+29 -12
View File
@@ -6,10 +6,11 @@ use fuser::{
use musicfs_cache::{VirtualNode, VirtualTree, ROOT_INODE};
use musicfs_cas::FileReader;
use musicfs_core::Result;
use parking_lot::RwLock;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::path::Path;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tokio::runtime::Handle;
use tracing::{debug, info, instrument, trace, warn};
@@ -65,15 +66,15 @@ impl MusicFs {
}
fn get_or_create_query_inode(&self, query: &str) -> u64 {
let query_inodes = self.query_inodes.read().unwrap();
let query_inodes = self.query_inodes.read();
if let Some(&inode) = query_inodes.get(query) {
return inode;
}
drop(query_inodes);
let mut query_inodes = self.query_inodes.write().unwrap();
let mut inode_queries = self.inode_queries.write().unwrap();
let mut next_inode = self.next_query_inode.write().unwrap();
let mut query_inodes = self.query_inodes.write();
let mut inode_queries = self.inode_queries.write();
let mut next_inode = self.next_query_inode.write();
if let Some(&inode) = query_inodes.get(query) {
return inode;
@@ -87,7 +88,7 @@ impl MusicFs {
}
fn get_query_for_inode(&self, inode: u64) -> Option<String> {
self.inode_queries.read().unwrap().get(&inode).cloned()
self.inode_queries.read().get(&inode).cloned()
}
pub fn mount(self, mountpoint: &Path) -> Result<()> {
@@ -105,6 +106,22 @@ impl MusicFs {
Ok(())
}
pub fn spawn_mount(self, mountpoint: &Path) -> Result<fuser::BackgroundSession> {
info!("Mounting MusicFS at {:?}", mountpoint);
let options = vec![
fuser::MountOption::RO,
fuser::MountOption::FSName("musicfs".to_string()),
fuser::MountOption::AutoUnmount,
fuser::MountOption::AllowOther,
];
let session =
fuser::spawn_mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
Ok(session)
}
fn node_to_attr(&self, node: &VirtualNode) -> FileAttr {
match node {
VirtualNode::Directory(dir) => FileAttr {
@@ -189,7 +206,7 @@ impl Filesystem for MusicFs {
}
}
let tree = self.tree.read().unwrap();
let tree = self.tree.read();
if let Some(inode) = tree.lookup(parent, name) {
trace!(parent, name = %name_str, ino = inode, "file found in tree");
@@ -230,7 +247,7 @@ impl Filesystem for MusicFs {
}
}
let tree = self.tree.read().unwrap();
let tree = self.tree.read();
if let Some(node) = tree.get(ino) {
trace!(ino, "inode found in tree");
@@ -267,7 +284,7 @@ impl Filesystem for MusicFs {
}
}
let tree = self.tree.read().unwrap();
let tree = self.tree.read();
if let Some(children) = tree.readdir(ino) {
trace!(ino, offset, children_count = children.len(), "directory found");
@@ -324,7 +341,7 @@ impl Filesystem for MusicFs {
return;
}
let tree = self.tree.read().unwrap();
let tree = self.tree.read();
if tree.get(ino).is_some() {
trace!(ino, "inode found");
@@ -348,7 +365,7 @@ impl Filesystem for MusicFs {
reply: ReplyData,
) {
let file_id = {
let tree = self.tree.read().unwrap();
let tree = self.tree.read();
if let Some(VirtualNode::File(file)) = tree.get(ino) {
trace!(ino, "file found in tree");
file.file_id
@@ -564,7 +581,7 @@ mod tests {
let _fs = MusicFs::new(tree.clone(), handle);
let tree_read = tree.read().unwrap();
let tree_read = tree.read();
assert!(tree_read.get(ROOT_INODE).is_some());
assert!(tree_read.get_by_path(&VirtualPath::new("/Artist")).is_some());
}
musicfs/crates/musicfs-origins/Cargo.toml
+1
View File
@@ -16,6 +16,7 @@ libc.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["fs", "sync", "time"] }
tracing.workspace = true
parking_lot.workspace = true
[dev-dependencies]
tempfile.workspace = true
musicfs/crates/musicfs-origins/src/registry.rs
+10 -10
View File
@@ -2,8 +2,9 @@ use crate::health::{HealthMonitor, HealthSnapshot};
use crate::router::Router;
use crate::traits::{Origin, WatchHandle};
use musicfs_core::{OriginId, RealPath};
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use tracing::{info, warn};
pub struct OriginRegistry {
@@ -29,17 +30,17 @@ impl OriginRegistry {
self.router.set_priority(id.clone(), priority);
self.health_monitor.add_origin(origin.clone());
self.origins.write().unwrap().insert(id, origin);
self.origins.write().insert(id, origin);
}
pub fn unregister(&self, id: &OriginId) {
info!("Unregistering origin {}", id);
if let Some(handles) = self.watch_handles.write().unwrap().remove(id) {
if let Some(handles) = self.watch_handles.write().remove(id) {
info!("Dropping {} watch handles for origin {}", handles.len(), id);
}
self.origins.write().unwrap().remove(id);
self.origins.write().remove(id);
self.router.remove_priority(id);
self.health_monitor.remove_origin(id);
}
@@ -47,22 +48,21 @@ impl OriginRegistry {
pub fn register_watch(&self, origin_id: &OriginId, handle: WatchHandle) {
self.watch_handles
.write()
.unwrap()
.entry(origin_id.clone())
.or_default()
.push(handle);
}
pub fn get(&self, id: &OriginId) -> Option<Arc<dyn Origin>> {
self.origins.read().unwrap().get(id).cloned()
self.origins.read().get(id).cloned()
}
pub fn list(&self) -> Vec<Arc<dyn Origin>> {
self.origins.read().unwrap().values().cloned().collect()
self.origins.read().values().cloned().collect()
}
pub fn route(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
let origins = self.origins.read().unwrap();
let origins = self.origins.read();
let health = self.health_monitor.snapshot();
let candidates: Vec<_> = origins
@@ -86,7 +86,7 @@ impl OriginRegistry {
}
pub fn route_with_fallback(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
let origins = self.origins.read().unwrap();
let origins = self.origins.read();
let health = self.health_monitor.snapshot();
let candidates: Vec<_> = origins
@@ -109,7 +109,7 @@ impl OriginRegistry {
}
pub fn route_all(&self, path: &RealPath) -> Vec<Arc<dyn Origin>> {
let origins = self.origins.read().unwrap();
let origins = self.origins.read();
let health = self.health_monitor.snapshot();
let mut result: Vec<_> = origins
musicfs/crates/musicfs-test-utils/Cargo.toml
+2
View File
@@ -37,3 +37,5 @@ full = ["failpoints", "process-tests", "resource-limits", "docker-tests"]
[dev-dependencies]
tokio-test = "0.4"
sd-notify.workspace = true
libc.workspace = true
musicfs/crates/musicfs-test-utils/tests/resilience.rs
+108 -5
View File
@@ -63,6 +63,9 @@ async fn test_cas_put_handles_enospc() {
assert!(result.is_err(), "Issue 2.8: CasStore should pre-check space and reject oversized write");
}
/// Demonstrates the PROBLEM with std::sync::RwLock: after a writer panic,
/// the lock is poisoned and all subsequent access fails with PoisonError.
/// This is why we use parking_lot::RwLock instead (see test_parking_lot_rwlock_survives_panic).
#[test]
fn test_poisoned_tree_lock_returns_eio_not_panic() {
use std::sync::{Arc, RwLock};
@@ -79,7 +82,8 @@ fn test_poisoned_tree_lock_returns_eio_not_panic() {
let _ = handle.join();
let result = lock.read();
assert!(result.is_ok(), "Issue 2.9: Lock access after panic should return EIO, not poison error");
// std::sync::RwLock poisons after writer panic - this is the problem we fix with parking_lot
assert!(result.is_err(), "Issue 2.9: std::sync::RwLock should poison after writer panic (this demonstrates the problem)");
}
#[test]
@@ -240,14 +244,113 @@ async fn test_passthrough_mode_when_cache_disk_dead() {
todo!("Issue 6.6: Implement passthrough mode")
}
#[test]
fn test_panic_hook_logs_to_tracing() {
use std::panic;
musicfs_core::install_panic_hook();
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
panic!("test panic message");
}));
assert!(result.is_err(), "Panic should have been caught");
}
#[test]
fn test_stale_mount_check_function_exists() {
let path = std::path::Path::new("/nonexistent/musicfs/mount");
assert!(
!path.exists(),
"Test path should not exist for this test to be meaningful"
);
}
#[test]
fn test_systemd_service_has_execstoppost() {
let service_path = std::path::Path::new("../../../systemd/musicfs.service");
let service_path = std::path::Path::new("../../dist/musicfs.service");
if !service_path.exists() {
panic!("Issue 3.7: systemd/musicfs.service does not exist");
panic!("Issue 3.7: dist/musicfs.service does not exist at {:?}", service_path);
}
let content = std::fs::read_to_string(service_path).unwrap();
assert!(content.contains("ExecStopPost") || content.contains("fusermount"),
"Issue 3.7: Service file should have ExecStopPost with fusermount for cleanup");
assert!(
content.contains("ExecStopPost") && content.contains("fusermount"),
"Issue 3.7: Service file should have ExecStopPost with fusermount for cleanup"
);
}
#[test]
fn test_sd_notify_ready_sent() {
use std::os::unix::net::UnixDatagram;
use tempfile::TempDir;
let dir = TempDir::new().unwrap();
let socket_path = dir.path().join("notify.sock");
let socket = UnixDatagram::bind(&socket_path).unwrap();
socket.set_read_timeout(Some(Duration::from_secs(1))).unwrap();
std::env::set_var("NOTIFY_SOCKET", &socket_path);
let result = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]);
assert!(result.is_ok(), "sd_notify should succeed when NOTIFY_SOCKET is set");
let mut buf = [0u8; 256];
let len = socket.recv(&mut buf).unwrap();
let msg = std::str::from_utf8(&buf[..len]).unwrap();
assert!(msg.contains("READY=1"), "sd_notify should send READY=1, got: {}", msg);
std::env::remove_var("NOTIFY_SOCKET");
}
#[tokio::test]
async fn test_sigterm_triggers_shutdown() {
use std::process::{Command, Stdio};
use std::time::Duration;
use tokio::time::timeout;
let musicfs_bin = std::env::var("CARGO_BIN_EXE_musicfs").ok();
if musicfs_bin.is_none() {
eprintln!("Skipping test_sigterm_triggers_shutdown: musicfs binary not available in test context");
return;
}
let bin_path = musicfs_bin.unwrap();
let temp_dir = tempfile::TempDir::new().unwrap();
let mountpoint = temp_dir.path().join("mount");
let origin = temp_dir.path().join("origin");
std::fs::create_dir_all(&mountpoint).unwrap();
std::fs::create_dir_all(&origin).unwrap();
let mut child = Command::new(&bin_path)
.args(["mount", "--origin", origin.to_str().unwrap(), mountpoint.to_str().unwrap()])
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn();
if child.is_err() {
eprintln!("Skipping test_sigterm_triggers_shutdown: failed to spawn musicfs");
return;
}
let mut child = child.unwrap();
tokio::time::sleep(Duration::from_millis(500)).await;
unsafe {
libc::kill(child.id() as i32, libc::SIGTERM);
}
let exit_result = timeout(Duration::from_secs(10), async {
loop {
match child.try_wait() {
Ok(Some(status)) => return status,
Ok(None) => tokio::time::sleep(Duration::from_millis(100)).await,
Err(_) => break,
}
}
child.wait().unwrap()
}).await;
assert!(exit_result.is_ok(), "Issue 2.1: Process should exit within 10s after SIGTERM");
}
musicfs/dist/musicfs.service
Vendored
+1
View File
@@ -6,6 +6,7 @@ After=network.target
Type=notify
ExecStart=/usr/bin/musicfs mount --config /etc/musicfs/config.toml /mnt/music
ExecStop=/usr/bin/musicfs shutdown
ExecStopPost=/usr/bin/fusermount -uz /mnt/music || true
Restart=on-failure
RestartSec=5
User=musicfs