Implement Phase A: Stop Dying resilience fixes

Implements all 6 critical resilience fixes from phase-a-stop-dying.md:

- Issue 2.9: Migrate std::sync::RwLock → parking_lot::RwLock (7 files)
  Prevents lock poisoning cascade on writer panic

- Issue 2.2: Add install_panic_hook() to log panics via tracing
  Ensures panics are captured in logs/journald before process death

- Issue 3.7: Add ExecStopPost to systemd service
  Cleans up stale FUSE mounts on service stop

- Issue 2.7: Add check_stale_mount() detection on startup
  Auto-cleans leftover mounts from previous crashes

- Issue 2.10: Integrate sd_notify for systemd lifecycle
  Sends READY=1 after mount, STOPPING on shutdown

- Issue 2.1: Add signal handling with spawn_mount
  Catches SIGTERM/SIGINT for clean shutdown instead of instant death

All 7 Phase A tests pass:
- test_poisoned_tree_lock_returns_eio_not_panic
- test_parking_lot_rwlock_survives_panic
- test_panic_hook_logs_to_tracing
- test_systemd_service_has_execstoppost
- test_stale_mount_check_function_exists
- test_sd_notify_ready_sent
- test_sigterm_triggers_shutdown
This commit is contained in:
Alexander
2026-05-13 14:48:32 +02:00
parent 24086cc744
commit 6285eeb6c0
18 changed files with 301 additions and 63 deletions
+2
View File
@@ -22,6 +22,8 @@ tracing-subscriber.workspace = true
tracing-appender.workspace = true
anyhow.workspace = true
dirs.workspace = true
parking_lot.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
tracing-journald.workspace = true
sd-notify.workspace = true
+65 -5
View File
@@ -6,10 +6,11 @@ use musicfs_core::{FileId, FileMeta, LoggingConfig, OriginId, RealPath, VirtualP
use musicfs_fuse::MusicFs;
use musicfs_metadata::MetadataParser;
use musicfs_origins::{LocalOrigin, Origin};
use parking_lot::RwLock;
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use std::time::SystemTime;
use tracing::{debug, info};
use tracing::{debug, info, warn};
use tracing_appender::non_blocking::WorkerGuard;
use tracing_subscriber::{fmt, prelude::*, EnvFilter, Layer};
@@ -87,6 +88,7 @@ enum OriginCommands {
}
fn main() -> Result<()> {
musicfs_core::install_panic_hook();
let cli = Cli::parse();
match cli.command {
@@ -188,14 +190,50 @@ fn run_mount(
Ok::<_, anyhow::Error>((tree, reader))
})?;
let fs = MusicFs::with_reader(tree, reader, handle);
check_stale_mount(&mountpoint)?;
let fs = MusicFs::with_reader(tree, reader, handle.clone());
info!("Mounting filesystem at {:?}", mountpoint);
info!("Press Ctrl+C to unmount");
fs.mount(&mountpoint)
let session = fs
.spawn_mount(&mountpoint)
.context("Failed to mount filesystem")?;
#[cfg(target_os = "linux")]
{
if let Err(e) = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) {
debug!("sd_notify not available (not running under systemd): {}", e);
}
}
info!("MusicFS ready, PID {}", std::process::id());
runtime.block_on(async {
let mut sigterm =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
let mut sigint =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
tokio::select! {
_ = sigterm.recv() => {
info!("Received SIGTERM, shutting down");
}
_ = sigint.recv() => {
info!("Received SIGINT, shutting down");
}
}
Ok::<_, anyhow::Error>(())
})?;
#[cfg(target_os = "linux")]
{
let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]);
}
info!("Unmounting filesystem");
drop(session);
info!("Shutdown complete");
Ok(())
}
@@ -437,3 +475,25 @@ fn sanitize(s: &str) -> String {
})
.collect()
}
fn check_stale_mount(mountpoint: &Path) -> Result<()> {
if let Ok(mounts) = std::fs::read_to_string("/proc/mounts") {
for line in mounts.lines() {
if line.contains(mountpoint.to_string_lossy().as_ref()) && line.contains("fuse") {
warn!(
"Stale FUSE mount detected at {:?}, attempting cleanup",
mountpoint
);
let status = std::process::Command::new("fusermount")
.args(["-uz", &mountpoint.to_string_lossy()])
.status();
match status {
Ok(s) if s.success() => info!("Stale mount cleaned up"),
Ok(s) => warn!("fusermount exited with: {}", s),
Err(e) => warn!("Failed to run fusermount: {}", e),
}
}
}
}
Ok(())
}