Implement Phase A: Stop Dying resilience fixes
Implements all 6 critical resilience fixes from phase-a-stop-dying.md: - Issue 2.9: Migrate std::sync::RwLock → parking_lot::RwLock (7 files) Prevents lock poisoning cascade on writer panic - Issue 2.2: Add install_panic_hook() to log panics via tracing Ensures panics are captured in logs/journald before process death - Issue 3.7: Add ExecStopPost to systemd service Cleans up stale FUSE mounts on service stop - Issue 2.7: Add check_stale_mount() detection on startup Auto-cleans leftover mounts from previous crashes - Issue 2.10: Integrate sd_notify for systemd lifecycle Sends READY=1 after mount, STOPPING on shutdown - Issue 2.1: Add signal handling with spawn_mount Catches SIGTERM/SIGINT for clean shutdown instead of instant death All 7 Phase A tests pass: - test_poisoned_tree_lock_returns_eio_not_panic - test_parking_lot_rwlock_survives_panic - test_panic_hook_logs_to_tracing - test_systemd_service_has_execstoppost - test_stale_mount_check_function_exists - test_sd_notify_ready_sent - test_sigterm_triggers_shutdown
This commit is contained in:
@@ -22,6 +22,8 @@ tracing-subscriber.workspace = true
|
||||
tracing-appender.workspace = true
|
||||
anyhow.workspace = true
|
||||
dirs.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
tracing-journald.workspace = true
|
||||
sd-notify.workspace = true
|
||||
|
||||
@@ -6,10 +6,11 @@ use musicfs_core::{FileId, FileMeta, LoggingConfig, OriginId, RealPath, VirtualP
|
||||
use musicfs_fuse::MusicFs;
|
||||
use musicfs_metadata::MetadataParser;
|
||||
use musicfs_origins::{LocalOrigin, Origin};
|
||||
use parking_lot::RwLock;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing_appender::non_blocking::WorkerGuard;
|
||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter, Layer};
|
||||
|
||||
@@ -87,6 +88,7 @@ enum OriginCommands {
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
musicfs_core::install_panic_hook();
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.command {
|
||||
@@ -188,14 +190,50 @@ fn run_mount(
|
||||
Ok::<_, anyhow::Error>((tree, reader))
|
||||
})?;
|
||||
|
||||
let fs = MusicFs::with_reader(tree, reader, handle);
|
||||
check_stale_mount(&mountpoint)?;
|
||||
|
||||
let fs = MusicFs::with_reader(tree, reader, handle.clone());
|
||||
|
||||
info!("Mounting filesystem at {:?}", mountpoint);
|
||||
info!("Press Ctrl+C to unmount");
|
||||
|
||||
fs.mount(&mountpoint)
|
||||
let session = fs
|
||||
.spawn_mount(&mountpoint)
|
||||
.context("Failed to mount filesystem")?;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if let Err(e) = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) {
|
||||
debug!("sd_notify not available (not running under systemd): {}", e);
|
||||
}
|
||||
}
|
||||
info!("MusicFS ready, PID {}", std::process::id());
|
||||
|
||||
runtime.block_on(async {
|
||||
let mut sigterm =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
|
||||
let mut sigint =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
|
||||
|
||||
tokio::select! {
|
||||
_ = sigterm.recv() => {
|
||||
info!("Received SIGTERM, shutting down");
|
||||
}
|
||||
_ = sigint.recv() => {
|
||||
info!("Received SIGINT, shutting down");
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, anyhow::Error>(())
|
||||
})?;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]);
|
||||
}
|
||||
info!("Unmounting filesystem");
|
||||
drop(session);
|
||||
info!("Shutdown complete");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -437,3 +475,25 @@ fn sanitize(s: &str) -> String {
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn check_stale_mount(mountpoint: &Path) -> Result<()> {
|
||||
if let Ok(mounts) = std::fs::read_to_string("/proc/mounts") {
|
||||
for line in mounts.lines() {
|
||||
if line.contains(mountpoint.to_string_lossy().as_ref()) && line.contains("fuse") {
|
||||
warn!(
|
||||
"Stale FUSE mount detected at {:?}, attempting cleanup",
|
||||
mountpoint
|
||||
);
|
||||
let status = std::process::Command::new("fusermount")
|
||||
.args(["-uz", &mountpoint.to_string_lossy()])
|
||||
.status();
|
||||
match status {
|
||||
Ok(s) if s.success() => info!("Stale mount cleaned up"),
|
||||
Ok(s) => warn!("fusermount exited with: {}", s),
|
||||
Err(e) => warn!("Failed to run fusermount: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user