Add comprehensive logging with tracing, file rotation, and systemd integration

- Add tracing-appender and tracing-journald for production logging
- Add LoggingConfig with trace_sample_rate, json_output, journald options
- Expand init_logging() with file rotation, journald, and stderr layers
- Add sanitize_path() helper for PII protection in logs
- Instrument FUSE operations with #[instrument] and trace decision points
- Instrument gRPC handlers (10 methods) with span correlation
- Add spawn instrumentation for health monitor, indexer, watcher tasks
- Add broadcast lag handling (RecvError::Lagged) in event subscribers
- Fix webhook.rs expect() calls with proper error handling
- Add logging to patterns.rs, collections.rs, artwork.rs database ops
- Add Drop impl logging for PluginManager and WatchHandle
- Update systemd service with rate limiting and journal output
- Add logrotate config and example config.toml with logging section

musicfs/crates/musicfs-core/Cargo.toml

@@ -9,6 +9,7 @@ serde.workspace = true
 serde_json.workspace = true
 toml.workspace = true
 tokio = { workspace = true, features = ["sync"] }
+tracing.workspace = true
 xxhash-rust.workspace = true
 hex.workspace = true
 

musicfs/crates/musicfs-core/src/config.rs

@@ -14,6 +14,9 @@ pub struct Config {
 
     #[serde(default)]
     pub health: HealthConfig,
+
+    #[serde(default)]
+    pub logging: LoggingConfig,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -120,6 +123,52 @@ fn default_unhealthy_threshold() -> u32 {
     3
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LoggingConfig {
+    #[serde(default = "default_log_dir")]
+    pub log_dir: PathBuf,
+
+    #[serde(default)]
+    pub json_output: bool,
+
+    #[serde(default = "default_true")]
+    pub journald: bool,
+
+    #[serde(default = "default_log_level")]
+    pub level: String,
+
+    #[serde(default = "default_sample_rate")]
+    pub trace_sample_rate: f32,
+}
+
+impl Default for LoggingConfig {
+    fn default() -> Self {
+        Self {
+            log_dir: default_log_dir(),
+            json_output: false,
+            journald: true,
+            level: default_log_level(),
+            trace_sample_rate: default_sample_rate(),
+        }
+    }
+}
+
+fn default_log_dir() -> PathBuf {
+    PathBuf::from("/var/log/musicfs")
+}
+
+fn default_log_level() -> String {
+    "musicfs=info,warn".to_string()
+}
+
+fn default_true() -> bool {
+    true
+}
+
+fn default_sample_rate() -> f32 {
+    1.0
+}
+
 impl Config {
     pub fn from_file(path: &std::path::Path) -> Result<Self, ConfigError> {
         let content =

musicfs/crates/musicfs-core/src/credentials.rs

@@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::path::PathBuf;
 use thiserror::Error;
+use tracing::{debug, info, trace, warn};
 
 #[derive(Clone)]
 pub struct CredentialStore {
@@ -106,16 +107,36 @@ impl CredentialStore {
         origin_id: &str,
         config: &CredentialConfig,
     ) -> Result<Credential, CredentialError> {
+        debug!(origin_id = %origin_id, "Loading credentials");
+
         if let Some(cred) = self.cache.get(origin_id) {
+            trace!(origin_id = %origin_id, "Credential cache hit");
             return Ok(cred.clone());
         }
 
         let cred = match config {
-            CredentialConfig::Environment { prefix } => self.load_from_env(prefix)?,
-            CredentialConfig::File { path } => self.load_from_file(path)?,
-            CredentialConfig::Inline(cred) => cred.clone(),
+            CredentialConfig::Environment { prefix } => {
+                trace!(origin_id = %origin_id, prefix = %prefix, "Loading from environment");
+                self.load_from_env(prefix)?
+            }
+            CredentialConfig::File { path } => {
+                trace!(origin_id = %origin_id, path = ?path, "Loading from file");
+                self.load_from_file(path)?
+            }
+            CredentialConfig::Inline(cred) => {
+                trace!(origin_id = %origin_id, "Using inline credential");
+                cred.clone()
+            }
         };
 
+        let cred_type = match &cred {
+            Credential::Basic { .. } => "Basic",
+            Credential::AwsKey { .. } => "AwsKey",
+            Credential::SshKey { .. } => "SshKey",
+            Credential::EnvVar { .. } => "EnvVar",
+        };
+        info!(origin_id = %origin_id, cred_type = %cred_type, "Credential loaded");
+
         self.cache.insert(origin_id.to_string(), cred.clone());
         Ok(cred)
     }
@@ -144,6 +165,7 @@ impl CredentialStore {
             });
         }
 
+        warn!(prefix = %prefix, "No credentials found in environment");
         Err(CredentialError::NotFound(format!(
             "No credentials found with prefix {}",
             prefix

musicfs/crates/musicfs-core/src/events.rs

@@ -1,5 +1,6 @@
 use crate::types::{FileId, OriginId, VirtualPath};
 use tokio::sync::broadcast;
+use tracing::{debug, trace};
 
 pub struct EventBus {
     sender: broadcast::Sender<Event>,
@@ -12,7 +13,11 @@ impl EventBus {
     }
 
     pub fn publish(&self, event: Event) {
-        let _ = self.sender.send(event);
+        trace!(event = ?event, "Publishing event");
+        let receiver_count = self.sender.receiver_count();
+        if self.sender.send(event).is_err() && receiver_count > 0 {
+            debug!(receiver_count = receiver_count, "Event dropped, no active receivers");
+        }
     }
 
     pub fn subscribe(&self) -> broadcast::Receiver<Event> {

musicfs/crates/musicfs-core/src/lib.rs

@@ -6,7 +6,19 @@ pub mod metrics;
 pub mod resolver;
 pub mod types;
 
-pub use config::{CacheConfig, Config, ConfigError, HealthConfig, OriginConfig, OriginType};
+pub use config::{
+    CacheConfig, Config, ConfigError, HealthConfig, LoggingConfig, OriginConfig, OriginType,
+};
+
+use std::path::Path;
+
+pub fn sanitize_path(path: &Path) -> String {
+    if let Ok(home) = std::env::var("HOME") {
+        path.to_string_lossy().replace(&home, "~")
+    } else {
+        path.to_string_lossy().to_string()
+    }
+}
 pub use credentials::{Credential, CredentialConfig, CredentialError, CredentialStore};
 pub use error::{Error, Result};
 pub use events::{Event, EventBus};