Add comprehensive logging with tracing, file rotation, and systemd integration

- Add tracing-appender and tracing-journald for production logging
- Add LoggingConfig with trace_sample_rate, json_output, journald options
- Expand init_logging() with file rotation, journald, and stderr layers
- Add sanitize_path() helper for PII protection in logs
- Instrument FUSE operations with #[instrument] and trace decision points
- Instrument gRPC handlers (10 methods) with span correlation
- Add spawn instrumentation for health monitor, indexer, watcher tasks
- Add broadcast lag handling (RecvError::Lagged) in event subscribers
- Fix webhook.rs expect() calls with proper error handling
- Add logging to patterns.rs, collections.rs, artwork.rs database ops
- Add Drop impl logging for PluginManager and WatchHandle
- Update systemd service with rate limiting and journal output
- Add logrotate config and example config.toml with logging section

musicfs/crates/musicfs-search/src/collections.rs

@@ -2,7 +2,7 @@ use parking_lot::Mutex;
 use serde::{Deserialize, Serialize};
 use std::path::Path;
 use std::time::{Duration, SystemTime};
-use tracing::warn;
+use tracing::{debug, info, warn};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SmartCollection {
@@ -103,6 +103,7 @@ impl CollectionStore {
             [],
         )?;
 
+        info!(path = ?db_path, "Collection store opened");
         Ok(Self { db: Mutex::new(db) })
     }
 
@@ -111,6 +112,7 @@ impl CollectionStore {
         name: &str,
         query: CollectionQuery,
     ) -> Result<SmartCollection, CollectionError> {
+        info!(name = %name, "Creating collection");
         let query_json = serde_json::to_string(&query)?;
         let now = SystemTime::now()
             .duration_since(SystemTime::UNIX_EPOCH)
@@ -124,6 +126,7 @@ impl CollectionStore {
         )?;
 
         let id = db.last_insert_rowid();
+        debug!(id = id, name = %name, "Collection created");
 
         Ok(SmartCollection {
             id,
@@ -199,6 +202,7 @@ impl CollectionStore {
     }
 
     pub fn delete(&self, name: &str) -> Result<(), CollectionError> {
+        info!(name = %name, "Deleting collection");
         let db = self.db.lock();
         db.execute("DELETE FROM collections WHERE name = ?1", [name])?;
         Ok(())

musicfs/crates/musicfs-search/src/indexer.rs

@@ -2,7 +2,7 @@ use crate::index::{SearchError, SearchIndex};
 use musicfs_core::{Event, EventBus, FileMeta};
 use std::sync::Arc;
 use tokio::sync::mpsc;
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info, info_span, warn, Instrument};
 
 pub trait MetadataLookup: Send + Sync {
     fn lookup(&self, path: &musicfs_core::VirtualPath) -> Option<FileMeta>;
@@ -31,43 +31,52 @@ impl<M: MetadataLookup + 'static> Indexer<M> {
         let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
         let mut event_rx = self.event_bus.subscribe();
 
-        tokio::spawn(async move {
-            let mut pending_commit = false;
-            let mut commit_timer = tokio::time::interval(std::time::Duration::from_secs(5));
+        info!("Search indexer starting");
 
-            loop {
-                tokio::select! {
-                    result = event_rx.recv() => {
-                        match result {
-                            Ok(event) => {
-                                if let Err(e) = self.handle_event(&event) {
-                                    error!("Indexer error: {}", e);
+        tokio::spawn(
+            async move {
+                let mut pending_commit = false;
+                let mut commit_timer = tokio::time::interval(std::time::Duration::from_secs(5));
+
+                loop {
+                    tokio::select! {
+                        result = event_rx.recv() => {
+                            match result {
+                                Ok(event) => {
+                                    if let Err(e) = self.handle_event(&event) {
+                                        error!("Indexer error: {}", e);
+                                    }
+                                    pending_commit = true;
+                                }
+                                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
+                                    warn!(skipped = n, "Indexer lagged, skipped events");
+                                }
+                                Err(tokio::sync::broadcast::error::RecvError::Closed) => {
+                                    debug!("Event channel closed");
+                                    break;
                                 }
-                                pending_commit = true;
-                            }
-                            Err(e) => {
-                                warn!("Event receive error: {}", e);
                             }
                         }
-                    }
-                    _ = commit_timer.tick() => {
-                        if pending_commit {
-                            if let Err(e) = self.index.commit() {
-                                error!("Index commit error: {}", e);
+                        _ = commit_timer.tick() => {
+                            if pending_commit {
+                                if let Err(e) = self.index.commit() {
+                                    error!("Index commit error: {}", e);
+                                }
+                                pending_commit = false;
                             }
-                            pending_commit = false;
                         }
-                    }
-                    _ = stop_rx.recv() => {
-                        info!("Indexer stopping");
-                        if pending_commit {
-                            let _ = self.index.commit();
+                        _ = stop_rx.recv() => {
+                            info!("Indexer stopping");
+                            if pending_commit {
+                                let _ = self.index.commit();
+                            }
+                            break;
                         }
-                        break;
                     }
                 }
             }
-        });
+            .instrument(info_span!("search_indexer")),
+        );
 
         IndexerHandle { stop_tx }
     }