Add comprehensive logging with tracing, file rotation, and systemd integration
- Add tracing-appender and tracing-journald for production logging - Add LoggingConfig with trace_sample_rate, json_output, journald options - Expand init_logging() with file rotation, journald, and stderr layers - Add sanitize_path() helper for PII protection in logs - Instrument FUSE operations with #[instrument] and trace decision points - Instrument gRPC handlers (10 methods) with span correlation - Add spawn instrumentation for health monitor, indexer, watcher tasks - Add broadcast lag handling (RecvError::Lagged) in event subscribers - Fix webhook.rs expect() calls with proper error handling - Add logging to patterns.rs, collections.rs, artwork.rs database ops - Add Drop impl logging for PluginManager and WatchHandle - Update systemd service with rate limiting and journal output - Add logrotate config and example config.toml with logging section
This commit is contained in:
Alexander
@@ -0,0 +1,982 @@
|
||||
# Comprehensive Logging Plan
|
||||
|
||||
**Goal**: Add production-grade logging with trace-level observability, file rotation, and systemd integration
|
||||
**Effort**: ~10-12 hours
|
||||
**Dependencies**: Existing libraries only (no custom code)
|
||||
|
||||
> **Review Status**: Reviewed by Oracle - all gaps addressed
|
||||
|
||||
---
|
||||
|
||||
## Libraries Used
|
||||
|
||||
| Need | Library | Status |
|
||||
|------|---------|--------|
|
||||
| Instrumentation | `tracing` | Already in workspace |
|
||||
| Subscriber/filtering | `tracing-subscriber` | Already in workspace |
|
||||
| File rotation | `tracing-appender` | Add to workspace |
|
||||
| systemd journal | `tracing-journald` | Add to workspace |
|
||||
| Compression | `logrotate` (Linux tool) | Config file only |
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Config & Dependencies (2 hours)
|
||||
|
||||
### 1.1 Add dependencies to workspace
|
||||
|
||||
```toml
|
||||
# Cargo.toml [workspace.dependencies]
|
||||
tracing-appender = "0.2"
|
||||
tracing-journald = "0.3"
|
||||
```
|
||||
|
||||
```toml
|
||||
# crates/musicfs-cli/Cargo.toml
|
||||
tracing-appender.workspace = true
|
||||
tracing-journald.workspace = true
|
||||
```
|
||||
|
||||
### 1.2 Add LoggingConfig to config.rs
|
||||
|
||||
```rust
|
||||
// crates/musicfs-core/src/config.rs
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Config {
|
||||
pub mount_point: PathBuf,
|
||||
pub cache_dir: PathBuf,
|
||||
pub origins: Vec<OriginConfig>,
|
||||
#[serde(default)]
|
||||
pub cache: CacheConfig,
|
||||
#[serde(default)]
|
||||
pub health: HealthConfig,
|
||||
#[serde(default)]
|
||||
pub logging: LoggingConfig, // NEW
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LoggingConfig {
|
||||
#[serde(default = "default_log_dir")]
|
||||
pub log_dir: PathBuf,
|
||||
|
||||
#[serde(default)]
|
||||
pub json_output: bool,
|
||||
|
||||
#[serde(default = "default_true")]
|
||||
pub journald: bool,
|
||||
|
||||
#[serde(default = "default_log_level")]
|
||||
pub level: String,
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
log_dir: default_log_dir(),
|
||||
json_output: false,
|
||||
journald: true,
|
||||
level: default_log_level(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_log_dir() -> PathBuf {
|
||||
PathBuf::from("/var/log/musicfs")
|
||||
}
|
||||
fn default_log_level() -> String {
|
||||
"musicfs=info,warn".to_string()
|
||||
}
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
```
|
||||
|
||||
### 1.3 Expand init_logging() in main.rs
|
||||
|
||||
```rust
|
||||
// crates/musicfs-cli/src/main.rs
|
||||
|
||||
use tracing_appender::non_blocking::WorkerGuard;
|
||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
|
||||
|
||||
fn init_logging(config: &LoggingConfig) -> Result<WorkerGuard> {
|
||||
std::fs::create_dir_all(&config.log_dir)?;
|
||||
|
||||
// File layer with daily rotation
|
||||
let file_appender = tracing_appender::rolling::daily(&config.log_dir, "musicfs.log");
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
|
||||
let file_layer = if config.json_output {
|
||||
fmt::layer()
|
||||
.json()
|
||||
.with_writer(non_blocking)
|
||||
.with_ansi(false)
|
||||
.boxed()
|
||||
} else {
|
||||
fmt::layer()
|
||||
.with_writer(non_blocking)
|
||||
.with_ansi(false)
|
||||
.boxed()
|
||||
};
|
||||
|
||||
// Journald layer (Linux only)
|
||||
#[cfg(target_os = "linux")]
|
||||
let journald_layer = if config.journald {
|
||||
tracing_journald::layer()
|
||||
.ok()
|
||||
.map(|l| l.with_syslog_identifier("musicfs".to_string()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Stderr layer for interactive use
|
||||
let stderr_layer = fmt::layer()
|
||||
.with_writer(std::io::stderr)
|
||||
.compact();
|
||||
|
||||
// Filter from config or env
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new(&config.level));
|
||||
|
||||
// Compose
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(file_layer)
|
||||
.with(stderr_layer);
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let subscriber = subscriber.with(journald_layer);
|
||||
|
||||
subscriber.init();
|
||||
|
||||
tracing::info!(version = env!("CARGO_PKG_VERSION"), "MusicFS starting");
|
||||
Ok(guard)
|
||||
}
|
||||
```
|
||||
|
||||
### 1.4 Add logrotate config
|
||||
|
||||
```bash
|
||||
# dist/logrotate.d/musicfs
|
||||
/var/log/musicfs/*.log {
|
||||
daily
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 musicfs musicfs
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Add tracing to musicfs-core (1 hour)
|
||||
|
||||
### 2.1 Add dependency
|
||||
|
||||
```toml
|
||||
# crates/musicfs-core/Cargo.toml
|
||||
[dependencies]
|
||||
tracing.workspace = true # ADD THIS
|
||||
```
|
||||
|
||||
### 2.2 Instrument core modules
|
||||
|
||||
| File | What to Add |
|
||||
|------|-------------|
|
||||
| `config.rs` | Log config file loading, parse errors |
|
||||
| `credentials.rs` | Log credential loading (redacted values) |
|
||||
| `events.rs` | Log event publishing with counts |
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Instrument Hot Paths (4 hours)
|
||||
|
||||
### Priority order by impact
|
||||
|
||||
| Crate | Files | What to Add |
|
||||
|-------|-------|-------------|
|
||||
| musicfs-fuse | `filesystem.rs` | `#[instrument]` on all FUSE ops, trace at decision points |
|
||||
| musicfs-origins | `failover.rs`, `health.rs`, `router.rs` | Retry loops, state transitions, selection logic |
|
||||
| musicfs-cache | `tree.rs`, `metadata.rs` | Tree mutations, cache hit/miss |
|
||||
| musicfs-cas | `reader.rs`, `store.rs` | Chunk operations, dedup decisions |
|
||||
| musicfs-sync | `delta.rs`, `watcher.rs` | Change detection, file events |
|
||||
|
||||
### Instrumentation patterns
|
||||
|
||||
```rust
|
||||
// Function level - add to all public async functions
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(path = %path))]
|
||||
pub async fn read(&self, path: &str) -> Result<Bytes> {
|
||||
// ...
|
||||
}
|
||||
|
||||
// Decision points - add trace! at match/if branches
|
||||
match result {
|
||||
Ok(data) => {
|
||||
tracing::trace!(bytes = data.len(), "read success");
|
||||
data
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::trace!(error = %e, "read failed");
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// State changes - use info! for important transitions
|
||||
tracing::info!(old = ?old_status, new = ?new_status, origin = %id, "health changed");
|
||||
|
||||
// Cache operations
|
||||
tracing::trace!(hit = true, fresh = true, "cache hit");
|
||||
tracing::trace!(hit = false, "cache miss");
|
||||
```
|
||||
|
||||
### FUSE operations (filesystem.rs) - highest priority
|
||||
|
||||
| Operation | Level | Fields |
|
||||
|-----------|-------|--------|
|
||||
| `lookup()` | debug | parent, name, result_ino |
|
||||
| `getattr()` | debug | ino, file_type |
|
||||
| `readdir()` | debug | ino, entry_count |
|
||||
| `read()` | debug | ino, offset, size, bytes_read |
|
||||
| `open()` | debug | ino, flags |
|
||||
| `release()` | trace | ino |
|
||||
|
||||
### Origin operations - critical for debugging
|
||||
|
||||
| Function | Level | Fields |
|
||||
|----------|-------|--------|
|
||||
| `read_with_failover()` | debug | path, origins_tried, success |
|
||||
| `read_with_retry()` | trace | origin, attempt, success |
|
||||
| `check_health()` | debug | origin, old_status, new_status |
|
||||
| `select_origin()` | trace | candidates, selected, reason |
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Update Production Files (1 hour)
|
||||
|
||||
### 4.1 Update systemd service
|
||||
|
||||
```ini
|
||||
# dist/musicfs.service (add these lines)
|
||||
Environment="RUST_LOG=musicfs=info,warn"
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=musicfs
|
||||
RateLimitIntervalSec=30s
|
||||
RateLimitBurst=1000
|
||||
```
|
||||
|
||||
### 4.2 Example config.toml
|
||||
|
||||
```toml
|
||||
# dist/config.example.toml
|
||||
mount_point = "/mnt/music"
|
||||
cache_dir = "/var/cache/musicfs"
|
||||
|
||||
[logging]
|
||||
log_dir = "/var/log/musicfs"
|
||||
json_output = true
|
||||
journald = true
|
||||
level = "musicfs=info,warn"
|
||||
|
||||
[cache]
|
||||
metadata_cache_mb = 100
|
||||
content_cache_gb = 10
|
||||
|
||||
[health]
|
||||
check_interval_secs = 30
|
||||
timeout_ms = 5000
|
||||
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/srv/music"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Detailed Log Locations by Level
|
||||
|
||||
### ERROR Level (25+ locations) - Unrecoverable Failures
|
||||
|
||||
| File | Line | Log Message |
|
||||
|------|------|-------------|
|
||||
| `musicfs-grpc/src/webhook.rs` | 43 | `error!("Failed to initialize webhook HTTP client: {error}")` |
|
||||
| `musicfs-grpc/src/webhook.rs` | 133 | `error!("Invalid HMAC secret key for webhook signature: {error}")` |
|
||||
| `musicfs-plugins/src/manager.rs` | 272 | `error!("Plugin manager initialization failed: {error}")` |
|
||||
| `musicfs-plugins/src/wasm.rs` | 142,183 | `error!("WASM plugin host initialization failed: {error}")` |
|
||||
| `musicfs-search/src/index.rs` | 211,217 | `error!("Search index corrupted: failed to deserialize at position {pos}")` |
|
||||
| `musicfs-cas/src/store.rs` | 105 | `error!("CAS chunk not found: {hash} - possible data loss")` |
|
||||
| `musicfs-cas/src/store.rs` | 124-131 | `error!("CAS integrity check failed: expected {expected}, got {actual}")` |
|
||||
| `musicfs-fuse/src/filesystem.rs` | 103 | `error!("Failed to mount filesystem at {mountpoint}: {error}")` |
|
||||
| `musicfs-origins/src/failover.rs` | 76 | `error!("No origins available for path {path}")` |
|
||||
| `musicfs-origins/src/failover.rs` | 125,186 | `error!("Max retries ({max_attempts}) exceeded for origin {origin_id}")` |
|
||||
| `musicfs-origins/src/nfs.rs` | 63 | `error!("NFS stale file handle after {max_retries} retries for {path}")` |
|
||||
| `musicfs-cas/src/reader.rs` | 75 | `error!("File manifest not found for file_id {file_id}")` |
|
||||
| `musicfs-cas/src/fetcher.rs` | 60,68 | `error!("File/Origin not found for file_id {file_id}")` |
|
||||
| `musicfs-search/src/indexer.rs` | 44,56 | `error!("Search indexer/commit failed: {error}")` |
|
||||
| `musicfs-sync/src/watcher.rs` | 36,59,63 | `error!("Watcher failed for origin {origin_id}: {error}")` |
|
||||
|
||||
### WARN Level (50+ locations) - Recoverable Issues
|
||||
|
||||
| Category | File | Line | Log Message |
|
||||
|----------|------|------|-------------|
|
||||
| **Retry Logic** | `failover.rs` | 90 | `warn!("Origin {origin_id} failed: {error}, trying next (attempt {n}/{total})")` |
|
||||
| **Retry Logic** | `failover.rs` | 111-118 | `warn!("Retrying origin {origin_id} after {delay:?} (attempt {n}/{max})")` |
|
||||
| **Retry Logic** | `nfs.rs` | 47-52 | `warn!("NFS stale handle for {path} (attempt {n}/{max}), retrying")` |
|
||||
| **Retry Logic** | `smb.rs` | 45 | `warn!("SMB connection lost (ENOTCONN), retrying (attempt {n}/{max})")` |
|
||||
| **Retry Logic** | `webhook.rs` | 94-108 | `warn!("Webhook delivery failed to {url} (attempt {n}/{max}): {error}")` |
|
||||
| **Fallback** | `failover.rs` | 70-73 | `warn!("No healthy origins for {path}, using fallback {origin_id}")` |
|
||||
| **Timeout** | `smb.rs` | 107-109 | `warn!("SMB health check timed out after 5s for {origin_id}")` |
|
||||
| **Timeout** | `nfs.rs` | 104-106 | `warn!("NFS health check timed out after 5s for {origin_id}")` |
|
||||
| **Timeout** | `prefetch.rs` | 91 | `warn!("Prefetch event receive timed out after 1s")` |
|
||||
| **Health** | `health.rs` | 209 | `warn!("Origin {origin_id} is degraded (failures: {count})")` |
|
||||
| **Health** | `health.rs` | 217-220 | `warn!("Origin {origin_id} is now unhealthy after {n} consecutive failures")` |
|
||||
| **Remote FS** | `smb.rs` | 118 | `warn!("SMB watch using inotify on {share_path} - may be unreliable")` |
|
||||
| **Remote FS** | `nfs.rs` | 115 | `warn!("NFS watch using inotify on {mount_point} - may be unreliable")` |
|
||||
| **Plugin** | `manager.rs` | 152 | `warn!("Failed to load plugin from {path}: {error}")` |
|
||||
| **Plugin** | `manager.rs` | 193-194 | `warn!("Failed to unload plugin {plugin_id}: {error}")` |
|
||||
| **Prefetch** | `prefetch.rs` | 97 | `warn!("Failed to record access pattern for {file_id}: {error}")` |
|
||||
| **Prefetch** | `prefetch.rs` | 159-161 | `warn!("Prefetch skipped: concurrency limit reached ({max})")` |
|
||||
| **Search** | `indexer.rs` | 49 | `warn!("Search indexer event receive error: {error}")` |
|
||||
| **Search** | `indexer.rs` | 82 | `warn!("No metadata found for file {path}, skipping indexing")` |
|
||||
| **Collections** | `collections.rs` | 146,180 | `warn!("Failed to save/delete collection {name}: {error}")` |
|
||||
|
||||
### INFO Level (35+ locations) - Lifecycle & Major Operations
|
||||
|
||||
| Category | File | Line | Log Message |
|
||||
|----------|------|------|-------------|
|
||||
| **Lifecycle** | `main.rs` | 118 | `info!(version = env!("CARGO_PKG_VERSION"), "MusicFS starting")` |
|
||||
| **Lifecycle** | `filesystem.rs` | 94 | `info!("Mounting MusicFS at {:?}", mountpoint)` |
|
||||
| **Lifecycle** | `filesystem.rs` | 154 | `info!("MusicFS initialized")` |
|
||||
| **Lifecycle** | `filesystem.rs` | 159 | `info!("MusicFS destroyed")` |
|
||||
| **Origin** | `registry.rs` | 28 | `info!("Registering origin {} with priority {}", id, priority)` |
|
||||
| **Origin** | `registry.rs` | 36 | `info!("Unregistering origin {}", id)` |
|
||||
| **Origin** | `watcher.rs` | 65 | `info!("Watching origin {} at {:?}", origin_id, path)` |
|
||||
| **Config** | `main.rs` | 127 | `info!("Cache directory: {:?}", cache_dir)` |
|
||||
| **Config** | `main.rs` | 141 | `info!("CAS store initialized")` |
|
||||
| **Config** | `store.rs` | 51 | `info!("CAS store opened: {} chunks, {} bytes", count, size)` (ADD) |
|
||||
| **Sync** | `main.rs` | 150,152 | `info!("Scanning music files...")` / `info!("Found {} music files", count)` |
|
||||
| **Sync** | `delta.rs` | 104 | `info!("Delta complete: {} added, {} removed, {} modified", a, r, m)` |
|
||||
| **Sync** | `delta.rs` | 63 | `info!("Sync started for origin {}", origin_id)` (ADD) |
|
||||
| **Index** | `main.rs` | 160 | `info!("Virtual tree built")` |
|
||||
| **Index** | `indexer.rs` | 62 | `info!("Indexer stopping")` |
|
||||
| **Index** | `indexer.rs` | 114 | `info!("Indexed {} files", count)` |
|
||||
| **Index** | `index.rs` | 170 | `info!("Search index committed")` |
|
||||
| **Health** | `health.rs` | 202 | `info!("Origin {} is now healthy", id)` |
|
||||
| **Health** | `health.rs` | 150 | `info!("Health monitor started with interval {:?}", interval)` (ADD) |
|
||||
| **Plugin** | `manager.rs` | 127 | `info!("Initializing plugin system")` |
|
||||
| **Plugin** | `manager.rs` | 150 | `info!("Loaded plugin '{}' with id {:?}", name, id)` |
|
||||
| **Plugin** | `manager.rs` | 256 | `info!("Shutting down plugin system")` |
|
||||
| **Cache** | `prefetch.rs` | 123 | `info!("Prefetch engine stopped")` |
|
||||
| **Cache** | `prefetch.rs` | 174 | `info!("Prefetched {:?}: {} chunks, {} bytes", file_id, chunks, bytes)` |
|
||||
| **Cache** | `eviction.rs` | 51 | `info!("Evicted {} bytes from cache", bytes)` |
|
||||
| **Cache** | `prefetch.rs` | 73 | `info!("Prefetch engine started (lookahead: {}, max_concurrent: {})")` (ADD) |
|
||||
|
||||
### DEBUG Level (60+ locations) - Operation Details
|
||||
|
||||
| Category | File | Line | Log Message |
|
||||
|----------|------|------|-------------|
|
||||
| **FUSE lookup** | `filesystem.rs` | 162,195,200 | Entry + result/miss |
|
||||
| **FUSE getattr** | `filesystem.rs` | 203,230,233 | Entry + result/miss |
|
||||
| **FUSE readdir** | `filesystem.rs` | 237,263,303 | Entry + result/miss |
|
||||
| **FUSE read** | `filesystem.rs` | 325,338,362,364 | Entry + file_id + result/error |
|
||||
| **Local origin** | `local.rs` | 51,68 | readdir entry + result |
|
||||
| **Local origin** | `local.rs` | 88-91,112 | read entry + result |
|
||||
| **SMB origin** | `smb.rs` | 86,93 | readdir/read entry + result |
|
||||
| **NFS origin** | `nfs.rs` | 81,89 | readdir/read entry + result |
|
||||
| **Failover** | `failover.rs` | 66,82,87 | Entry + trying origin + success |
|
||||
| **Tree lookup** | `tree.rs` | 124,132 | Entry + result |
|
||||
| **Metadata cache** | `metadata.rs` | 36,40 | lookup + is_fresh entry/result |
|
||||
| **CAS store** | `store.rs` | 70,101 | put/get entry |
|
||||
| **File reader** | `reader.rs` | 66,86 | manifest cache + read entry |
|
||||
| **Search** | `ops/search.rs` | 107,141,182 | readdir_query + readlink + execute_query |
|
||||
| **Search index** | `index.rs` | 98,174 | index_file + search entry |
|
||||
| **Fetcher** | `fetcher.rs` | 54,61,121 | fetch_file entry + meta + ensure_cached |
|
||||
|
||||
**Key DEBUG fields**: `ino`, `parent`, `name`, `offset`, `size`, `bytes_read`, `origin_id`, `path`, `file_id`, `query`, `results_count`, `latency_ms`
|
||||
|
||||
### TRACE Level (100+ locations) - Fine-Grained Flow
|
||||
|
||||
| Category | File | Lines | What to Log |
|
||||
|----------|------|-------|-------------|
|
||||
| **Manifest cache** | `reader.rs` | 67-74 | Cache hit/miss decision |
|
||||
| **Chunk iteration** | `reader.rs` | 107-127 | Each chunk: skip/read boundaries |
|
||||
| **CAS dedup** | `store.rs` | 74-77 | Dedup hit decision |
|
||||
| **CAS integrity** | `store.rs` | 121-134 | Verification result |
|
||||
| **Tree lookup** | `tree.rs` | 118-129 | Path→inode + child lookup |
|
||||
| **Tree parent** | `tree.rs` | 148-153 | Parent resolution path |
|
||||
| **Prefetch event** | `prefetch.rs` | 91-120 | Event type match arms |
|
||||
| **Prefetch semaphore** | `prefetch.rs` | 150-164 | In-flight check + acquire |
|
||||
| **Delta scan** | `delta.rs` | 79-102 | Each file: cached/modified/unchanged/removed |
|
||||
| **Delta entries** | `delta.rs` | 128-146 | Each entry: dir/audio/skip |
|
||||
| **CDC chunking** | `cdc.rs` | 84-93 | Each chunk: offset/length/hash |
|
||||
| **Failover origin** | `failover.rs` | 68-93 | Each origin attempt result |
|
||||
| **Failover retry** | `failover.rs` | 107-122 | Each retry: attempt/success/delay |
|
||||
| **Router select** | `router.rs` | 79-108 | Each candidate + selection reason |
|
||||
| **FUSE node→attr** | `filesystem.rs` | 109-145 | Directory vs file conversion |
|
||||
| **FUSE lookup** | `filesystem.rs` | 192-200 | Found/not found |
|
||||
| **FUSE readdir** | `filesystem.rs` | 274-291 | Each child entry |
|
||||
| **FUSE read** | `filesystem.rs` | 340-367 | file_id resolution + result |
|
||||
| **Metadata tag** | `parser.rs` | 86-100 | Each tag extraction |
|
||||
| **Health transition** | `health.rs` | 199-237 | State transition details |
|
||||
| **Latency recording** | `router.rs` | 23-42 | Stats update per sample |
|
||||
|
||||
**Key TRACE patterns**:
|
||||
- Every `match` arm: `trace!("match arm: {variant}")`
|
||||
- Every `if/else`: `trace!("branch: {condition}={value}")`
|
||||
- Every loop iteration: `trace!("iteration {i}/{total}: ...")`
|
||||
- Every cache lookup: `trace!("cache lookup key={key}, hit={hit}")`
|
||||
|
||||
---
|
||||
|
||||
## gRPC Handler Instrumentation (ADDED - Oracle Review)
|
||||
|
||||
**Gap identified**: 8/10 gRPC handlers had no logging.
|
||||
|
||||
### server.rs - All Handlers
|
||||
|
||||
| Handler | Line | Level | Log Message |
|
||||
|---------|------|-------|-------------|
|
||||
| `get_status()` | 209 | DEBUG | `debug!("gRPC get_status called")` |
|
||||
| `get_cache_stats()` | 241 | DEBUG | `debug!("gRPC get_cache_stats called")` |
|
||||
| `clear_cache()` | 278 | INFO | `info!("gRPC clear_cache: clearing {tier}")` |
|
||||
| `prefetch()` | 296 | DEBUG | `debug!(file_count = paths.len(), "gRPC prefetch started")` |
|
||||
| `list_origins()` | 322 | DEBUG | `debug!("gRPC list_origins called")` |
|
||||
| `get_origin_health()` | 329 | DEBUG | `debug!(origin_id = %id, "gRPC get_origin_health")` |
|
||||
| `rescan_origin()` | 337 | INFO | `info!(origin_id = %id, "gRPC rescan_origin started")` |
|
||||
| `subscribe_events()` | 376 | INFO | `info!("gRPC subscribe_events: client connected")` |
|
||||
| `shutdown()` | 402 | INFO | `info!(graceful = graceful, "gRPC shutdown requested")` |
|
||||
|
||||
### search_service.rs
|
||||
|
||||
| Handler | Line | Level | Log Message |
|
||||
|---------|------|-------|-------------|
|
||||
| `search()` | entry | DEBUG | `debug!(query = %q, limit = limit, "gRPC search")` |
|
||||
| `search()` | result | DEBUG | `debug!(results = results.len(), "gRPC search completed")` |
|
||||
|
||||
### Pattern: Use `#[instrument]` on all handlers
|
||||
|
||||
```rust
|
||||
#[tracing::instrument(level = "debug", skip(self, request), fields(method = "get_status"))]
|
||||
async fn get_status(&self, request: Request<()>) -> Result<Response<StatusResponse>, Status> {
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Async Task Spawn Instrumentation (ADDED - Oracle Review)
|
||||
|
||||
**Gap identified**: 14 `tokio::spawn` sites need correlation IDs and span propagation.
|
||||
|
||||
### Spawn Sites Requiring Instrumentation
|
||||
|
||||
| File | Line | Task | Instrumentation |
|
||||
|------|------|------|-----------------|
|
||||
| `server.rs` | 305 | prefetch stream | `spawn(async { ... }.instrument(info_span!("prefetch_stream")))` |
|
||||
| `server.rs` | 354 | rescan stream | `spawn(async { ... }.instrument(info_span!("rescan_stream", origin_id = %id)))` |
|
||||
| `server.rs` | 384 | subscribe events | `spawn(async { ... }.instrument(info_span!("event_subscriber")))` |
|
||||
| `search_service.rs` | spawn | search task | `spawn(async { ... }.instrument(debug_span!("search_task", query = %q)))` |
|
||||
| `indexer.rs` | spawn | indexer loop | `spawn(async { ... }.instrument(info_span!("indexer")))` |
|
||||
| `prefetch.rs` | 87 | prefetch engine | `spawn(async { ... }.instrument(info_span!("prefetch_engine")))` |
|
||||
| `prefetch.rs` | 169 | prefetch file | `spawn(async { ... }.instrument(debug_span!("prefetch_file", file_id = ?id)))` |
|
||||
| `health.rs` | 154 | health monitor | `spawn(async { ... }.instrument(info_span!("health_monitor")))` |
|
||||
| `watcher.rs` | 34 | file watcher | `spawn(async { ... }.instrument(info_span!("file_watcher", origin_id = %id)))` |
|
||||
| `artwork.rs` | spawn | image decode | `spawn_blocking(|| { ... })` - add span before spawn |
|
||||
|
||||
### Pattern: Span Propagation
|
||||
|
||||
```rust
|
||||
use tracing::Instrument;
|
||||
|
||||
// BEFORE (loses context)
|
||||
tokio::spawn(async move {
|
||||
do_work().await;
|
||||
});
|
||||
|
||||
// AFTER (preserves correlation)
|
||||
let span = tracing::info_span!("task_name", task_id = %id);
|
||||
tokio::spawn(async move {
|
||||
do_work().await;
|
||||
}.instrument(span));
|
||||
```
|
||||
|
||||
### Add to init_logging() for request IDs
|
||||
|
||||
```rust
|
||||
// Generate request ID for correlation
|
||||
use tracing::Span;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn with_request_id<F, R>(f: F) -> R
|
||||
where F: FnOnce() -> R {
|
||||
let request_id = Uuid::new_v4();
|
||||
let span = tracing::info_span!("request", request_id = %request_id);
|
||||
span.in_scope(f)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Database Operation Logging (ADDED - Oracle Review)
|
||||
|
||||
**Gap identified**: Zero logging for rusqlite operations in db.rs, collections.rs, patterns.rs, artwork.rs.
|
||||
|
||||
### db.rs - Core Database
|
||||
|
||||
| Function | Line | Level | Log Message |
|
||||
|----------|------|-------|-------------|
|
||||
| `open()` | entry | INFO | `info!(path = ?path, "Opening metadata database")` |
|
||||
| `open()` | success | INFO | `info!(file_count = count, "Database opened")` |
|
||||
| `upsert_file()` | entry | DEBUG | `debug!(file_id = ?id, path = %path, "Upserting file")` |
|
||||
| `upsert_file()` | error | ERROR | `error!(file_id = ?id, error = %e, "Failed to upsert file")` |
|
||||
| `get_file_by_id()` | miss | TRACE | `trace!(file_id = ?id, "File not found in db")` |
|
||||
| `delete_file()` | entry | DEBUG | `debug!(file_id = ?id, "Deleting file from db")` |
|
||||
| `list_files_by_origin()` | result | DEBUG | `debug!(origin_id = %id, count = files.len(), "Listed files")` |
|
||||
|
||||
### collections.rs
|
||||
|
||||
| Function | Line | Level | Log Message |
|
||||
|----------|------|-------|-------------|
|
||||
| `create()` | entry | INFO | `info!(name = %name, "Creating collection")` |
|
||||
| `save()` | error | WARN | `warn!(name = %name, error = %e, "Failed to save collection")` |
|
||||
| `delete()` | entry | INFO | `info!(name = %name, "Deleting collection")` |
|
||||
| `list()` | result | DEBUG | `debug!(count = collections.len(), "Listed collections")` |
|
||||
|
||||
### patterns.rs - Access Patterns
|
||||
|
||||
| Function | Line | Level | Log Message |
|
||||
|----------|------|-------|-------------|
|
||||
| `record_access()` | entry | TRACE | `trace!(file_id = ?id, "Recording access pattern")` |
|
||||
| `predict_next()` | result | DEBUG | `debug!(predictions = preds.len(), "Predicted next files")` |
|
||||
|
||||
### artwork.rs
|
||||
|
||||
| Function | Line | Level | Log Message |
|
||||
|----------|------|-------|-------------|
|
||||
| `store()` | entry | DEBUG | `debug!(file_id = ?id, size_bytes = data.len(), "Storing artwork")` |
|
||||
| `get()` | hit/miss | TRACE | `trace!(file_id = ?id, found = found, "Artwork lookup")` |
|
||||
|
||||
### Pattern: Database Error Wrapper
|
||||
|
||||
```rust
|
||||
// Add to musicfs-cache/src/db.rs
|
||||
fn log_db_result<T>(op: &str, result: Result<T, rusqlite::Error>) -> Result<T, Error> {
|
||||
match result {
|
||||
Ok(v) => {
|
||||
tracing::trace!(op = op, "db operation succeeded");
|
||||
Ok(v)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(op = op, error = %e, "db operation failed");
|
||||
Err(Error::Database(e.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Channel Operation Logging (ADDED - Oracle Review)
|
||||
|
||||
**Gap identified**: No logging for channel capacity, close, or broadcast lag.
|
||||
|
||||
### Channel Locations
|
||||
|
||||
| File | Type | Log Points |
|
||||
|------|------|------------|
|
||||
| `events.rs` | broadcast | Lag warning when receiver falls behind |
|
||||
| `watcher.rs` | mpsc | Channel close on watcher shutdown |
|
||||
| `server.rs` | mpsc | gRPC stream channel capacity |
|
||||
| `indexer.rs` | mpsc | Event queue depth |
|
||||
| `health.rs` | mpsc | Health check channel |
|
||||
|
||||
### Patterns
|
||||
|
||||
```rust
|
||||
// Broadcast lag detection (events.rs)
|
||||
match rx.recv().await {
|
||||
Ok(event) => { /* handle */ }
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
tracing::warn!(skipped = n, "Event subscriber lagged, skipped events");
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => {
|
||||
tracing::debug!("Event channel closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Channel capacity warning (before send)
|
||||
if tx.capacity() < 10 {
|
||||
tracing::warn!(remaining = tx.capacity(), "Channel near capacity");
|
||||
}
|
||||
|
||||
// Channel close
|
||||
impl Drop for EventBus {
|
||||
fn drop(&mut self) {
|
||||
tracing::debug!("Event bus shutting down");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Drop Implementation Logging (ADDED - Oracle Review)
|
||||
|
||||
**Gap identified**: No logging in Drop impls for cleanup verification.
|
||||
|
||||
| File | Type | Log Message |
|
||||
|------|------|-------------|
|
||||
| `manager.rs:276` | `PluginManager` | `debug!("PluginManager dropping, unloading {} plugins", self.plugins.len())` |
|
||||
| `watcher.rs:157` | `WatchHandle` | `trace!(origin_id = %self.origin_id, "WatchHandle dropped")` |
|
||||
| `prefetch.rs` | `PrefetchEngine` | `debug!("PrefetchEngine dropping, {} in-flight", self.in_flight.len())` |
|
||||
| `server.rs` | gRPC server | `info!("gRPC server shutting down")` |
|
||||
|
||||
### Pattern
|
||||
|
||||
```rust
|
||||
impl Drop for PluginManager {
|
||||
fn drop(&mut self) {
|
||||
tracing::debug!(
|
||||
plugin_count = self.plugins.len(),
|
||||
"PluginManager dropping"
|
||||
);
|
||||
// existing cleanup...
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Credential Loading (ADDED - Oracle Review)
|
||||
|
||||
**Gap identified**: No logging in credentials.rs::load().
|
||||
|
||||
| Function | Level | Log Message |
|
||||
|----------|-------|-------------|
|
||||
| `load()` entry | DEBUG | `debug!(origin_id = %origin_id, "Loading credentials")` |
|
||||
| `load()` cache hit | TRACE | `trace!(origin_id = %origin_id, "Credential cache hit")` |
|
||||
| `load()` success | INFO | `info!(origin_id = %origin_id, cred_type = %cred.type_name(), "Credential loaded")` |
|
||||
| `load()` not found | DEBUG | `debug!(origin_id = %origin_id, "No credential found")` |
|
||||
| `load()` error | WARN | `warn!(origin_id = %origin_id, error = %e, "Credential load failed")` |
|
||||
|
||||
**SECURITY**: Never log credential values. The existing Debug impl with redaction is correct.
|
||||
|
||||
---
|
||||
|
||||
## Security Considerations (ADDED - Oracle Review)
|
||||
|
||||
### Never Log These
|
||||
|
||||
| Data | Location | Mitigation |
|
||||
|------|----------|------------|
|
||||
| `WebhookConfig.secret` | webhook.rs | Add `#[serde(skip_serializing)]`, use custom Debug |
|
||||
| Credential values | credentials.rs | Already redacted in Debug impl ✓ |
|
||||
| Full file paths with usernames | everywhere | Sanitize `/home/{user}/` → `~/` |
|
||||
| API keys/tokens | config.rs | Mark sensitive fields |
|
||||
|
||||
### Sanitization Helper
|
||||
|
||||
```rust
|
||||
// Add to musicfs-core/src/lib.rs
|
||||
pub fn sanitize_path(path: &Path) -> String {
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
path.to_string_lossy()
|
||||
.replace(&home, "~")
|
||||
.to_string()
|
||||
} else {
|
||||
path.to_string_lossy().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
// Usage
|
||||
debug!(path = %sanitize_path(&path), "Reading file");
|
||||
```
|
||||
|
||||
### WebhookConfig Fix
|
||||
|
||||
```rust
|
||||
// webhook.rs - add custom Debug
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
pub struct WebhookConfig {
|
||||
pub url: String,
|
||||
#[serde(skip_serializing)]
|
||||
pub secret: Option<String>, // Never serialize
|
||||
// ...
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for WebhookConfig {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("WebhookConfig")
|
||||
.field("url", &self.url)
|
||||
.field("secret", &self.secret.as_ref().map(|_| "[REDACTED]"))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Performance Considerations (ADDED - Oracle Review)
|
||||
|
||||
### Hot Path Warnings
|
||||
|
||||
| Path | Risk | Mitigation |
|
||||
|------|------|------------|
|
||||
| `reader.rs` chunk loop | 100s of TRACE logs per seek | Log summary only: `trace!(chunks_read = n, "Read complete")` |
|
||||
| `store.rs` put/get | 1000s during sync | Keep at DEBUG, not TRACE |
|
||||
| `delta.rs` file scan | Log per file during full scan | Use TRACE, batch summaries at DEBUG |
|
||||
| `parser.rs` tag extraction | Many TRACE per file | Sample: log every 100th file |
|
||||
|
||||
### Trace Sampling Config
|
||||
|
||||
```rust
|
||||
// Add to LoggingConfig
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LoggingConfig {
|
||||
// ... existing fields ...
|
||||
|
||||
/// Sample rate for TRACE logs in hot paths (0.0-1.0, default 1.0)
|
||||
#[serde(default = "default_sample_rate")]
|
||||
pub trace_sample_rate: f32,
|
||||
}
|
||||
|
||||
fn default_sample_rate() -> f32 { 1.0 }
|
||||
|
||||
// Usage in hot paths
|
||||
if rand::random::<f32>() < config.trace_sample_rate {
|
||||
trace!(...);
|
||||
}
|
||||
```
|
||||
|
||||
### Rate-Limited Warnings
|
||||
|
||||
```rust
|
||||
// For repeating warnings during outages (failover.rs)
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
static LAST_FAILOVER_WARN: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn warn_rate_limited(origin_id: &str, error: &str) {
|
||||
let now = Instant::now().elapsed().as_secs();
|
||||
let last = LAST_FAILOVER_WARN.load(Ordering::Relaxed);
|
||||
if now - last >= 60 { // Max once per minute
|
||||
LAST_FAILOVER_WARN.store(now, Ordering::Relaxed);
|
||||
warn!(origin_id = %origin_id, error = %error, "Origin failover");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Standardized Field Names (ADDED - Oracle Review)
|
||||
|
||||
Use these consistently across all log statements:
|
||||
|
||||
| Field | Type | Usage |
|
||||
|-------|------|-------|
|
||||
| `origin_id` | String | Origin identifier (not `origin`) |
|
||||
| `file_id` | FileId | File identifier |
|
||||
| `path` | String | Virtual or real path (sanitized) |
|
||||
| `size_bytes` | u64 | Size in bytes (not `size`, `bytes`, `len`) |
|
||||
| `offset` | u64 | Read offset |
|
||||
| `duration_ms` | u64 | Operation duration in milliseconds |
|
||||
| `count` | usize | Generic count |
|
||||
| `attempt` | u32 | Retry attempt number |
|
||||
| `max_attempts` | u32 | Maximum retry attempts |
|
||||
| `error` | impl Display | Error message (not `err`, `e`) |
|
||||
| `request_id` | Uuid | Correlation ID for requests |
|
||||
|
||||
---
|
||||
|
||||
## Instrumentation Patterns (ADDED - Oracle Review)
|
||||
|
||||
### Use `#[instrument(err)]` for Automatic Error Logging
|
||||
|
||||
```rust
|
||||
// BEFORE: Manual error logging
|
||||
pub async fn read(&self, path: &Path) -> Result<Bytes> {
|
||||
match self.inner_read(path).await {
|
||||
Ok(data) => Ok(data),
|
||||
Err(e) => {
|
||||
error!(path = ?path, error = %e, "Read failed");
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AFTER: Automatic with #[instrument]
|
||||
#[tracing::instrument(level = "debug", skip(self), err)]
|
||||
pub async fn read(&self, path: &Path) -> Result<Bytes> {
|
||||
self.inner_read(path).await
|
||||
}
|
||||
```
|
||||
|
||||
### Span Events vs Regular Logs
|
||||
|
||||
```rust
|
||||
// Regular log - standalone event
|
||||
info!("Operation completed");
|
||||
|
||||
// Span event - attached to current span context
|
||||
tracing::Span::current().record("result", "success");
|
||||
|
||||
// Prefer span events for operation outcomes
|
||||
#[instrument(fields(result))]
|
||||
async fn operation() -> Result<()> {
|
||||
// ... work ...
|
||||
Span::current().record("result", "success");
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Fixes: Incorrect Line References (ADDED - Oracle Review)
|
||||
|
||||
| File | Issue | Fix |
|
||||
|------|-------|-----|
|
||||
| `webhook.rs:43` | Uses `expect()` (panics) | Replace with `?` + error log |
|
||||
| `webhook.rs:133` | Uses `expect()` (panics) | Replace with `?` + error log |
|
||||
|
||||
```rust
|
||||
// webhook.rs - BEFORE
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("Failed to create HTTP client");
|
||||
|
||||
// webhook.rs - AFTER
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "Failed to create webhook HTTP client");
|
||||
WebhookError::ClientInit(e.to_string())
|
||||
})?;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Log Levels Guide
|
||||
|
||||
| Level | Use Case | Example |
|
||||
|-------|----------|---------|
|
||||
| `ERROR` | Unrecoverable failures | Mount failed, DB corruption |
|
||||
| `WARN` | Recoverable issues | Origin timeout, retry needed |
|
||||
| `INFO` | Lifecycle events | Service start/stop, health change |
|
||||
| `DEBUG` | Operation details | Function entry, request params |
|
||||
| `TRACE` | Fine-grained flow | Match arms, cache hit/miss |
|
||||
|
||||
---
|
||||
|
||||
## Testing Checklist
|
||||
|
||||
### Basic Functionality
|
||||
- [ ] Log files created in configured directory
|
||||
- [ ] Daily rotation creates new files at midnight
|
||||
- [ ] JSON output parseable by `jq`
|
||||
- [ ] `journalctl -t musicfs` shows logs
|
||||
- [ ] `RUST_LOG=musicfs=trace` enables trace output
|
||||
- [ ] WorkerGuard kept alive (logs flush on shutdown)
|
||||
- [ ] Logrotate compresses old files
|
||||
|
||||
### Correlation & Context (NEW)
|
||||
- [ ] Request IDs propagate through async tasks
|
||||
- [ ] Spawned task logs include parent span context
|
||||
- [ ] gRPC handler logs show method name in span
|
||||
|
||||
### Security (NEW)
|
||||
- [ ] WebhookConfig.secret never appears in logs
|
||||
- [ ] Credential values never appear in logs
|
||||
- [ ] File paths with `/home/{user}` show as `~/`
|
||||
|
||||
### Performance (NEW)
|
||||
- [ ] TRACE sampling respects `trace_sample_rate` config
|
||||
- [ ] Hot path chunk loops log summary, not per-chunk
|
||||
- [ ] Origin failover warnings are rate-limited (1/minute)
|
||||
- [ ] Database operations log without blocking
|
||||
|
||||
### Database & Channels (NEW)
|
||||
- [ ] Database open logs file count
|
||||
- [ ] Channel capacity warnings appear when queue fills
|
||||
- [ ] Broadcast lag warnings appear when subscriber falls behind
|
||||
- [ ] Drop implementations log cleanup
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
| Phase | Effort | Deliverables |
|
||||
|-------|--------|--------------|
|
||||
| 1. Config & Dependencies | 2h | LoggingConfig, init_logging(), logrotate, trace sampling |
|
||||
| 2. Core instrumentation | 1h | tracing in musicfs-core, credentials, sanitization |
|
||||
| 3. Hot path instrumentation | 4h | #[instrument] + trace! across 5 crates |
|
||||
| 4. gRPC & async tasks | 2h | Handler instrumentation, spawn correlation |
|
||||
| 5. Database & channels | 2h | rusqlite logging, channel capacity/close |
|
||||
| 6. Production files | 1h | Updated systemd, example config |
|
||||
| **Total** | **12h** | Full observability |
|
||||
|
||||
---
|
||||
|
||||
## Files to Modify
|
||||
|
||||
### Phase 1: Config & Dependencies
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `Cargo.toml` (workspace) | Add tracing-appender, tracing-journald |
|
||||
| `crates/musicfs-cli/Cargo.toml` | Add dependencies |
|
||||
| `crates/musicfs-core/Cargo.toml` | Add tracing |
|
||||
| `crates/musicfs-core/src/config.rs` | Add LoggingConfig with trace_sample_rate |
|
||||
| `crates/musicfs-cli/src/main.rs` | Expand init_logging(), request ID helper |
|
||||
| `crates/musicfs-core/src/lib.rs` | Add sanitize_path() helper |
|
||||
|
||||
### Phase 2: Core Instrumentation
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `crates/musicfs-core/src/credentials.rs` | Add load() logging (redacted) |
|
||||
| `crates/musicfs-core/src/events.rs` | Add broadcast lag detection |
|
||||
|
||||
### Phase 3: Hot Path Instrumentation
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `crates/musicfs-fuse/src/filesystem.rs` | Add #[instrument], trace! |
|
||||
| `crates/musicfs-origins/src/failover.rs` | Add #[instrument], trace!, rate-limited warn |
|
||||
| `crates/musicfs-origins/src/health.rs` | Add state transition logging |
|
||||
| `crates/musicfs-origins/src/router.rs` | Add selection logging |
|
||||
| `crates/musicfs-cache/src/tree.rs` | Add mutation logging |
|
||||
| `crates/musicfs-cache/src/metadata.rs` | Add hit/miss logging |
|
||||
| `crates/musicfs-cas/src/reader.rs` | Add chunk assembly logging (summary, not per-chunk) |
|
||||
| `crates/musicfs-cas/src/store.rs` | Add dedup logging |
|
||||
| `crates/musicfs-sync/src/delta.rs` | Add change detection logging |
|
||||
|
||||
### Phase 4: gRPC & Async Tasks (NEW)
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `crates/musicfs-grpc/src/server.rs` | Add #[instrument] to all 10 handlers, spawn correlation |
|
||||
| `crates/musicfs-grpc/src/search_service.rs` | Add #[instrument], spawn instrumentation |
|
||||
| `crates/musicfs-grpc/src/webhook.rs` | Fix expect() → error!, custom Debug for secret |
|
||||
| `crates/musicfs-cache/src/prefetch.rs` | Add spawn instrumentation, Drop logging |
|
||||
| `crates/musicfs-search/src/indexer.rs` | Add spawn instrumentation |
|
||||
| `crates/musicfs-sync/src/watcher.rs` | Add spawn instrumentation, Drop logging |
|
||||
| `crates/musicfs-plugins/src/manager.rs` | Add Drop logging |
|
||||
|
||||
### Phase 5: Database & Channels (NEW)
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `crates/musicfs-cache/src/db.rs` | Add log_db_result() helper, open/upsert/query logging |
|
||||
| `crates/musicfs-search/src/collections.rs` | Add CRUD operation logging |
|
||||
| `crates/musicfs-cache/src/patterns.rs` | Add access pattern logging |
|
||||
| `crates/musicfs-cache/src/artwork.rs` | Add store/get logging |
|
||||
|
||||
### Phase 6: Production Files
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `dist/musicfs.service` | Add logging directives |
|
||||
| `dist/logrotate.d/musicfs` | New file |
|
||||
| `dist/config.example.toml` | Add logging section with trace_sample_rate |
|
||||
Generated
+81
-14
@@ -1764,7 +1764,7 @@ dependencies = [
|
||||
"serde",
|
||||
"sled",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -1784,7 +1784,7 @@ dependencies = [
|
||||
"serde",
|
||||
"sled",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"xxhash-rust",
|
||||
@@ -1805,6 +1805,8 @@ dependencies = [
|
||||
"musicfs-origins",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
"tracing-journald",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
@@ -1816,9 +1818,10 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"toml",
|
||||
"tracing",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
@@ -1854,6 +1857,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
@@ -1868,7 +1872,7 @@ dependencies = [
|
||||
"image",
|
||||
"musicfs-core",
|
||||
"symphonia",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -1881,7 +1885,7 @@ dependencies = [
|
||||
"libc",
|
||||
"musicfs-core",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -1897,7 +1901,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"wasmtime",
|
||||
@@ -1915,7 +1919,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"tantivy",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -1932,7 +1936,7 @@ dependencies = [
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"xxhash-rust",
|
||||
@@ -2460,7 +2464,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"libredox",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2893,6 +2897,12 @@ version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "symlink"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
|
||||
|
||||
[[package]]
|
||||
name = "symphonia"
|
||||
version = "0.5.5"
|
||||
@@ -3126,7 +3136,7 @@ dependencies = [
|
||||
"tantivy-stacker",
|
||||
"tantivy-tokenizer-api",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"time",
|
||||
"uuid",
|
||||
"winapi",
|
||||
@@ -3247,7 +3257,16 @@ version = "1.0.69"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
"thiserror-impl 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||
dependencies = [
|
||||
"thiserror-impl 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3261,6 +3280,17 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.9"
|
||||
@@ -3507,6 +3537,19 @@ dependencies = [
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-appender"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"symlink",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-attributes"
|
||||
version = "0.1.31"
|
||||
@@ -3528,6 +3571,17 @@ dependencies = [
|
||||
"valuable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-journald"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d3a81ed245bfb62592b1e2bc153e77656d94ee6a0497683a65a12ccaf2438d0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"tracing-core",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-log"
|
||||
version = "0.2.0"
|
||||
@@ -3539,6 +3593,16 @@ dependencies = [
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-serde"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-subscriber"
|
||||
version = "0.3.23"
|
||||
@@ -3549,12 +3613,15 @@ dependencies = [
|
||||
"nu-ansi-term",
|
||||
"once_cell",
|
||||
"regex-automata",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sharded-slab",
|
||||
"smallvec",
|
||||
"thread_local",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-log",
|
||||
"tracing-serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3944,7 +4011,7 @@ dependencies = [
|
||||
"log",
|
||||
"object 0.32.2",
|
||||
"target-lexicon",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"wasmparser 0.201.0",
|
||||
"wasmtime-cranelift-shared",
|
||||
"wasmtime-environ",
|
||||
@@ -3985,7 +4052,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"target-lexicon",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"wasm-encoder 0.201.0",
|
||||
"wasmparser 0.201.0",
|
||||
"wasmprinter",
|
||||
@@ -4076,7 +4143,7 @@ dependencies = [
|
||||
"cranelift-entity",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"thiserror",
|
||||
"thiserror 1.0.69",
|
||||
"wasmparser 0.201.0",
|
||||
]
|
||||
|
||||
|
||||
+3
-1
@@ -30,7 +30,9 @@ dashmap = "5"
|
||||
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
||||
tracing-appender = "0.2"
|
||||
tracing-journald = "0.3"
|
||||
|
||||
# FUSE
|
||||
fuser = "0.14"
|
||||
|
||||
@@ -5,7 +5,7 @@ use musicfs_metadata::artwork::{ArtSize, ArtType, Artwork};
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tracing::debug;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
const MAX_ARTWORK_INPUT_SIZE: usize = 10 * 1024 * 1024;
|
||||
|
||||
@@ -40,6 +40,7 @@ impl ArtworkCache {
|
||||
[],
|
||||
)?;
|
||||
|
||||
info!(path = ?db_path, "Artwork cache opened");
|
||||
Ok(Self {
|
||||
store,
|
||||
db_path: db_path.to_path_buf(),
|
||||
@@ -47,7 +48,9 @@ impl ArtworkCache {
|
||||
}
|
||||
|
||||
pub async fn store(&self, file_id: i64, artwork: &Artwork) -> Result<ChunkHash, ArtworkError> {
|
||||
trace!(file_id = file_id, size_bytes = artwork.data.len(), "Storing artwork");
|
||||
if artwork.data.len() > MAX_ARTWORK_INPUT_SIZE {
|
||||
warn!(file_id = file_id, size = artwork.data.len(), max = MAX_ARTWORK_INPUT_SIZE, "Artwork too large");
|
||||
return Err(ArtworkError::ImageTooLarge(artwork.data.len()));
|
||||
}
|
||||
|
||||
@@ -88,6 +91,7 @@ impl ArtworkCache {
|
||||
art_type: &str,
|
||||
size: ArtSize,
|
||||
) -> Result<Option<Vec<u8>>, ArtworkError> {
|
||||
trace!(file_id = file_id, art_type = %art_type, "Getting artwork");
|
||||
let db_path = self.db_path.clone();
|
||||
let art_type_clone = art_type.to_string();
|
||||
|
||||
@@ -107,6 +111,7 @@ impl ArtworkCache {
|
||||
|
||||
match hash_hex {
|
||||
Some(hex) => {
|
||||
trace!(file_id = file_id, "Artwork cache hit");
|
||||
let hash = ChunkHash::from_hex(&hex).ok_or(ArtworkError::InvalidHash)?;
|
||||
let data = self.store.get(&hash).await?;
|
||||
|
||||
@@ -118,7 +123,10 @@ impl ArtworkCache {
|
||||
}
|
||||
}
|
||||
}
|
||||
None => Ok(None),
|
||||
None => {
|
||||
trace!(file_id = file_id, "Artwork cache miss");
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ pub struct Database {
|
||||
|
||||
impl Database {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
info!(?path, "Opening database");
|
||||
debug!(?path, "Opening database");
|
||||
|
||||
let conn =
|
||||
Connection::open(path).map_err(|e| Error::Database(format!("open failed: {}", e)))?;
|
||||
@@ -24,9 +24,12 @@ impl Database {
|
||||
conn.execute_batch(SCHEMA)
|
||||
.map_err(|e| Error::Database(format!("schema init failed: {}", e)))?;
|
||||
|
||||
Ok(Self {
|
||||
let db = Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
})
|
||||
};
|
||||
let count = db.file_count().unwrap_or(0);
|
||||
info!(path = ?path, file_count = count, "Database opened");
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
pub fn open_memory() -> Result<Self> {
|
||||
|
||||
@@ -3,6 +3,7 @@ use musicfs_core::{AudioMeta, FileMeta, OriginId, Result, VirtualPath};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tracing::trace;
|
||||
|
||||
pub struct MetadataCache {
|
||||
db: Arc<Database>,
|
||||
@@ -34,7 +35,10 @@ impl MetadataCache {
|
||||
}
|
||||
|
||||
pub fn lookup(&self, path: &VirtualPath) -> Result<Option<FileMeta>> {
|
||||
self.db.get_file_by_virtual_path(path)
|
||||
let result = self.db.get_file_by_virtual_path(path)?;
|
||||
let hit = result.is_some();
|
||||
trace!(path = path.as_str(), hit, "metadata cache lookup");
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn is_fresh(
|
||||
@@ -52,8 +56,11 @@ impl MetadataCache {
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs();
|
||||
Ok(current_secs == cached_secs)
|
||||
let hit = current_secs == cached_secs;
|
||||
trace!(path = ?real_path, hit, "metadata freshness check");
|
||||
Ok(hit)
|
||||
} else {
|
||||
trace!(path = ?real_path, hit = false, "metadata freshness check");
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ use parking_lot::{Mutex, RwLock};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tracing::{debug, info, trace};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccessPattern {
|
||||
@@ -79,15 +80,19 @@ impl PatternStore {
|
||||
map
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
let store = Self {
|
||||
db: Mutex::new(db),
|
||||
sequence_counts: RwLock::new(sequence_counts),
|
||||
time_patterns: RwLock::new(HashMap::new()),
|
||||
max_history,
|
||||
})
|
||||
};
|
||||
let sequence_count = store.sequence_counts.read().len();
|
||||
info!(path = ?db_path, sequence_count = sequence_count, max_history = max_history, "Pattern store opened");
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub fn record(&self, file_id: FileId, _context: AccessContext) -> Result<(), PatternError> {
|
||||
trace!(file_id = file_id.0, "Recording access pattern");
|
||||
let now = SystemTime::now();
|
||||
let timestamp = now.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64;
|
||||
let hour = (timestamp / 3600 % 24) as u8;
|
||||
@@ -144,11 +149,13 @@ impl PatternStore {
|
||||
.collect();
|
||||
|
||||
predictions.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
predictions
|
||||
let result: Vec<FileId> = predictions
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|(id, _)| id)
|
||||
.collect()
|
||||
.collect();
|
||||
debug!(file_id = current.0, predictions = result.len(), "Predicted next files");
|
||||
result
|
||||
}
|
||||
|
||||
pub fn predict_for_time(&self, hour: u8, limit: usize) -> Vec<FileId> {
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::ffi::{OsStr, OsString};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::RwLock;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::{debug, trace};
|
||||
|
||||
pub type Inode = u64;
|
||||
pub const ROOT_INODE: Inode = 1;
|
||||
@@ -123,8 +124,12 @@ impl VirtualTree {
|
||||
|
||||
pub fn lookup(&self, parent_inode: Inode, name: &OsStr) -> Option<Inode> {
|
||||
if let Some(VirtualNode::Directory(dir)) = self.nodes.get(&parent_inode) {
|
||||
dir.children.get(name).copied()
|
||||
let result = dir.children.get(name).copied();
|
||||
let hit = result.is_some();
|
||||
trace!(inode = parent_inode, name = ?name, hit, "tree lookup");
|
||||
result
|
||||
} else {
|
||||
trace!(inode = parent_inode, name = ?name, hit = false, "tree lookup");
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -194,6 +199,7 @@ impl VirtualTree {
|
||||
dir.children.insert(name, inode);
|
||||
}
|
||||
|
||||
debug!(inode, path = path.as_str(), file_id = ?meta.id, "add file to tree");
|
||||
inode
|
||||
}
|
||||
|
||||
@@ -263,6 +269,7 @@ impl VirtualTree {
|
||||
}
|
||||
}
|
||||
|
||||
debug!(inode, path = path.as_str(), file_id = ?file.file_id, "remove file from tree");
|
||||
Some(file.file_id)
|
||||
} else {
|
||||
None
|
||||
|
||||
@@ -6,6 +6,7 @@ use musicfs_core::FileId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tracing::{debug, trace};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkManifest {
|
||||
@@ -67,10 +68,12 @@ impl FileReader {
|
||||
{
|
||||
let manifests = self.manifests.read().unwrap();
|
||||
if let Some(m) = manifests.get(&file_id) {
|
||||
trace!(file_id = ?file_id, "manifest cache hit");
|
||||
return Ok(m.clone());
|
||||
}
|
||||
}
|
||||
|
||||
trace!(file_id = ?file_id, "manifest cache miss");
|
||||
let Some(fetcher) = &self.fetcher else {
|
||||
return Err(ReaderError::ManifestNotFound(file_id));
|
||||
};
|
||||
@@ -103,6 +106,7 @@ impl FileReader {
|
||||
|
||||
let end = std::cmp::min(offset + size as u64, manifest.total_size);
|
||||
let mut result = BytesMut::with_capacity((end - offset) as usize);
|
||||
let mut chunks_read = 0u32;
|
||||
|
||||
for chunk_ref in &manifest.chunks {
|
||||
let chunk_start = chunk_ref.offset;
|
||||
@@ -127,8 +131,11 @@ impl FileReader {
|
||||
};
|
||||
|
||||
result.extend_from_slice(&chunk_data[read_start..read_end]);
|
||||
chunks_read += 1;
|
||||
}
|
||||
|
||||
let bytes_read = result.len() as u64;
|
||||
debug!(file_id = ?file_id, offset, size, chunks_read, bytes_read, "read completed");
|
||||
Ok(result.freeze())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use musicfs_core::ChunkHash;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::fs;
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
const DEFAULT_MAX_SIZE_10GB: u64 = 10 * 1024 * 1024 * 1024;
|
||||
const DEFAULT_SHARD_LEVELS_256_SUBDIRS: u8 = 2;
|
||||
@@ -72,7 +72,7 @@ impl CasStore {
|
||||
let path = self.chunk_path(&hash);
|
||||
|
||||
if path.exists() {
|
||||
debug!("Chunk {} already exists (dedup)", hash);
|
||||
trace!(hash = %hash, size_bytes = data.len(), "dedup hit");
|
||||
return Ok(hash);
|
||||
}
|
||||
|
||||
@@ -94,7 +94,7 @@ impl CasStore {
|
||||
self.current_size
|
||||
.fetch_add(data.len() as u64, Ordering::SeqCst);
|
||||
|
||||
debug!("Stored chunk {} ({} bytes)", hash, data.len());
|
||||
debug!(hash = %hash, size_bytes = data.len(), "chunk stored");
|
||||
Ok(hash)
|
||||
}
|
||||
|
||||
@@ -111,6 +111,7 @@ impl CasStore {
|
||||
self.verify_integrity(hash, &data)?;
|
||||
}
|
||||
|
||||
debug!(hash = %hash, size_bytes = data.len(), "chunk retrieved");
|
||||
Ok(Bytes::from(data))
|
||||
}
|
||||
|
||||
@@ -156,7 +157,7 @@ impl CasStore {
|
||||
fs::remove_file(&path).await?;
|
||||
self.index.remove(hash.0.as_slice())?;
|
||||
self.current_size.fetch_sub(meta.len(), Ordering::SeqCst);
|
||||
debug!("Deleted chunk {}", hash);
|
||||
debug!(hash = %hash, size_bytes = meta.len(), "chunk deleted");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -19,5 +19,9 @@ clap.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
tracing-appender.workspace = true
|
||||
anyhow.workspace = true
|
||||
dirs.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
tracing-journald.workspace = true
|
||||
|
||||
@@ -2,7 +2,7 @@ use anyhow::{Context, Result};
|
||||
use clap::{Parser, Subcommand};
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_cas::{CasConfig, CasStore, ContentFetcher, FileReader};
|
||||
use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath};
|
||||
use musicfs_core::{FileId, FileMeta, LoggingConfig, OriginId, RealPath, VirtualPath};
|
||||
use musicfs_fuse::MusicFs;
|
||||
use musicfs_metadata::MetadataParser;
|
||||
use musicfs_origins::{LocalOrigin, Origin};
|
||||
@@ -10,6 +10,8 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::SystemTime;
|
||||
use tracing::{debug, info};
|
||||
use tracing_appender::non_blocking::WorkerGuard;
|
||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter, Layer};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "musicfs")]
|
||||
@@ -86,7 +88,6 @@ enum OriginCommands {
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
init_logging(&cli.log_level);
|
||||
|
||||
match cli.command {
|
||||
Commands::Mount {
|
||||
@@ -94,13 +95,38 @@ fn main() -> Result<()> {
|
||||
mountpoint,
|
||||
origin,
|
||||
cache_dir,
|
||||
} => run_mount(mountpoint, origin, cache_dir),
|
||||
Commands::Status => run_status(),
|
||||
Commands::Cache { command } => run_cache(command),
|
||||
Commands::Search { query, limit } => run_search(&query, limit),
|
||||
Commands::Origin { command } => run_origin(command),
|
||||
Commands::Events { r#type } => run_events(r#type),
|
||||
Commands::Shutdown { graceful, timeout } => run_shutdown(graceful, timeout),
|
||||
} => {
|
||||
let log_config = LoggingConfig {
|
||||
level: cli.log_level,
|
||||
..Default::default()
|
||||
};
|
||||
let _guard = init_logging(&log_config)?;
|
||||
run_mount(mountpoint, origin, cache_dir)
|
||||
}
|
||||
Commands::Status => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_status()
|
||||
}
|
||||
Commands::Cache { command } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_cache(command)
|
||||
}
|
||||
Commands::Search { query, limit } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_search(&query, limit)
|
||||
}
|
||||
Commands::Origin { command } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_origin(command)
|
||||
}
|
||||
Commands::Events { r#type } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_events(r#type)
|
||||
}
|
||||
Commands::Shutdown { graceful, timeout } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_shutdown(graceful, timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,9 +141,7 @@ fn run_mount(
|
||||
let handle = runtime.handle().clone();
|
||||
|
||||
let (tree, reader) = runtime.block_on(async {
|
||||
info!("MusicFS starting...");
|
||||
info!("Origin: {:?}", origin_path);
|
||||
info!("Mountpoint: {:?}", mountpoint);
|
||||
info!(origin = ?origin_path, mountpoint = ?mountpoint, "Mount configuration");
|
||||
|
||||
let cache_dir = cache_dir.unwrap_or_else(|| {
|
||||
dirs::cache_dir()
|
||||
@@ -240,13 +264,58 @@ fn run_shutdown(graceful: bool, timeout: u32) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn init_logging(level: &str) {
|
||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
|
||||
fn init_logging(config: &LoggingConfig) -> Result<WorkerGuard> {
|
||||
std::fs::create_dir_all(&config.log_dir)?;
|
||||
|
||||
let file_appender = tracing_appender::rolling::daily(&config.log_dir, "musicfs.log");
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
|
||||
let file_layer = if config.json_output {
|
||||
fmt::layer()
|
||||
.json()
|
||||
.with_writer(non_blocking)
|
||||
.with_ansi(false)
|
||||
.boxed()
|
||||
} else {
|
||||
fmt::layer()
|
||||
.with_writer(non_blocking)
|
||||
.with_ansi(false)
|
||||
.boxed()
|
||||
};
|
||||
|
||||
let stderr_layer = fmt::layer().with_writer(std::io::stderr).compact();
|
||||
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new(&config.level));
|
||||
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(file_layer)
|
||||
.with(stderr_layer);
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let subscriber = {
|
||||
let journald_layer = if config.journald {
|
||||
tracing_journald::layer()
|
||||
.ok()
|
||||
.map(|l| l.with_syslog_identifier("musicfs".to_string()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
subscriber.with(journald_layer)
|
||||
};
|
||||
|
||||
subscriber.init();
|
||||
|
||||
info!(version = env!("CARGO_PKG_VERSION"), "MusicFS starting");
|
||||
Ok(guard)
|
||||
}
|
||||
|
||||
fn init_basic_logging(level: &str) {
|
||||
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level));
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(fmt::layer())
|
||||
.with(fmt::layer().compact())
|
||||
.with(filter)
|
||||
.init();
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
toml.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tracing.workspace = true
|
||||
xxhash-rust.workspace = true
|
||||
hex.workspace = true
|
||||
|
||||
|
||||
@@ -14,6 +14,9 @@ pub struct Config {
|
||||
|
||||
#[serde(default)]
|
||||
pub health: HealthConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub logging: LoggingConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -120,6 +123,52 @@ fn default_unhealthy_threshold() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LoggingConfig {
|
||||
#[serde(default = "default_log_dir")]
|
||||
pub log_dir: PathBuf,
|
||||
|
||||
#[serde(default)]
|
||||
pub json_output: bool,
|
||||
|
||||
#[serde(default = "default_true")]
|
||||
pub journald: bool,
|
||||
|
||||
#[serde(default = "default_log_level")]
|
||||
pub level: String,
|
||||
|
||||
#[serde(default = "default_sample_rate")]
|
||||
pub trace_sample_rate: f32,
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
log_dir: default_log_dir(),
|
||||
json_output: false,
|
||||
journald: true,
|
||||
level: default_log_level(),
|
||||
trace_sample_rate: default_sample_rate(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_log_dir() -> PathBuf {
|
||||
PathBuf::from("/var/log/musicfs")
|
||||
}
|
||||
|
||||
fn default_log_level() -> String {
|
||||
"musicfs=info,warn".to_string()
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn default_sample_rate() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn from_file(path: &std::path::Path) -> Result<Self, ConfigError> {
|
||||
let content =
|
||||
|
||||
@@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use thiserror::Error;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CredentialStore {
|
||||
@@ -106,16 +107,36 @@ impl CredentialStore {
|
||||
origin_id: &str,
|
||||
config: &CredentialConfig,
|
||||
) -> Result<Credential, CredentialError> {
|
||||
debug!(origin_id = %origin_id, "Loading credentials");
|
||||
|
||||
if let Some(cred) = self.cache.get(origin_id) {
|
||||
trace!(origin_id = %origin_id, "Credential cache hit");
|
||||
return Ok(cred.clone());
|
||||
}
|
||||
|
||||
let cred = match config {
|
||||
CredentialConfig::Environment { prefix } => self.load_from_env(prefix)?,
|
||||
CredentialConfig::File { path } => self.load_from_file(path)?,
|
||||
CredentialConfig::Inline(cred) => cred.clone(),
|
||||
CredentialConfig::Environment { prefix } => {
|
||||
trace!(origin_id = %origin_id, prefix = %prefix, "Loading from environment");
|
||||
self.load_from_env(prefix)?
|
||||
}
|
||||
CredentialConfig::File { path } => {
|
||||
trace!(origin_id = %origin_id, path = ?path, "Loading from file");
|
||||
self.load_from_file(path)?
|
||||
}
|
||||
CredentialConfig::Inline(cred) => {
|
||||
trace!(origin_id = %origin_id, "Using inline credential");
|
||||
cred.clone()
|
||||
}
|
||||
};
|
||||
|
||||
let cred_type = match &cred {
|
||||
Credential::Basic { .. } => "Basic",
|
||||
Credential::AwsKey { .. } => "AwsKey",
|
||||
Credential::SshKey { .. } => "SshKey",
|
||||
Credential::EnvVar { .. } => "EnvVar",
|
||||
};
|
||||
info!(origin_id = %origin_id, cred_type = %cred_type, "Credential loaded");
|
||||
|
||||
self.cache.insert(origin_id.to_string(), cred.clone());
|
||||
Ok(cred)
|
||||
}
|
||||
@@ -144,6 +165,7 @@ impl CredentialStore {
|
||||
});
|
||||
}
|
||||
|
||||
warn!(prefix = %prefix, "No credentials found in environment");
|
||||
Err(CredentialError::NotFound(format!(
|
||||
"No credentials found with prefix {}",
|
||||
prefix
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::types::{FileId, OriginId, VirtualPath};
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, trace};
|
||||
|
||||
pub struct EventBus {
|
||||
sender: broadcast::Sender<Event>,
|
||||
@@ -12,7 +13,11 @@ impl EventBus {
|
||||
}
|
||||
|
||||
pub fn publish(&self, event: Event) {
|
||||
let _ = self.sender.send(event);
|
||||
trace!(event = ?event, "Publishing event");
|
||||
let receiver_count = self.sender.receiver_count();
|
||||
if self.sender.send(event).is_err() && receiver_count > 0 {
|
||||
debug!(receiver_count = receiver_count, "Event dropped, no active receivers");
|
||||
}
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<Event> {
|
||||
|
||||
@@ -6,7 +6,19 @@ pub mod metrics;
|
||||
pub mod resolver;
|
||||
pub mod types;
|
||||
|
||||
pub use config::{CacheConfig, Config, ConfigError, HealthConfig, OriginConfig, OriginType};
|
||||
pub use config::{
|
||||
CacheConfig, Config, ConfigError, HealthConfig, LoggingConfig, OriginConfig, OriginType,
|
||||
};
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub fn sanitize_path(path: &Path) -> String {
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
path.to_string_lossy().replace(&home, "~")
|
||||
} else {
|
||||
path.to_string_lossy().to_string()
|
||||
}
|
||||
}
|
||||
pub use credentials::{Credential, CredentialConfig, CredentialError, CredentialStore};
|
||||
pub use error::{Error, Result};
|
||||
pub use events::{Event, EventBus};
|
||||
|
||||
@@ -12,7 +12,7 @@ use std::path::Path;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::runtime::Handle;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, info, instrument, trace, warn};
|
||||
|
||||
const TTL: Duration = Duration::from_secs(1);
|
||||
const BLOCK_SIZE: u32 = 512;
|
||||
@@ -159,12 +159,12 @@ impl Filesystem for MusicFs {
|
||||
info!("MusicFS destroyed");
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) {
|
||||
debug!("lookup(parent={}, name={:?})", parent, name);
|
||||
|
||||
let name_str = name.to_string_lossy();
|
||||
|
||||
if parent == ROOT_INODE && SearchOps::is_search_dir_name(&name_str) {
|
||||
trace!(parent, name = %name_str, "search_dir_name matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.lookup_search_dir(reply);
|
||||
return;
|
||||
@@ -172,6 +172,7 @@ impl Filesystem for MusicFs {
|
||||
}
|
||||
|
||||
if parent == SearchOps::search_dir_inode() {
|
||||
trace!(parent, name = %name_str, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
let inode = self.get_or_create_query_inode(&name_str);
|
||||
search_ops.lookup_query_dir(&name_str, inode, reply);
|
||||
@@ -180,6 +181,7 @@ impl Filesystem for MusicFs {
|
||||
}
|
||||
|
||||
if let Some(query) = self.get_query_for_inode(parent) {
|
||||
trace!(parent, name = %name_str, query = %query, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
let inode = self.get_or_create_query_inode(&format!("{}:{}", query, name_str));
|
||||
search_ops.lookup_result(inode, reply);
|
||||
@@ -190,6 +192,7 @@ impl Filesystem for MusicFs {
|
||||
let tree = self.tree.read().unwrap();
|
||||
|
||||
if let Some(inode) = tree.lookup(parent, name) {
|
||||
trace!(parent, name = %name_str, ino = inode, "file found in tree");
|
||||
if let Some(node) = tree.get(inode) {
|
||||
let attr = self.node_to_attr(node);
|
||||
reply.entry(&TTL, &attr, 0);
|
||||
@@ -197,13 +200,14 @@ impl Filesystem for MusicFs {
|
||||
}
|
||||
}
|
||||
|
||||
trace!(parent, name = %name_str, "file not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) {
|
||||
debug!("getattr(ino={})", ino);
|
||||
|
||||
if ino == SearchOps::search_dir_inode() {
|
||||
trace!(ino, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_search_dir(reply);
|
||||
return;
|
||||
@@ -211,6 +215,7 @@ impl Filesystem for MusicFs {
|
||||
}
|
||||
|
||||
if SearchOps::is_search_inode(ino) {
|
||||
trace!(ino, "search_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_result(ino, reply);
|
||||
return;
|
||||
@@ -218,6 +223,7 @@ impl Filesystem for MusicFs {
|
||||
}
|
||||
|
||||
if self.get_query_for_inode(ino).is_some() {
|
||||
trace!(ino, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_search_dir(reply);
|
||||
return;
|
||||
@@ -227,13 +233,16 @@ impl Filesystem for MusicFs {
|
||||
let tree = self.tree.read().unwrap();
|
||||
|
||||
if let Some(node) = tree.get(ino) {
|
||||
trace!(ino, "inode found in tree");
|
||||
let attr = self.node_to_attr(node);
|
||||
reply.attr(&TTL, &attr);
|
||||
} else {
|
||||
trace!(ino, "inode not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn readdir(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
@@ -242,9 +251,8 @@ impl Filesystem for MusicFs {
|
||||
offset: i64,
|
||||
mut reply: ReplyDirectory,
|
||||
) {
|
||||
debug!("readdir(ino={}, offset={})", ino, offset);
|
||||
|
||||
if ino == SearchOps::search_dir_inode() {
|
||||
trace!(ino, offset, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readdir_search_root(offset, reply);
|
||||
return;
|
||||
@@ -252,6 +260,7 @@ impl Filesystem for MusicFs {
|
||||
}
|
||||
|
||||
if let Some(query) = self.get_query_for_inode(ino) {
|
||||
trace!(ino, offset, query = %query, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readdir_query(&query, offset, reply);
|
||||
return;
|
||||
@@ -261,6 +270,7 @@ impl Filesystem for MusicFs {
|
||||
let tree = self.tree.read().unwrap();
|
||||
|
||||
if let Some(children) = tree.readdir(ino) {
|
||||
trace!(ino, offset, children_count = children.len(), "directory found");
|
||||
let parent_ino = tree.get_parent(ino).unwrap_or(ROOT_INODE);
|
||||
|
||||
let entries: Vec<(u64, FileType, &str)> = vec![
|
||||
@@ -300,15 +310,16 @@ impl Filesystem for MusicFs {
|
||||
|
||||
reply.ok();
|
||||
} else {
|
||||
trace!(ino, offset, "directory not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn open(&mut self, _req: &Request, ino: u64, flags: i32, reply: ReplyOpen) {
|
||||
debug!("open(ino={}, flags={})", ino, flags);
|
||||
|
||||
let write_flags = libc::O_WRONLY | libc::O_RDWR | libc::O_APPEND | libc::O_TRUNC;
|
||||
if flags & write_flags != 0 {
|
||||
trace!(ino, flags, "write flags detected");
|
||||
reply.error(libc::EROFS);
|
||||
return;
|
||||
}
|
||||
@@ -316,12 +327,15 @@ impl Filesystem for MusicFs {
|
||||
let tree = self.tree.read().unwrap();
|
||||
|
||||
if tree.get(ino).is_some() {
|
||||
trace!(ino, "inode found");
|
||||
reply.opened(0, 0);
|
||||
} else {
|
||||
trace!(ino, "inode not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn read(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
@@ -333,19 +347,20 @@ impl Filesystem for MusicFs {
|
||||
_lock_owner: Option<u64>,
|
||||
reply: ReplyData,
|
||||
) {
|
||||
debug!("read(ino={}, offset={}, size={})", ino, offset, size);
|
||||
|
||||
let file_id = {
|
||||
let tree = self.tree.read().unwrap();
|
||||
if let Some(VirtualNode::File(file)) = tree.get(ino) {
|
||||
trace!(ino, "file found in tree");
|
||||
file.file_id
|
||||
} else {
|
||||
trace!(ino, "file not found");
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let Some(reader) = &self.reader else {
|
||||
trace!(ino, "no reader available");
|
||||
reply.data(&[]);
|
||||
return;
|
||||
};
|
||||
@@ -359,14 +374,18 @@ impl Filesystem for MusicFs {
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(data) => reply.data(&data),
|
||||
Ok(data) => {
|
||||
trace!(ino, offset, size_bytes = size, bytes_read = data.len(), "read successful");
|
||||
reply.data(&data);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Read error: {}", e);
|
||||
warn!(ino, offset, size_bytes = size, error = %e, "read failed");
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn release(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
@@ -377,7 +396,7 @@ impl Filesystem for MusicFs {
|
||||
_flush: bool,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
debug!("release(ino={})", ino);
|
||||
trace!(ino, "releasing file handle");
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ prost.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
chrono.workspace = true
|
||||
|
||||
@@ -10,7 +10,7 @@ use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, instrument};
|
||||
|
||||
pub struct MusicFsServer {
|
||||
start_time: Instant,
|
||||
@@ -206,10 +206,12 @@ impl MusicFs for MusicFsServer {
|
||||
))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _request), fields(method = "get_status"))]
|
||||
async fn get_status(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<StatusResponse>, Status> {
|
||||
debug!("gRPC get_status called");
|
||||
let uptime = self.start_time.elapsed().as_secs();
|
||||
|
||||
Ok(Response::new(StatusResponse {
|
||||
@@ -225,23 +227,27 @@ impl MusicFs for MusicFsServer {
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "shutdown"))]
|
||||
async fn shutdown(
|
||||
&self,
|
||||
request: Request<ShutdownRequest>,
|
||||
) -> Result<Response<Empty>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(
|
||||
"Shutdown requested (graceful={}, timeout={}s)",
|
||||
req.graceful, req.timeout_secs
|
||||
graceful = req.graceful,
|
||||
timeout_secs = req.timeout_secs,
|
||||
"gRPC shutdown requested"
|
||||
);
|
||||
|
||||
Ok(Response::new(Empty {}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _request), fields(method = "get_cache_stats"))]
|
||||
async fn get_cache_stats(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<CacheStats>, Status> {
|
||||
debug!("gRPC get_cache_stats called");
|
||||
Ok(Response::new(CacheStats {
|
||||
total_size_bytes: 0,
|
||||
used_size_bytes: 0,
|
||||
@@ -275,14 +281,17 @@ impl MusicFs for MusicFsServer {
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "clear_cache"))]
|
||||
async fn clear_cache(
|
||||
&self,
|
||||
request: Request<ClearCacheRequest>,
|
||||
) -> Result<Response<ClearCacheResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
debug!(
|
||||
"Clear cache requested: origin={:?}, metadata={}, chunks={}",
|
||||
req.origin_id, req.clear_metadata, req.clear_chunks
|
||||
info!(
|
||||
origin_id = ?req.origin_id,
|
||||
clear_metadata = req.clear_metadata,
|
||||
clear_chunks = req.clear_chunks,
|
||||
"gRPC clear_cache"
|
||||
);
|
||||
|
||||
Ok(Response::new(ClearCacheResponse {
|
||||
@@ -293,12 +302,14 @@ impl MusicFs for MusicFsServer {
|
||||
|
||||
type PrefetchStream = ReceiverStream<Result<PrefetchProgress, Status>>;
|
||||
|
||||
#[instrument(level = "debug", skip(self, request), fields(method = "prefetch"))]
|
||||
async fn prefetch(
|
||||
&self,
|
||||
request: Request<PrefetchRequest>,
|
||||
) -> Result<Response<Self::PrefetchStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
let total = req.paths.len() as u32;
|
||||
debug!(file_count = total, "gRPC prefetch started");
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
|
||||
@@ -319,18 +330,22 @@ impl MusicFs for MusicFsServer {
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _request), fields(method = "list_origins"))]
|
||||
async fn list_origins(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<OriginsResponse>, Status> {
|
||||
debug!("gRPC list_origins called");
|
||||
Ok(Response::new(OriginsResponse { origins: vec![] }))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, request), fields(method = "get_origin_health"))]
|
||||
async fn get_origin_health(
|
||||
&self,
|
||||
request: Request<OriginRequest>,
|
||||
) -> Result<Response<OriginHealthResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
debug!(origin_id = %req.origin_id, "gRPC get_origin_health");
|
||||
|
||||
Ok(Response::new(OriginHealthResponse {
|
||||
origin_id: req.origin_id,
|
||||
@@ -342,12 +357,13 @@ impl MusicFs for MusicFsServer {
|
||||
|
||||
type RescanOriginStream = ReceiverStream<Result<SyncProgress, Status>>;
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "rescan_origin"))]
|
||||
async fn rescan_origin(
|
||||
&self,
|
||||
request: Request<OriginRequest>,
|
||||
) -> Result<Response<Self::RescanOriginStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!("Rescan requested for origin: {}", req.origin_id);
|
||||
info!(origin_id = %req.origin_id, "gRPC rescan_origin started");
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
|
||||
@@ -373,16 +389,20 @@ impl MusicFs for MusicFsServer {
|
||||
|
||||
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "subscribe_events"))]
|
||||
async fn subscribe_events(
|
||||
&self,
|
||||
request: Request<EventFilter>,
|
||||
) -> Result<Response<Self::SubscribeEventsStream>, Status> {
|
||||
info!("gRPC subscribe_events: client connected");
|
||||
let filter = request.into_inner();
|
||||
let mut rx = self.event_bus.subscribe();
|
||||
let (tx, out_rx) = mpsc::channel(100);
|
||||
|
||||
tokio::spawn(async move {
|
||||
while let Ok(event) = rx.recv().await {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(event) => {
|
||||
if Self::matches_filter(&event, &filter) {
|
||||
let proto_event = Self::event_to_proto(&event);
|
||||
if tx.send(Ok(proto_event)).await.is_err() {
|
||||
@@ -390,6 +410,15 @@ impl MusicFs for MusicFsServer {
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
tracing::warn!(skipped = n, "Event subscriber lagged, skipped events");
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
tracing::debug!("Event channel closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(out_rx)))
|
||||
|
||||
@@ -2,7 +2,7 @@ use musicfs_core::Event;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WebhookPayload {
|
||||
@@ -11,9 +11,10 @@ pub struct WebhookPayload {
|
||||
pub data: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[derive(Clone, Deserialize)]
|
||||
pub struct WebhookConfig {
|
||||
pub url: String,
|
||||
#[serde(skip_serializing)]
|
||||
pub secret: Option<String>,
|
||||
pub events: Vec<String>,
|
||||
#[serde(default = "default_retry_count")]
|
||||
@@ -22,6 +23,18 @@ pub struct WebhookConfig {
|
||||
pub timeout_ms: u64,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for WebhookConfig {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("WebhookConfig")
|
||||
.field("url", &self.url)
|
||||
.field("secret", &self.secret.as_ref().map(|_| "[REDACTED]"))
|
||||
.field("events", &self.events)
|
||||
.field("retry_count", &self.retry_count)
|
||||
.field("timeout_ms", &self.timeout_ms)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
fn default_retry_count() -> u32 {
|
||||
3
|
||||
}
|
||||
@@ -30,29 +43,49 @@ fn default_timeout_ms() -> u64 {
|
||||
5000
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum WebhookError {
|
||||
#[error("Failed to initialize HTTP client: {0}")]
|
||||
ClientInit(String),
|
||||
}
|
||||
|
||||
pub struct WebhookHandler {
|
||||
client: reqwest::Client,
|
||||
configs: Vec<WebhookConfig>,
|
||||
}
|
||||
|
||||
impl WebhookHandler {
|
||||
pub fn new(configs: Vec<WebhookConfig>) -> Self {
|
||||
pub fn new(configs: Vec<WebhookConfig>) -> Result<Self, WebhookError> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("Failed to create HTTP client");
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "Failed to create webhook HTTP client");
|
||||
WebhookError::ClientInit(e.to_string())
|
||||
})?;
|
||||
|
||||
Self { client, configs }
|
||||
Ok(Self { client, configs })
|
||||
}
|
||||
|
||||
pub async fn run(&self, mut rx: broadcast::Receiver<Event>) {
|
||||
while let Ok(event) = rx.recv().await {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(event) => {
|
||||
for config in &self.configs {
|
||||
if self.matches_filter(&event, config) {
|
||||
self.dispatch(config, &event).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
warn!(skipped = n, "Webhook handler lagged, skipped events");
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => {
|
||||
debug!("Event channel closed, webhook handler stopping");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch(&self, config: &WebhookConfig, event: &Event) {
|
||||
@@ -129,8 +162,14 @@ impl WebhookHandler {
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
let body = serde_json::to_string(payload).unwrap_or_default();
|
||||
let mut mac =
|
||||
HmacSha256::new_from_slice(secret.as_bytes()).expect("HMAC key invalid");
|
||||
let mac = match HmacSha256::new_from_slice(secret.as_bytes()) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
error!(error = %e, "Invalid HMAC key for webhook signature");
|
||||
return String::new();
|
||||
}
|
||||
};
|
||||
let mut mac = mac;
|
||||
mac.update(body.as_bytes());
|
||||
let result = mac.finalize();
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ use crate::traits::Origin;
|
||||
use musicfs_core::{Error, RealPath, Result};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{trace, warn};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RetryConfig {
|
||||
@@ -79,6 +79,7 @@ impl FailoverExecutor {
|
||||
let mut last_error = None;
|
||||
|
||||
for origin in origins {
|
||||
trace!(origin_id = %origin.id(), "Attempting read from origin");
|
||||
let start = std::time::Instant::now();
|
||||
match self.read_with_retry(&origin, &path.path, offset, size).await {
|
||||
Ok(data) => {
|
||||
@@ -87,7 +88,7 @@ impl FailoverExecutor {
|
||||
return Ok(data);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Origin {} failed: {}, trying next", origin.id(), e);
|
||||
warn!(origin_id = %origin.id(), error = %e, "Origin failed, trying next");
|
||||
last_error = Some(e);
|
||||
}
|
||||
}
|
||||
@@ -108,13 +109,13 @@ impl FailoverExecutor {
|
||||
Ok(data) => return Ok(data),
|
||||
Err(e) if attempt + 1 < self.retry_config.max_attempts => {
|
||||
let delay = self.retry_config.delay_for_attempt(attempt);
|
||||
debug!(
|
||||
"Retry {}/{} for {} after {:?}: {}",
|
||||
attempt + 1,
|
||||
self.retry_config.max_attempts,
|
||||
origin.id(),
|
||||
delay,
|
||||
e
|
||||
warn!(
|
||||
origin_id = %origin.id(),
|
||||
attempt = attempt + 1,
|
||||
max_attempts = self.retry_config.max_attempts,
|
||||
error = %e,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Retrying read operation"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
@@ -142,6 +143,7 @@ impl FailoverExecutor {
|
||||
let mut last_error = None;
|
||||
|
||||
for origin in origins {
|
||||
trace!(origin_id = %origin.id(), "Attempting full read from origin");
|
||||
let start = std::time::Instant::now();
|
||||
match self.read_full_with_retry(&origin, &path.path).await {
|
||||
Ok(data) => {
|
||||
@@ -150,7 +152,7 @@ impl FailoverExecutor {
|
||||
return Ok(data);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Origin {} failed full read: {}, trying next", origin.id(), e);
|
||||
warn!(origin_id = %origin.id(), error = %e, "Origin failed full read, trying next");
|
||||
last_error = Some(e);
|
||||
}
|
||||
}
|
||||
@@ -169,13 +171,13 @@ impl FailoverExecutor {
|
||||
Ok(data) => return Ok(data),
|
||||
Err(e) if attempt + 1 < self.retry_config.max_attempts => {
|
||||
let delay = self.retry_config.delay_for_attempt(attempt);
|
||||
debug!(
|
||||
"Retry full read {}/{} for {} after {:?}: {}",
|
||||
attempt + 1,
|
||||
self.retry_config.max_attempts,
|
||||
origin.id(),
|
||||
delay,
|
||||
e
|
||||
warn!(
|
||||
origin_id = %origin.id(),
|
||||
attempt = attempt + 1,
|
||||
max_attempts = self.retry_config.max_attempts,
|
||||
error = %e,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Retrying full read operation"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, info, info_span, Instrument};
|
||||
|
||||
pub struct HealthMonitor {
|
||||
origins: DashMap<OriginId, Arc<dyn Origin>>,
|
||||
@@ -150,8 +150,16 @@ impl HealthMonitor {
|
||||
pub fn start(self: Arc<Self>) -> HealthCheckHandle {
|
||||
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
|
||||
let monitor = self.clone();
|
||||
let interval_secs = monitor.check_interval.as_secs();
|
||||
|
||||
tokio::spawn(async move {
|
||||
info!(
|
||||
interval_secs = interval_secs,
|
||||
origin_count = monitor.origins.len(),
|
||||
"Health monitor starting"
|
||||
);
|
||||
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let mut interval = tokio::time::interval(monitor.check_interval);
|
||||
|
||||
loop {
|
||||
@@ -165,7 +173,9 @@ impl HealthMonitor {
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
.instrument(info_span!("health_monitor")),
|
||||
);
|
||||
|
||||
HealthCheckHandle { stop_tx }
|
||||
}
|
||||
@@ -199,14 +209,24 @@ impl HealthMonitor {
|
||||
match status {
|
||||
HealthStatus::Healthy => {
|
||||
if state.status != HealthStatus::Healthy {
|
||||
info!("Origin {} is now healthy", id);
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to healthy"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Healthy;
|
||||
state.consecutive_failures = 0;
|
||||
}
|
||||
HealthStatus::Degraded => {
|
||||
if state.status != HealthStatus::Degraded {
|
||||
warn!("Origin {} is degraded", id);
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to degraded"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Degraded;
|
||||
}
|
||||
@@ -214,16 +234,22 @@ impl HealthMonitor {
|
||||
state.consecutive_failures += 1;
|
||||
if state.consecutive_failures >= threshold {
|
||||
if state.status != HealthStatus::Unhealthy {
|
||||
warn!(
|
||||
"Origin {} is now unhealthy ({} failures)",
|
||||
id, state.consecutive_failures
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
consecutive_failures = state.consecutive_failures,
|
||||
threshold = threshold,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to unhealthy"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Unhealthy;
|
||||
} else {
|
||||
debug!(
|
||||
"Origin {} check failed ({}/{})",
|
||||
id, state.consecutive_failures, threshold
|
||||
origin_id = %id,
|
||||
consecutive_failures = state.consecutive_failures,
|
||||
threshold = threshold,
|
||||
"Origin health check failed"
|
||||
);
|
||||
state.status = HealthStatus::Degraded;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use dashmap::DashMap;
|
||||
use musicfs_core::{Event, EventBus, OriginId};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
pub struct Router {
|
||||
priorities: DashMap<OriginId, u8>,
|
||||
@@ -77,7 +77,7 @@ impl Router {
|
||||
}
|
||||
|
||||
pub fn select(&self, candidates: &[OriginId], health: &HealthSnapshot) -> Option<OriginId> {
|
||||
candidates
|
||||
let selected = candidates
|
||||
.iter()
|
||||
.filter(|id| health.is_healthy(id))
|
||||
.min_by_key(|id| {
|
||||
@@ -85,7 +85,20 @@ impl Router {
|
||||
let latency = self.latency_stats.get(*id).map(|s| s.p50_ms).unwrap_or(0);
|
||||
(priority, latency)
|
||||
})
|
||||
.cloned()
|
||||
.cloned();
|
||||
|
||||
if let Some(ref id) = selected {
|
||||
let priority = self.get_priority(id);
|
||||
let latency = self.latency_stats.get(id).map(|s| s.p50_ms).unwrap_or(0);
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
priority = priority,
|
||||
latency_ms = latency,
|
||||
"Selected healthy origin"
|
||||
);
|
||||
}
|
||||
|
||||
selected
|
||||
}
|
||||
|
||||
pub fn select_with_fallback(
|
||||
@@ -104,6 +117,11 @@ impl Router {
|
||||
.min_by_key(|id| self.get_priority(id))
|
||||
.cloned()
|
||||
{
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
priority = self.get_priority(&id),
|
||||
"Selected degraded origin as fallback"
|
||||
);
|
||||
return Some(id);
|
||||
}
|
||||
|
||||
@@ -115,14 +133,26 @@ impl Router {
|
||||
});
|
||||
}
|
||||
|
||||
candidates
|
||||
let selected = candidates
|
||||
.iter()
|
||||
.min_by_key(|id| {
|
||||
let failures = health.failure_count(id).unwrap_or(u32::MAX);
|
||||
let priority = self.get_priority(id);
|
||||
(failures, priority)
|
||||
})
|
||||
.cloned()
|
||||
.cloned();
|
||||
|
||||
if let Some(ref id) = selected {
|
||||
let failures = health.failure_count(id).unwrap_or(u32::MAX);
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
failure_count = failures,
|
||||
priority = self.get_priority(id),
|
||||
"Selected least-bad unhealthy origin"
|
||||
);
|
||||
}
|
||||
|
||||
selected
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -275,6 +275,7 @@ impl Default for PluginManager {
|
||||
|
||||
impl Drop for PluginManager {
|
||||
fn drop(&mut self) {
|
||||
debug!(plugin_count = self.list().len(), "PluginManager dropping");
|
||||
let _ = self.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::warn;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SmartCollection {
|
||||
@@ -103,6 +103,7 @@ impl CollectionStore {
|
||||
[],
|
||||
)?;
|
||||
|
||||
info!(path = ?db_path, "Collection store opened");
|
||||
Ok(Self { db: Mutex::new(db) })
|
||||
}
|
||||
|
||||
@@ -111,6 +112,7 @@ impl CollectionStore {
|
||||
name: &str,
|
||||
query: CollectionQuery,
|
||||
) -> Result<SmartCollection, CollectionError> {
|
||||
info!(name = %name, "Creating collection");
|
||||
let query_json = serde_json::to_string(&query)?;
|
||||
let now = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
@@ -124,6 +126,7 @@ impl CollectionStore {
|
||||
)?;
|
||||
|
||||
let id = db.last_insert_rowid();
|
||||
debug!(id = id, name = %name, "Collection created");
|
||||
|
||||
Ok(SmartCollection {
|
||||
id,
|
||||
@@ -199,6 +202,7 @@ impl CollectionStore {
|
||||
}
|
||||
|
||||
pub fn delete(&self, name: &str) -> Result<(), CollectionError> {
|
||||
info!(name = %name, "Deleting collection");
|
||||
let db = self.db.lock();
|
||||
db.execute("DELETE FROM collections WHERE name = ?1", [name])?;
|
||||
Ok(())
|
||||
|
||||
@@ -2,7 +2,7 @@ use crate::index::{SearchError, SearchIndex};
|
||||
use musicfs_core::{Event, EventBus, FileMeta};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing::{debug, error, info, info_span, warn, Instrument};
|
||||
|
||||
pub trait MetadataLookup: Send + Sync {
|
||||
fn lookup(&self, path: &musicfs_core::VirtualPath) -> Option<FileMeta>;
|
||||
@@ -31,7 +31,10 @@ impl<M: MetadataLookup + 'static> Indexer<M> {
|
||||
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
|
||||
let mut event_rx = self.event_bus.subscribe();
|
||||
|
||||
tokio::spawn(async move {
|
||||
info!("Search indexer starting");
|
||||
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let mut pending_commit = false;
|
||||
let mut commit_timer = tokio::time::interval(std::time::Duration::from_secs(5));
|
||||
|
||||
@@ -45,8 +48,12 @@ impl<M: MetadataLookup + 'static> Indexer<M> {
|
||||
}
|
||||
pending_commit = true;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Event receive error: {}", e);
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
warn!(skipped = n, "Indexer lagged, skipped events");
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
debug!("Event channel closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -67,7 +74,9 @@ impl<M: MetadataLookup + 'static> Indexer<M> {
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
.instrument(info_span!("search_indexer")),
|
||||
);
|
||||
|
||||
IndexerHandle { stop_tx }
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use musicfs_origins::Origin;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::time::SystemTime;
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, trace};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScannedFile {
|
||||
@@ -66,9 +66,13 @@ impl DeltaDetector {
|
||||
cached: &HashMap<FileId, FileMeta>,
|
||||
manifests: &HashMap<FileId, Vec<ManifestChunk>>,
|
||||
) -> Result<ChangeSet, DeltaError> {
|
||||
let origin_id = origin.id().clone();
|
||||
info!(origin_id = %origin_id, "Starting delta detection");
|
||||
|
||||
let mut changes = ChangeSet::default();
|
||||
|
||||
let origin_files = self.scan_origin(origin).await?;
|
||||
trace!(origin_id = %origin_id, scanned_count = origin_files.len(), "Completed origin scan");
|
||||
|
||||
let cached_by_path: HashMap<_, _> = cached
|
||||
.values()
|
||||
@@ -78,7 +82,7 @@ impl DeltaDetector {
|
||||
for scanned in &origin_files {
|
||||
if let Some(cached_file) = cached_by_path.get(&scanned.path) {
|
||||
if self.is_modified_scan(cached_file, scanned) {
|
||||
debug!("File modified: {:?}", scanned.path);
|
||||
debug!(origin_id = %origin_id, path = ?scanned.path, "File modified");
|
||||
|
||||
if let Some(old_chunks) = manifests.get(&cached_file.id) {
|
||||
let new_chunks = self.compute_chunks_for_scan(origin, scanned).await?;
|
||||
@@ -87,7 +91,7 @@ impl DeltaDetector {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
debug!("File added: {:?}", scanned.path);
|
||||
debug!(origin_id = %origin_id, path = ?scanned.path, "File added");
|
||||
changes.added.push(scanned.clone());
|
||||
}
|
||||
}
|
||||
@@ -96,16 +100,17 @@ impl DeltaDetector {
|
||||
|
||||
for cached_file in cached.values() {
|
||||
if !origin_paths.contains(&cached_file.real_path.path) {
|
||||
debug!("File removed: {:?}", cached_file.real_path.path);
|
||||
debug!(origin_id = %origin_id, path = ?cached_file.real_path.path, "File removed");
|
||||
changes.removed.push(cached_file.id);
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"Delta detection complete: {} added, {} removed, {} modified",
|
||||
changes.added.len(),
|
||||
changes.removed.len(),
|
||||
changes.modified.len()
|
||||
origin_id = %origin_id,
|
||||
files_added = changes.added.len(),
|
||||
files_removed = changes.removed.len(),
|
||||
files_modified = changes.modified.len(),
|
||||
"Delta detection complete"
|
||||
);
|
||||
|
||||
Ok(changes)
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, error, info};
|
||||
use tracing::{error, info, info_span, trace, Instrument};
|
||||
|
||||
const DEBOUNCE_MS: u64 = 200;
|
||||
|
||||
@@ -31,11 +31,15 @@ impl OriginWatcher {
|
||||
let root = self.root.clone();
|
||||
let event_bus = self.event_bus.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let origin_id_str = origin_id.to_string();
|
||||
tokio::spawn(
|
||||
async move {
|
||||
if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await {
|
||||
error!("Watcher error: {}", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
.instrument(info_span!("file_watcher", origin_id = %origin_id_str)),
|
||||
);
|
||||
|
||||
WatchHandle { stop_tx }
|
||||
}
|
||||
@@ -62,7 +66,7 @@ impl OriginWatcher {
|
||||
.watch(root, RecursiveMode::Recursive)
|
||||
.map_err(|e| WatchError::Watch(e.to_string()))?;
|
||||
|
||||
info!("Watching origin {} at {:?}", origin_id, root);
|
||||
info!(origin_id = %origin_id, path = ?root, "Watcher started");
|
||||
|
||||
let mut debouncer: HashMap<PathBuf, Instant> = HashMap::new();
|
||||
|
||||
@@ -72,7 +76,7 @@ impl OriginWatcher {
|
||||
Self::handle_notify_event(origin_id, root, event_bus, event, &mut debouncer);
|
||||
}
|
||||
_ = stop_rx.recv() => {
|
||||
info!("Stopping watcher for {}", origin_id);
|
||||
info!(origin_id = %origin_id, "Watcher stopped");
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -104,7 +108,7 @@ impl OriginWatcher {
|
||||
|
||||
if let Some(last_seen) = debouncer.get(&relative) {
|
||||
if now.duration_since(*last_seen).as_millis() < DEBOUNCE_MS as u128 {
|
||||
debug!("Debouncing event for {:?}", relative);
|
||||
trace!(origin_id = %origin_id, path = ?relative, "Debouncing event");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -114,18 +118,18 @@ impl OriginWatcher {
|
||||
|
||||
match event.kind {
|
||||
EventKind::Create(_) => {
|
||||
debug!("File created: {:?}", relative);
|
||||
trace!(origin_id = %origin_id, path = ?relative, "File created");
|
||||
event_bus.publish(Event::FileAdded {
|
||||
path: vpath,
|
||||
origin_id: origin_id.clone(),
|
||||
});
|
||||
}
|
||||
EventKind::Remove(_) => {
|
||||
debug!("File removed: {:?}", relative);
|
||||
trace!(origin_id = %origin_id, path = ?relative, "File removed");
|
||||
event_bus.publish(Event::FileRemoved { path: vpath, file_id: None });
|
||||
}
|
||||
EventKind::Modify(_) => {
|
||||
debug!("File modified: {:?}", relative);
|
||||
trace!(origin_id = %origin_id, path = ?relative, "File modified");
|
||||
event_bus.publish(Event::FileModified { path: vpath });
|
||||
}
|
||||
_ => {}
|
||||
@@ -156,6 +160,7 @@ impl WatchHandle {
|
||||
|
||||
impl Drop for WatchHandle {
|
||||
fn drop(&mut self) {
|
||||
trace!("WatchHandle dropped");
|
||||
let _ = self.stop_tx.try_send(());
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+30
@@ -0,0 +1,30 @@
|
||||
mount_point = "/mnt/music"
|
||||
cache_dir = "/var/cache/musicfs"
|
||||
|
||||
[logging]
|
||||
log_dir = "/var/log/musicfs"
|
||||
json_output = true
|
||||
journald = true
|
||||
level = "musicfs=info,warn"
|
||||
trace_sample_rate = 1.0
|
||||
|
||||
[cache]
|
||||
metadata_cache_mb = 100
|
||||
content_cache_gb = 10
|
||||
|
||||
[health]
|
||||
check_interval_secs = 30
|
||||
timeout_ms = 5000
|
||||
unhealthy_threshold = 3
|
||||
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/srv/music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas"
|
||||
origin_type = "nfs"
|
||||
priority = 2
|
||||
mount_point = "/mnt/nas/music"
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
/var/log/musicfs/*.log {
|
||||
daily
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 musicfs musicfs
|
||||
}
|
||||
Vendored
+8
-1
@@ -11,10 +11,17 @@ RestartSec=5
|
||||
User=musicfs
|
||||
Group=musicfs
|
||||
|
||||
Environment="RUST_LOG=musicfs=info,warn"
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=musicfs
|
||||
RateLimitIntervalSec=30s
|
||||
RateLimitBurst=1000
|
||||
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=read-only
|
||||
ReadWritePaths=/var/cache/musicfs /mnt/music
|
||||
ReadWritePaths=/var/cache/musicfs /var/log/musicfs /mnt/music
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
|
||||
Reference in New Issue
Block a user