diff --git a/docs/v2/mvp-performance-review.md b/docs/v2/mvp-performance-review.md new file mode 100644 index 0000000..b88e6e4 --- /dev/null +++ b/docs/v2/mvp-performance-review.md @@ -0,0 +1,239 @@ +# MusicFS MVP Performance Review + +**Date**: 2026-05-12 +**Test Data**: Metallica - 72 Seasons (12 FLAC tracks, 625MB, 16-bit/44.1kHz) +**Origin**: Local filesystem (Docker volume) +**System**: Linux, NixOS + +--- + +## Executive Summary + +**Phase 1 MVP is functional** - the system mounts, browses, and reads files successfully. Audio playback works with valid FLAC headers served. However, there's a **critical gap** between the architecture specification and current implementation regarding content chunking. + +--- + +## Benchmark Results + +### Throughput Comparison + +| Metric | Direct FS | MusicFS Cold | MusicFS Warm | Target (Spec) | Status | +|--------|-----------|--------------|--------------|---------------|--------| +| Single file read (64MB) | 0.022s (3 GB/s) | 0.035s (1.8 GB/s) | 0.020s (3.2 GB/s) | >500 MB/s | ✅ | +| Full album read (625MB) | 0.149s (4.2 GB/s) | 0.274s (2.3 GB/s) | 0.211s (3.0 GB/s) | >500 MB/s | ✅ | + +### Metadata Operations + +| Operation | Result | Target (Spec) | Status | +|-----------|--------|---------------|--------| +| Root listing | 0.006s | <10ms | ✅ | +| Full tree traversal (12 files) | 0.007s | <50ms | ✅ | +| stat() per operation | 0.003s | <1ms | ⚠️ | +| 4KB small reads (per op) | 0.006s | <1ms | ⚠️ | +| Random seek 1MB | 0.008-0.015s | <50ms | ✅ | +| Mount time | ~8ms | <500ms | ✅ | + +### Cache Performance + +| Metric | Value | +|--------|-------| +| Cache speedup (single file) | 1.75x | +| Cache speedup (full album) | 1.30x | +| Cache size | 25MB | +| Chunk count | 12 | +| Expected cache size | 625MB | + +### FUSE Overhead + +| Scenario | Overhead vs Direct | +|----------|-------------------| +| Single file cold cache | 59% slower | +| Single file warm cache | 9% faster* | +| Full album cold cache | 84% slower | +| Full album warm cache | 42% slower | + +*Warm cache appears faster due to OS page cache effects on both paths. + +--- + +## What's Working Well ✅ + +### 1. Mount Performance +- Mount completes in ~8ms (spec: <500ms) — **62x better than target** +- O(1) mount time achieved — no file scanning blocks mount +- Lazy loading working as designed per architecture section 4.3.1 + +### 2. Virtual Tree Organization +- Correct Artist/Album/Track hierarchy derived from metadata +- Example path: `/Metallica/72 Seasons/01. 72 Seasons.flac` +- Special character sanitization working (`/`, `\`, `:`, etc.) + +### 3. File Reading +- Valid FLAC headers served (`fLaC` magic bytes verified) +- Sequential reads work correctly +- Random access (seek) functional +- Concurrent reads from multiple processes work + +### 4. FUSE Integration +- Read-only enforcement (EROFS returned on write attempts) +- Proper inode assignment and file attributes +- AllowOther mount option working +- Clean unmount via fusermount3 + +### 5. Throughput +- Exceeds 500 MB/s target significantly (2-3 GB/s achieved) +- Parallel reads scale appropriately (4 files in 0.060s) + +--- + +## Critical Issues 🔴 + +### Issue 1: Incomplete File Caching + +**Symptom**: Cache is 25MB instead of expected 625MB (12 files × ~2MB each instead of full files) + +**Root Cause**: In `fetcher.rs:74`: +```rust +let data = origin.read(&meta.real_path.path, 0, meta.size as u32).await?; +``` + +And in `local.rs:96-98`: +```rust +let mut buffer = vec![0u8; size as usize]; +let bytes_read = file.read(&mut buffer).await?; +buffer.truncate(bytes_read); +``` + +`tokio::fs::File::read()` reads **up to** buffer size but returns when the kernel buffer is exhausted (~2MB typical). Only first ~2MB of each file is being cached. + +**Impact**: +- Subsequent reads beyond 2MB offset hit origin every time +- No cache benefit for majority of file content +- Cache eviction policy not being exercised + +**Required Fix**: Use `read_to_end()` or loop until all bytes read: +```rust +let mut buffer = Vec::with_capacity(size as usize); +file.read_to_end(&mut buffer).await?; +``` + +### Issue 2: No CDC Chunking Implemented + +**Architecture Spec** (Section 4.3.2): +> "All file content is stored as content-addressed chunks... Avg chunk: 64KB, Min: 16KB, Max: 256KB" + +**Current Implementation**: Each file stored as ONE chunk (no FastCDC integration) + +**Impact**: +- No content deduplication possible +- Delta sync impossible (FR-11.2 unmet) +- Cache efficiency severely reduced for similar files + +--- + +## Architecture Gaps 🟡 + +| Spec Requirement | Current State | Gap | +|------------------|---------------|-----| +| CDC chunking (64KB avg) | No chunking | Missing FastCDC integration | +| Delta sync (>90% bandwidth reduction) | Not implemented | Requires CDC first | +| Deduplication (FR-20) | Not implemented | Requires CDC first | +| Search engine (tantivy) | Not implemented | Phase 3 scope | +| gRPC Control API | Not implemented | Phase 4 scope | +| Multi-origin federation | Single origin only | Phase 2 scope | +| Metadata persistence (SQLite) | In-memory HashMap | Missing persistence | + +--- + +## Performance Analysis + +### Why Warm Cache Appears Faster Than Direct FS + +The warm cache shows 3.2 GB/s vs direct 3.0 GB/s because: +1. OS page cache is warm for both MusicFS chunks AND origin files +2. Both measurements are essentially hitting RAM, variance expected +3. MusicFS chunks may have slightly better cache locality + +### stat() Latency Above Target + +Current: 3ms per stat() vs target <1ms + +Possible causes: +1. `RwLock` contention overhead +2. HashMap lookup plus FUSE context switch +3. Measurement includes full round-trip through FUSE + +Mitigation options: +- Consider lock-free concurrent data structures +- Implement finer-grained locking +- Cache hot inodes in separate fast-path structure + +--- + +## Recommendations + +### Immediate Fixes (Before Phase 2) + +1. **Fix file reading** — Use `read_to_end()` or implement proper streaming read loop +2. **Add CDC chunking** — Integrate FastCDC per architecture spec section 4.3.2 +3. **Persist metadata** — Move from in-memory HashMap to SQLite as specified + +### Phase 2 Priorities + +1. Complete CDC chunking implementation (prerequisite for delta sync) +2. Add SQLite metadata persistence (FR-7.2) +3. Implement multi-origin support (FR-13) + +### Testing Gaps to Address + +1. No automated E2E tests for real FUSE operations +2. No stress testing with concurrent access patterns +3. No large library testing (target: 1M+ files per NFR-3.1) +4. No offline mode testing (origin unavailable scenarios) + +--- + +## Test Environment Details + +``` +Origin Path: /home/fujin/.local/share/docker/volumes/containers_downloads/_data/Metallica - 72 Seasons (2023) [FLAC] 88/ +Mount Point: /tmp/musicfs-benchmark/mount +Cache Dir: /tmp/musicfs-benchmark/cache +Binary: target/release/musicfs (via nix develop) + +Files: + 01. 72 Seasons.flac 64MB + 02. Shadows Follow.flac 50MB + 03. Screaming Suicide.flac 45MB + 04. Sleepwalk My Life Away.flac 54MB + 05. You Must Burn!.flac 57MB + 06. Lux Æterna.flac 27MB + 07. Crown Of Barbed Wire.flac 46MB + 08. Chasing Light.flac 55MB + 09. If Darkness Had A Son.flac 51MB + 10. Too Far Gone_.flac 37MB + 11. Room Of Mirrors.flac 45MB + 12. Inamorata.flac 89MB + Total: 625MB, 12 tracks +``` + +--- + +## Conclusion + +**The MVP demonstrates core functionality works** — mounting, browsing, and reading audio files through FUSE. Throughput performance exceeds targets significantly. + +However, **the cache implementation is incomplete**: +- Only ~4% of file content is being cached (25MB/625MB) +- No CDC chunking means no deduplication or delta sync capability +- Architecture requirements FR-8.2, FR-11.2, FR-20 are unmet + +**Recommendation**: Fix the file reading issue and add CDC chunking before proceeding to Phase 2. The architecture is sound; implementation needs to catch up to specification. + +--- + +## References + +- [Architecture Specification](architecture.md) — Section 4.3.2 (CAS), Section 4.3.5 (Read Flow) +- [Requirements Specification](requirements.md) — FR-8 (Content Cache), FR-11 (Delta Sync), FR-20 (CAS) +- [Week 4b Plan](plans/week-04b-origin-connector.md) — ContentFetcher implementation diff --git a/musicfs/Cargo.lock b/musicfs/Cargo.lock index 847279b..90d634a 100644 --- a/musicfs/Cargo.lock +++ b/musicfs/Cargo.lock @@ -14,6 +14,65 @@ dependencies = [ "zerocopy 0.8.48", ] +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -89,6 +148,52 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "crc32fast" version = "1.5.0" @@ -325,6 +430,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" @@ -390,6 +501,15 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "memchr" version = "2.8.0" @@ -446,6 +566,20 @@ dependencies = [ [[package]] name = "musicfs-cli" version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "dirs", + "musicfs-cache", + "musicfs-cas", + "musicfs-core", + "musicfs-fuse", + "musicfs-metadata", + "musicfs-origins", + "tokio", + "tracing", + "tracing-subscriber", +] [[package]] name = "musicfs-core" @@ -508,6 +642,15 @@ version = "0.1.0" name = "musicfs-sync" version = "0.1.0" +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -523,6 +666,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "option-ext" version = "0.2.0" @@ -662,6 +811,23 @@ dependencies = [ "thiserror", ] +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + [[package]] name = "rmp" version = "0.8.15" @@ -763,6 +929,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -811,6 +986,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "symphonia" version = "0.5.5" @@ -988,6 +1169,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "tokio" version = "1.52.3" @@ -1045,6 +1235,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -1059,6 +1279,18 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/musicfs/Cargo.toml b/musicfs/Cargo.toml index 9fa740e..9111ca3 100644 --- a/musicfs/Cargo.toml +++ b/musicfs/Cargo.toml @@ -50,5 +50,8 @@ bytes = "1" # Platform directories dirs = "5" +# CLI +clap = { version = "4", features = ["derive"] } + # Testing tempfile = "3" diff --git a/musicfs/crates/musicfs-cli/Cargo.toml b/musicfs/crates/musicfs-cli/Cargo.toml index 8e52f26..1a7902f 100644 --- a/musicfs/crates/musicfs-cli/Cargo.toml +++ b/musicfs/crates/musicfs-cli/Cargo.toml @@ -8,3 +8,16 @@ name = "musicfs" path = "src/main.rs" [dependencies] +musicfs-core.path = "../musicfs-core" +musicfs-origins.path = "../musicfs-origins" +musicfs-cache.path = "../musicfs-cache" +musicfs-cas.path = "../musicfs-cas" +musicfs-fuse.path = "../musicfs-fuse" +musicfs-metadata.path = "../musicfs-metadata" + +clap.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +anyhow.workspace = true +dirs.workspace = true diff --git a/musicfs/crates/musicfs-cli/src/main.rs b/musicfs/crates/musicfs-cli/src/main.rs index d0c1637..7471200 100644 --- a/musicfs/crates/musicfs-cli/src/main.rs +++ b/musicfs/crates/musicfs-cli/src/main.rs @@ -1,3 +1,195 @@ -fn main() { - println!("MusicFS CLI - placeholder"); +use anyhow::{Context, Result}; +use clap::Parser; +use musicfs_cache::TreeBuilder; +use musicfs_cas::{CasConfig, CasStore, ContentFetcher, FileReader}; +use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath}; +use musicfs_fuse::MusicFs; +use musicfs_metadata::MetadataParser; +use musicfs_origins::{LocalOrigin, Origin}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; +use std::time::SystemTime; +use tracing::{debug, info}; + +#[derive(Parser)] +#[command(name = "musicfs")] +#[command(about = "Virtual FUSE filesystem for music libraries")] +struct Cli { + #[arg(help = "Mount point for the virtual filesystem")] + mountpoint: PathBuf, + + #[arg(short, long, help = "Source music directory (origin)")] + origin: PathBuf, + + #[arg(short, long, help = "Cache directory for CAS chunks")] + cache_dir: Option, + + #[arg(short, long, default_value = "info", help = "Log level (debug, info, warn, error)")] + log_level: String, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + init_logging(&cli.log_level); + + let runtime = tokio::runtime::Runtime::new().context("Failed to create Tokio runtime")?; + let handle = runtime.handle().clone(); + + let (tree, reader) = runtime.block_on(async { + info!("MusicFS starting..."); + info!("Origin: {:?}", cli.origin); + info!("Mountpoint: {:?}", cli.mountpoint); + + let cache_dir = cli.cache_dir.unwrap_or_else(|| { + dirs::cache_dir() + .unwrap_or_else(|| PathBuf::from("/tmp")) + .join("musicfs") + }); + info!("Cache directory: {:?}", cache_dir); + + std::fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?; + std::fs::create_dir_all(&cli.mountpoint).context("Failed to create mountpoint")?; + + let cas_config = CasConfig { + chunks_dir: cache_dir.join("chunks"), + ..Default::default() + }; + let store = Arc::new(CasStore::open(cas_config).await.context("Failed to open CAS store")?); + info!("CAS store initialized"); + + let origin_id = OriginId::from("local"); + let origin = Arc::new(LocalOrigin::new(origin_id.clone(), cli.origin.clone())); + info!("Origin registered: {}", origin.display_name()); + + let fetcher = Arc::new(ContentFetcher::new(store.clone())); + fetcher.register_origin(origin); + + info!("Scanning music files..."); + let files = scan_music_files(&cli.origin, &origin_id).await?; + info!("Found {} music files", files.len()); + + let mut builder = TreeBuilder::new(); + for file in &files { + builder.add_file(file); + fetcher.register_file(file.clone()); + } + let tree = Arc::new(RwLock::new(builder.build())); + info!("Virtual tree built"); + + let reader = Arc::new(FileReader::with_fetcher(store, fetcher)); + + Ok::<_, anyhow::Error>((tree, reader)) + })?; + + let fs = MusicFs::with_reader(tree, reader, handle); + + info!("Mounting filesystem at {:?}", cli.mountpoint); + info!("Press Ctrl+C to unmount"); + + fs.mount(&cli.mountpoint).context("Failed to mount filesystem")?; + + Ok(()) +} + +fn init_logging(level: &str) { + use tracing_subscriber::{fmt, prelude::*, EnvFilter}; + + let filter = EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new(level)); + + tracing_subscriber::registry() + .with(fmt::layer()) + .with(filter) + .init(); +} + +async fn scan_music_files(dir: &Path, origin_id: &OriginId) -> Result> { + let parser = MetadataParser::new(); + let mut files = Vec::new(); + let mut file_id_counter = 1i64; + + scan_dir_recursive(dir, dir, origin_id, &parser, &mut files, &mut file_id_counter).await?; + + Ok(files) +} + +async fn scan_dir_recursive( + base: &Path, + dir: &Path, + origin_id: &OriginId, + parser: &MetadataParser, + files: &mut Vec, + id_counter: &mut i64, +) -> Result<()> { + let mut entries = tokio::fs::read_dir(dir).await?; + + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let metadata = entry.metadata().await?; + + if metadata.is_dir() { + Box::pin(scan_dir_recursive(base, &path, origin_id, parser, files, id_counter)).await?; + } else if is_audio_file(&path) { + let relative_path = path.strip_prefix(base).unwrap_or(&path); + + let audio_meta = match parser.parse_file(&path) { + Ok(meta) => Some(meta), + Err(e) => { + debug!("Failed to parse metadata for {:?}: {}", path, e); + None + } + }; + + let virtual_path = build_virtual_path(&path, audio_meta.as_ref()); + + let file_meta = FileMeta { + id: FileId(*id_counter), + virtual_path, + real_path: RealPath { + origin_id: origin_id.clone(), + path: PathBuf::from("/").join(relative_path), + }, + size: metadata.len(), + mtime: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH), + content_hash: None, + audio: audio_meta, + }; + + debug!("Found: {:?} -> {:?}", file_meta.real_path.path, file_meta.virtual_path); + files.push(file_meta); + *id_counter += 1; + } + } + + Ok(()) +} + +fn is_audio_file(path: &Path) -> bool { + matches!( + path.extension().and_then(|e| e.to_str()).map(|e| e.to_lowercase()).as_deref(), + Some("flac" | "mp3" | "ogg" | "wav" | "m4a" | "aac" | "opus") + ) +} + +fn build_virtual_path(path: &Path, audio: Option<&musicfs_core::AudioMeta>) -> VirtualPath { + if let Some(meta) = audio { + let artist = meta.artist.as_deref().unwrap_or("Unknown Artist"); + let album = meta.album.as_deref().unwrap_or("Unknown Album"); + let filename = path.file_name().and_then(|n| n.to_str()).unwrap_or("track"); + + VirtualPath::new(&format!("/{}/{}/{}", sanitize(artist), sanitize(album), filename)) + } else { + let filename = path.file_name().and_then(|n| n.to_str()).unwrap_or("unknown"); + VirtualPath::new(&format!("/Unknown Artist/Unknown Album/{}", filename)) + } +} + +fn sanitize(s: &str) -> String { + s.chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_', + _ => c, + }) + .collect() } diff --git a/musicfs/crates/musicfs-fuse/src/filesystem.rs b/musicfs/crates/musicfs-fuse/src/filesystem.rs index 6262bec..de56177 100644 --- a/musicfs/crates/musicfs-fuse/src/filesystem.rs +++ b/musicfs/crates/musicfs-fuse/src/filesystem.rs @@ -9,6 +9,7 @@ use std::ffi::OsStr; use std::path::Path; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime}; +use tokio::runtime::Handle; use tracing::{debug, info, warn}; const TTL: Duration = Duration::from_secs(1); @@ -17,24 +18,27 @@ const BLOCK_SIZE: u32 = 512; pub struct MusicFs { tree: Arc>, reader: Option>, + runtime_handle: Handle, uid: u32, gid: u32, } impl MusicFs { - pub fn new(tree: Arc>) -> Self { + pub fn new(tree: Arc>, runtime_handle: Handle) -> Self { Self { tree, reader: None, + runtime_handle, uid: unsafe { libc::getuid() }, gid: unsafe { libc::getgid() }, } } - pub fn with_reader(tree: Arc>, reader: Arc) -> Self { + pub fn with_reader(tree: Arc>, reader: Arc, runtime_handle: Handle) -> Self { Self { tree, reader: Some(reader), + runtime_handle, uid: unsafe { libc::getuid() }, gid: unsafe { libc::getgid() }, } @@ -241,8 +245,11 @@ impl Filesystem for MusicFs { }; let reader = reader.clone(); - let result = tokio::runtime::Handle::current().block_on(async { - reader.read(file_id, offset as u64, size).await + let handle = self.runtime_handle.clone(); + let result = std::thread::scope(|_| { + handle.block_on(async { + reader.read(file_id, offset as u64, size).await + }) }); match result { @@ -410,11 +417,14 @@ mod tests { #[test] fn test_tree_integration() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let handle = runtime.handle().clone(); + let mut builder = TreeBuilder::new(); builder.add_file(&make_file_meta(1, "/Artist/Album/Track.flac", 30_000_000)); let tree = Arc::new(RwLock::new(builder.build())); - let _fs = MusicFs::new(tree.clone()); + let _fs = MusicFs::new(tree.clone(), handle); let tree_read = tree.read().unwrap(); assert!(tree_read.get(ROOT_INODE).is_some()); diff --git a/musicfs/flake.nix b/musicfs/flake.nix index 20e9454..d909c77 100644 --- a/musicfs/flake.nix +++ b/musicfs/flake.nix @@ -18,7 +18,7 @@ }; in { - devShells.default = pkgs.mkShell { + devShells.default = pkgs.mkShell rec { buildInputs = with pkgs; [ rustToolchain pkg-config @@ -26,20 +26,18 @@ sqlite openssl - # Linker toolchain clang lld - # Dev tools cargo-watch cargo-nextest cargo-criterion - # gRPC tooling (Week 10+) protobuf grpcurl ]; + LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs; RUST_BACKTRACE = "1"; RUST_LOG = "debug"; };