Move the files around

This commit is contained in:
Alexander
2026-05-13 20:34:14 +02:00
parent 90e9683076
commit 305d027c8b
113 changed files with 650 additions and 3569 deletions
+33
View File
@@ -14,4 +14,37 @@ tests/*.log
# Nix # Nix
result result
.cargo/
.direnv/
.pre-commit-config.yaml
dist/
###
# Rust
###
result-* result-*
# Generated by Cargo
# will have compiled files and executables
debug
target
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
# Generated by cargo mutants
# Contains mutation testing data
**/mutants.out*/
# rustc will dump stack traces when hitting an internal compiler error to PWD
rustc-ice-*.txt
# RustRover
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
View File
View File
-7
View File
@@ -1,7 +0,0 @@
Organising a music library can be a hassle. With the wealth of online stores all providing music tagged in various formats, it can be a nightmare to unify them all.
This is where beetFs comes in. Derived from beets, beetFs presents a FUSE filesystem that is based on your tags.
Modifying the tags within the beetFs mountpoint will not change the data on the hard disk, merely update the beet database. When an application requests a music file from within the beetFs mountpoint, beetFs provides tag information from its own database, instead of from the original file, but music data from the on-disk location.
This enables completely transparent modification of tags within an audio file with no change to the underlying on-disk data.
-2
View File
@@ -1,2 +0,0 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
-1144
View File
File diff suppressed because it is too large Load Diff
@@ -48,9 +48,18 @@ impl ArtworkCache {
} }
pub async fn store(&self, file_id: i64, artwork: &Artwork) -> Result<ChunkHash, ArtworkError> { pub async fn store(&self, file_id: i64, artwork: &Artwork) -> Result<ChunkHash, ArtworkError> {
trace!(file_id = file_id, size_bytes = artwork.data.len(), "Storing artwork"); trace!(
file_id = file_id,
size_bytes = artwork.data.len(),
"Storing artwork"
);
if artwork.data.len() > MAX_ARTWORK_INPUT_SIZE { if artwork.data.len() > MAX_ARTWORK_INPUT_SIZE {
warn!(file_id = file_id, size = artwork.data.len(), max = MAX_ARTWORK_INPUT_SIZE, "Artwork too large"); warn!(
file_id = file_id,
size = artwork.data.len(),
max = MAX_ARTWORK_INPUT_SIZE,
"Artwork too large"
);
return Err(ArtworkError::ImageTooLarge(artwork.data.len())); return Err(ArtworkError::ImageTooLarge(artwork.data.len()));
} }
@@ -35,8 +35,8 @@ impl Database {
pub fn open_with_integrity_check(path: &Path) -> Result<Self> { pub fn open_with_integrity_check(path: &Path) -> Result<Self> {
debug!(?path, "Opening database with integrity check"); debug!(?path, "Opening database with integrity check");
let conn = Connection::open(path) let conn =
.map_err(|e| Error::Database(format!("open failed: {}", e)))?; Connection::open(path).map_err(|e| Error::Database(format!("open failed: {}", e)))?;
let integrity: String = conn let integrity: String = conn
.query_row("PRAGMA integrity_check(1)", [], |row| row.get(0)) .query_row("PRAGMA integrity_check(1)", [], |row| row.get(0))
@@ -45,7 +45,8 @@ impl Database {
if integrity != "ok" { if integrity != "ok" {
warn!(path = ?path, result = %integrity, "Database integrity check failed"); warn!(path = ?path, result = %integrity, "Database integrity check failed");
return Err(Error::DatabaseCorrupted(format!( return Err(Error::DatabaseCorrupted(format!(
"integrity check failed: {}", integrity "integrity check failed: {}",
integrity
))); )));
} }
@@ -250,11 +251,9 @@ impl Database {
pub fn file_count(&self) -> Result<u64> { pub fn file_count(&self) -> Result<u64> {
let conn = self.conn.lock().unwrap(); let conn = self.conn.lock().unwrap();
conn.query_row("SELECT COUNT(*) FROM files", [], |row| { conn.query_row("SELECT COUNT(*) FROM files", [], |row| row.get::<_, i64>(0))
row.get::<_, i64>(0) .map(|c| c as u64)
}) .map_err(|e| Error::Database(format!("count failed: {}", e)))
.map(|c| c as u64)
.map_err(|e| Error::Database(format!("count failed: {}", e)))
} }
pub fn update_content_hash(&self, id: FileId, hash: &ContentHash) -> Result<()> { pub fn update_content_hash(&self, id: FileId, hash: &ContentHash) -> Result<()> {
@@ -352,10 +351,7 @@ mod tests {
) )
.unwrap(); .unwrap();
let retrieved = db let retrieved = db.get_file_by_virtual_path(&virtual_path).unwrap().unwrap();
.get_file_by_virtual_path(&virtual_path)
.unwrap()
.unwrap();
assert_eq!(retrieved.id, id); assert_eq!(retrieved.id, id);
assert_eq!( assert_eq!(
retrieved.audio.as_ref().unwrap().title, retrieved.audio.as_ref().unwrap().title,
@@ -401,10 +397,7 @@ mod tests {
assert_eq!(db.file_count().unwrap(), 1); assert_eq!(db.file_count().unwrap(), 1);
let retrieved = db let retrieved = db.get_file_by_virtual_path(&virtual_path).unwrap().unwrap();
.get_file_by_virtual_path(&virtual_path)
.unwrap()
.unwrap();
assert_eq!( assert_eq!(
retrieved.audio.as_ref().unwrap().title, retrieved.audio.as_ref().unwrap().title,
Some("Updated".to_string()) Some("Updated".to_string())
@@ -94,7 +94,14 @@ mod tests {
}; };
cache cache
.store(&origin_id, real_path, &virtual_path, &meta, UNIX_EPOCH, 5000) .store(
&origin_id,
real_path,
&virtual_path,
&meta,
UNIX_EPOCH,
5000,
)
.unwrap(); .unwrap();
let retrieved = cache.lookup(&virtual_path).unwrap().unwrap(); let retrieved = cache.lookup(&virtual_path).unwrap().unwrap();
@@ -63,13 +63,11 @@ impl PatternStore {
let sequence_counts = { let sequence_counts = {
let mut map = HashMap::new(); let mut map = HashMap::new();
let mut stmt = db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?; let mut stmt =
db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?;
let rows = stmt.query_map([], |row| { let rows = stmt.query_map([], |row| {
Ok(( Ok((
( (FileId(row.get::<_, i64>(0)?), FileId(row.get::<_, i64>(1)?)),
FileId(row.get::<_, i64>(0)?),
FileId(row.get::<_, i64>(1)?),
),
row.get::<_, u32>(2)?, row.get::<_, u32>(2)?,
)) ))
})?; })?;
@@ -154,7 +152,11 @@ impl PatternStore {
.take(limit) .take(limit)
.map(|(id, _)| id) .map(|(id, _)| id)
.collect(); .collect();
debug!(file_id = current.0, predictions = result.len(), "Predicted next files"); debug!(
file_id = current.0,
predictions = result.len(),
"Predicted next files"
);
result result
} }
@@ -102,13 +102,8 @@ impl PrefetchEngine {
pattern_store.predict_next(file_id, config.lookahead); pattern_store.predict_next(file_id, config.lookahead);
for predicted_id in predictions { for predicted_id in predictions {
prefetch_file( prefetch_file(predicted_id, &fetcher, &in_flight, &semaphore)
predicted_id, .await;
&fetcher,
&in_flight,
&semaphore,
)
.await;
} }
tokio::time::sleep(config.cooldown).await; tokio::time::sleep(config.cooldown).await;
@@ -102,8 +102,7 @@ impl VirtualTree {
mtime: SystemTime::now(), mtime: SystemTime::now(),
}), }),
); );
tree.path_to_inode tree.path_to_inode.insert(VirtualPath::new("/"), ROOT_INODE);
.insert(VirtualPath::new("/"), ROOT_INODE);
tree tree
} }
@@ -161,13 +160,11 @@ impl VirtualTree {
fn find_parent_by_path_lookup(&self, inode: Inode) -> Option<Inode> { fn find_parent_by_path_lookup(&self, inode: Inode) -> Option<Inode> {
for (path, &ino) in &self.path_to_inode { for (path, &ino) in &self.path_to_inode {
if ino == inode { if ino == inode {
return std::path::Path::new(path.as_str()) return std::path::Path::new(path.as_str()).parent().and_then(|p| {
.parent() self.path_to_inode
.and_then(|p| { .get(&VirtualPath::new(p.to_string_lossy().into_owned()))
self.path_to_inode .copied()
.get(&VirtualPath::new(p.to_string_lossy().into_owned())) });
.copied()
});
} }
} }
None None
@@ -69,11 +69,7 @@ impl ContentFetcher {
.ok_or_else(|| FetchError::OriginNotFound(meta.real_path.origin_id.clone()))? .ok_or_else(|| FetchError::OriginNotFound(meta.real_path.origin_id.clone()))?
}; };
info!( info!("Fetching file {:?} from origin {}", file_id, origin.id());
"Fetching file {:?} from origin {}",
file_id,
origin.id()
);
let data = origin let data = origin
.read_full(&meta.real_path.path) .read_full(&meta.real_path.path)
@@ -26,7 +26,12 @@ impl ChunkManifest {
rmp_serde::from_slice(data).ok() rmp_serde::from_slice(data).ok()
} }
pub fn from_db(file_id: FileId, total_size: u64, mtime: i64, chunk_blob: &[u8]) -> Option<Self> { pub fn from_db(
file_id: FileId,
total_size: u64,
mtime: i64,
chunk_blob: &[u8],
) -> Option<Self> {
let chunks = Self::chunks_from_bytes(chunk_blob)?; let chunks = Self::chunks_from_bytes(chunk_blob)?;
Some(Self { Some(Self {
file_id, file_id,
@@ -80,9 +85,7 @@ impl FileReader {
}; };
let manifest = fetcher.ensure_cached(file_id).await?; let manifest = fetcher.ensure_cached(file_id).await?;
self.manifests self.manifests.write().insert(file_id, manifest.clone());
.write()
.insert(file_id, manifest.clone());
Ok(manifest) Ok(manifest)
} }
@@ -126,7 +129,9 @@ impl FileReader {
self.manifests.write().insert(file_id, new_manifest); self.manifests.write().insert(file_id, new_manifest);
self.store.get(&chunk_ref.hash).await? self.store.get(&chunk_ref.hash).await?
} else { } else {
return Err(ReaderError::Cas(CasError::NotFound(chunk_ref.hash.as_hex()))); return Err(ReaderError::Cas(CasError::NotFound(
chunk_ref.hash.as_hex(),
)));
} }
} }
Err(CasError::NotFound(_)) => { Err(CasError::NotFound(_)) => {
@@ -136,7 +141,9 @@ impl FileReader {
self.manifests.write().insert(file_id, new_manifest); self.manifests.write().insert(file_id, new_manifest);
self.store.get(&chunk_ref.hash).await? self.store.get(&chunk_ref.hash).await?
} else { } else {
return Err(ReaderError::Cas(CasError::NotFound(chunk_ref.hash.as_hex()))); return Err(ReaderError::Cas(CasError::NotFound(
chunk_ref.hash.as_hex(),
)));
} }
} }
Err(e) => return Err(ReaderError::Cas(e)), Err(e) => return Err(ReaderError::Cas(e)),
@@ -58,8 +58,7 @@ impl CasStore {
Err(repair_err) => { Err(repair_err) => {
warn!(error = %repair_err, "sled repair failed, recreating index"); warn!(error = %repair_err, "sled repair failed, recreating index");
if index_path.exists() { if index_path.exists() {
std::fs::remove_dir_all(&index_path) std::fs::remove_dir_all(&index_path).map_err(CasError::Io)?;
.map_err(CasError::Io)?;
} }
sled::open(&index_path)? sled::open(&index_path)?
} }
@@ -80,7 +79,9 @@ impl CasStore {
Self::calculate_size_recursive(dir).await Self::calculate_size_recursive(dir).await
} }
fn calculate_size_recursive(dir: &Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = u64> + Send + '_>> { fn calculate_size_recursive(
dir: &Path,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = u64> + Send + '_>> {
Box::pin(async move { Box::pin(async move {
let mut size = 0u64; let mut size = 0u64;
if let Ok(mut entries) = fs::read_dir(dir).await { if let Ok(mut entries) = fs::read_dir(dir).await {
@@ -117,7 +117,10 @@ async fn test_fetcher_cache_miss_flow() {
let store = Arc::new(CasStore::open(config).await.unwrap()); let store = Arc::new(CasStore::open(config).await.unwrap());
let origin_id = OriginId::from("test-origin"); let origin_id = OriginId::from("test-origin");
let origin = Arc::new(LocalOrigin::new(origin_id.clone(), origin_dir.path().to_path_buf())); let origin = Arc::new(LocalOrigin::new(
origin_id.clone(),
origin_dir.path().to_path_buf(),
));
let fetcher = ContentFetcher::new(store.clone()); let fetcher = ContentFetcher::new(store.clone());
fetcher.register_origin(origin); fetcher.register_origin(origin);
@@ -163,7 +166,10 @@ async fn test_reader_with_fetcher_integration() {
let store = Arc::new(CasStore::open(config).await.unwrap()); let store = Arc::new(CasStore::open(config).await.unwrap());
let origin_id = OriginId::from("local"); let origin_id = OriginId::from("local");
let origin = Arc::new(LocalOrigin::new(origin_id.clone(), origin_dir.path().to_path_buf())); let origin = Arc::new(LocalOrigin::new(
origin_id.clone(),
origin_dir.path().to_path_buf(),
));
let fetcher = ContentFetcher::new(store.clone()); let fetcher = ContentFetcher::new(store.clone());
fetcher.register_origin(origin); fetcher.register_origin(origin);
@@ -82,12 +82,8 @@ enum CacheCommands {
#[derive(Subcommand)] #[derive(Subcommand)]
enum OriginCommands { enum OriginCommands {
List, List,
Health { Health { origin_id: String },
origin_id: String, Rescan { origin_id: String },
},
Rescan {
origin_id: String,
},
} }
struct LockFile { struct LockFile {
@@ -245,8 +241,7 @@ fn run_mount(
runtime.block_on(async { runtime.block_on(async {
let mut sigterm = let mut sigterm =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
let mut sigint = let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
tokio::select! { tokio::select! {
_ = sigterm.recv() => { _ = sigterm.recv() => {
@@ -290,10 +285,7 @@ fn run_cache(command: CacheCommands) -> Result<()> {
println!("Cache stats: gRPC client integration pending"); println!("Cache stats: gRPC client integration pending");
} }
CacheCommands::Clear { origin } => { CacheCommands::Clear { origin } => {
println!( println!("Clearing cache for: {}", origin.as_deref().unwrap_or("all"));
"Clearing cache for: {}",
origin.as_deref().unwrap_or("all")
);
println!("gRPC client integration pending"); println!("gRPC client integration pending");
} }
CacheCommands::Prefetch { paths } => { CacheCommands::Prefetch { paths } => {
@@ -364,8 +356,8 @@ fn init_logging(config: &LoggingConfig) -> Result<WorkerGuard> {
let stderr_layer = fmt::layer().with_writer(std::io::stderr).compact(); let stderr_layer = fmt::layer().with_writer(std::io::stderr).compact();
let filter = EnvFilter::try_from_default_env() let filter =
.unwrap_or_else(|_| EnvFilter::new(&config.level)); EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&config.level));
let subscriber = tracing_subscriber::registry() let subscriber = tracing_subscriber::registry()
.with(filter) .with(filter)
@@ -488,10 +480,7 @@ fn build_virtual_path(path: &Path, audio: Option<&musicfs_core::AudioMeta>) -> V
if let Some(meta) = audio { if let Some(meta) = audio {
let artist = meta.artist.as_deref().unwrap_or("Unknown Artist"); let artist = meta.artist.as_deref().unwrap_or("Unknown Artist");
let album = meta.album.as_deref().unwrap_or("Unknown Album"); let album = meta.album.as_deref().unwrap_or("Unknown Album");
let filename = path let filename = path.file_name().and_then(|n| n.to_str()).unwrap_or("track");
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("track");
VirtualPath::new(&format!( VirtualPath::new(&format!(
"/{}/{}/{}", "/{}/{}/{}",
@@ -16,7 +16,10 @@ impl EventBus {
trace!(event = ?event, "Publishing event"); trace!(event = ?event, "Publishing event");
let receiver_count = self.sender.receiver_count(); let receiver_count = self.sender.receiver_count();
if self.sender.send(event).is_err() && receiver_count > 0 { if self.sender.send(event).is_err() && receiver_count > 0 {
debug!(receiver_count = receiver_count, "Event dropped, no active receivers"); debug!(
receiver_count = receiver_count,
"Event dropped, no active receivers"
);
} }
} }
@@ -22,9 +22,7 @@ impl Metrics {
} }
pub fn uptime_secs(&self) -> u64 { pub fn uptime_secs(&self) -> u64 {
self.start_time self.start_time.map(|t| t.elapsed().as_secs()).unwrap_or(0)
.map(|t| t.elapsed().as_secs())
.unwrap_or(0)
} }
pub fn to_prometheus(&self) -> String { pub fn to_prometheus(&self) -> String {
@@ -55,11 +53,16 @@ impl Metrics {
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.99\"}} {:.6}\n\ musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.99\"}} {:.6}\n\
musicfs_fuse_latency_seconds_sum{{op=\"{}\"}} {:.6}\n\ musicfs_fuse_latency_seconds_sum{{op=\"{}\"}} {:.6}\n\
musicfs_fuse_latency_seconds_count{{op=\"{}\"}} {}\n", musicfs_fuse_latency_seconds_count{{op=\"{}\"}} {}\n",
op, quantiles.p50, op,
op, quantiles.p95, quantiles.p50,
op, quantiles.p99, op,
op, histogram.sum_secs(), quantiles.p95,
op, histogram.count(), op,
quantiles.p99,
op,
histogram.sum_secs(),
op,
histogram.count(),
)); ));
} }
@@ -266,9 +269,7 @@ pub struct OriginHealthMetrics {
impl OriginHealthMetrics { impl OriginHealthMetrics {
pub fn set_health(&self, origin_id: &str, healthy: bool) { pub fn set_health(&self, origin_id: &str, healthy: bool) {
self.status self.status.write().insert(origin_id.to_string(), healthy);
.write()
.insert(origin_id.to_string(), healthy);
} }
} }
@@ -46,7 +46,11 @@ impl MusicFs {
} }
} }
pub fn with_reader(tree: Arc<RwLock<VirtualTree>>, reader: Arc<FileReader>, runtime_handle: Handle) -> Self { pub fn with_reader(
tree: Arc<RwLock<VirtualTree>>,
reader: Arc<FileReader>,
runtime_handle: Handle,
) -> Self {
Self { Self {
tree, tree,
reader: Some(reader), reader: Some(reader),
@@ -287,7 +291,12 @@ impl Filesystem for MusicFs {
let tree = self.tree.read(); let tree = self.tree.read();
if let Some(children) = tree.readdir(ino) { if let Some(children) = tree.readdir(ino) {
trace!(ino, offset, children_count = children.len(), "directory found"); trace!(
ino,
offset,
children_count = children.len(),
"directory found"
);
let parent_ino = tree.get_parent(ino).unwrap_or(ROOT_INODE); let parent_ino = tree.get_parent(ino).unwrap_or(ROOT_INODE);
let entries: Vec<(u64, FileType, &str)> = vec![ let entries: Vec<(u64, FileType, &str)> = vec![
@@ -396,7 +405,13 @@ impl Filesystem for MusicFs {
match result { match result {
Ok(Ok(data)) => { Ok(Ok(data)) => {
trace!(ino, offset, size_bytes = size, bytes_read = data.len(), "read successful"); trace!(
ino,
offset,
size_bytes = size,
bytes_read = data.len(),
"read successful"
);
reply.data(&data); reply.data(&data);
} }
Ok(Err(e)) => { Ok(Err(e)) => {
@@ -582,7 +597,7 @@ mod tests {
fn test_tree_integration() { fn test_tree_integration() {
let runtime = tokio::runtime::Runtime::new().unwrap(); let runtime = tokio::runtime::Runtime::new().unwrap();
let handle = runtime.handle().clone(); let handle = runtime.handle().clone();
let mut builder = TreeBuilder::new(); let mut builder = TreeBuilder::new();
builder.add_file(&make_file_meta(1, "/Artist/Album/Track.flac", 30_000_000)); builder.add_file(&make_file_meta(1, "/Artist/Album/Track.flac", 30_000_000));
let tree = Arc::new(RwLock::new(builder.build())); let tree = Arc::new(RwLock::new(builder.build()));
@@ -591,6 +606,8 @@ mod tests {
let tree_read = tree.read(); let tree_read = tree.read();
assert!(tree_read.get(ROOT_INODE).is_some()); assert!(tree_read.get(ROOT_INODE).is_some());
assert!(tree_read.get_by_path(&VirtualPath::new("/Artist")).is_some()); assert!(tree_read
.get_by_path(&VirtualPath::new("/Artist"))
.is_some());
} }
} }
@@ -43,10 +43,7 @@ impl PrefetchOps {
} }
} }
pub fn start_engine( pub fn start_engine(&self, event_bus: Arc<EventBus>) -> Option<musicfs_cache::PrefetchHandle> {
&self,
event_bus: Arc<EventBus>,
) -> Option<musicfs_cache::PrefetchHandle> {
self.engine self.engine
.as_ref() .as_ref()
.map(|e| e.clone().start(event_bus, self.pattern_store.clone())) .map(|e| e.clone().start(event_bus, self.pattern_store.clone()))
@@ -266,7 +263,8 @@ mod tests {
#[test] #[test]
fn test_prefetch_ops_new() { fn test_prefetch_ops_new() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let pattern_store = Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap()); let pattern_store =
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
let _ops = PrefetchOps::new(pattern_store, 1000, 1000); let _ops = PrefetchOps::new(pattern_store, 1000, 1000);
} }
@@ -283,11 +281,18 @@ mod tests {
#[test] #[test]
fn test_hint_name_to_inode() { fn test_hint_name_to_inode() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let pattern_store = Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap()); let pattern_store =
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
let ops = PrefetchOps::new(pattern_store, 1000, 1000); let ops = PrefetchOps::new(pattern_store, 1000, 1000);
assert_eq!(ops.hint_name_to_inode("hint_0001"), Some(PREFETCH_HINTS_BASE + 1)); assert_eq!(
assert_eq!(ops.hint_name_to_inode("hint_9999"), Some(PREFETCH_HINTS_BASE + 9999)); ops.hint_name_to_inode("hint_0001"),
Some(PREFETCH_HINTS_BASE + 1)
);
assert_eq!(
ops.hint_name_to_inode("hint_9999"),
Some(PREFETCH_HINTS_BASE + 9999)
);
assert_eq!(ops.hint_name_to_inode("invalid"), None); assert_eq!(ops.hint_name_to_inode("invalid"), None);
} }
} }
@@ -160,16 +160,17 @@ impl SearchOps {
} }
fn safe_symlink_target(&self, virtual_path: &str) -> Option<String> { fn safe_symlink_target(&self, virtual_path: &str) -> Option<String> {
let normalized = Path::new(virtual_path) let normalized = Path::new(virtual_path).components().fold(
.components() std::path::PathBuf::new(),
.fold(std::path::PathBuf::new(), |mut acc, comp| { |mut acc, comp| {
match comp { match comp {
std::path::Component::Normal(s) => acc.push(s), std::path::Component::Normal(s) => acc.push(s),
std::path::Component::RootDir => acc.push("/"), std::path::Component::RootDir => acc.push("/"),
_ => {} _ => {}
} }
acc acc
}); },
);
let path_str = normalized.to_string_lossy(); let path_str = normalized.to_string_lossy();
if path_str.contains("..") { if path_str.contains("..") {
@@ -198,7 +199,9 @@ impl SearchOps {
fn result_filename(&self, hit: &SearchHit, index: usize) -> String { fn result_filename(&self, hit: &SearchHit, index: usize) -> String {
let artist = hit.artist.as_deref().unwrap_or("Unknown"); let artist = hit.artist.as_deref().unwrap_or("Unknown");
let title = hit.title.as_deref().unwrap_or("Unknown"); let title = hit.title.as_deref().unwrap_or("Unknown");
let ext = hit.virtual_path.as_str() let ext = hit
.virtual_path
.as_str()
.rsplit('.') .rsplit('.')
.next() .next()
.unwrap_or("flac"); .unwrap_or("flac");
@@ -35,7 +35,9 @@ impl MusicFs for SearchService {
} }
if req.query.len() > 256 { if req.query.len() > 256 {
return Err(Status::invalid_argument("Query exceeds maximum length (256)")); return Err(Status::invalid_argument(
"Query exceeds maximum length (256)",
));
} }
let limit = req.limit.unwrap_or(100).min(10000) as usize; let limit = req.limit.unwrap_or(100).min(10000) as usize;
@@ -228,10 +228,7 @@ impl MusicFs for MusicFsServer {
} }
#[instrument(level = "info", skip(self, request), fields(method = "shutdown"))] #[instrument(level = "info", skip(self, request), fields(method = "shutdown"))]
async fn shutdown( async fn shutdown(&self, request: Request<ShutdownRequest>) -> Result<Response<Empty>, Status> {
&self,
request: Request<ShutdownRequest>,
) -> Result<Response<Empty>, Status> {
let req = request.into_inner(); let req = request.into_inner();
info!( info!(
graceful = req.graceful, graceful = req.graceful,
@@ -242,7 +239,11 @@ impl MusicFs for MusicFsServer {
Ok(Response::new(Empty {})) Ok(Response::new(Empty {}))
} }
#[instrument(level = "debug", skip(self, _request), fields(method = "get_cache_stats"))] #[instrument(
level = "debug",
skip(self, _request),
fields(method = "get_cache_stats")
)]
async fn get_cache_stats( async fn get_cache_stats(
&self, &self,
_request: Request<Empty>, _request: Request<Empty>,
@@ -339,7 +340,11 @@ impl MusicFs for MusicFsServer {
Ok(Response::new(OriginsResponse { origins: vec![] })) Ok(Response::new(OriginsResponse { origins: vec![] }))
} }
#[instrument(level = "debug", skip(self, request), fields(method = "get_origin_health"))] #[instrument(
level = "debug",
skip(self, request),
fields(method = "get_origin_health")
)]
async fn get_origin_health( async fn get_origin_health(
&self, &self,
request: Request<OriginRequest>, request: Request<OriginRequest>,
@@ -389,7 +394,11 @@ impl MusicFs for MusicFsServer {
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>; type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
#[instrument(level = "info", skip(self, request), fields(method = "subscribe_events"))] #[instrument(
level = "info",
skip(self, request),
fields(method = "subscribe_events")
)]
async fn subscribe_events( async fn subscribe_events(
&self, &self,
request: Request<EventFilter>, request: Request<EventFilter>,
@@ -52,8 +52,7 @@ impl MetadataParser {
if let Some(n_frames) = params.n_frames { if let Some(n_frames) = params.n_frames {
if let Some(sample_rate) = params.sample_rate { if let Some(sample_rate) = params.sample_rate {
audio_meta.duration_ms = audio_meta.duration_ms = Some((n_frames as u64 * 1000) / sample_rate as u64);
Some((n_frames as u64 * 1000) / sample_rate as u64);
audio_meta.sample_rate = Some(sample_rate); audio_meta.sample_rate = Some(sample_rate);
} }
} }
@@ -61,9 +60,8 @@ impl MetadataParser {
if let Some(bits_per_sample) = params.bits_per_sample { if let Some(bits_per_sample) = params.bits_per_sample {
if let Some(sample_rate) = params.sample_rate { if let Some(sample_rate) = params.sample_rate {
if let Some(channels) = params.channels { if let Some(channels) = params.channels {
audio_meta.bitrate = Some( audio_meta.bitrate =
bits_per_sample * sample_rate * channels.count() as u32 / 1000, Some(bits_per_sample * sample_rate * channels.count() as u32 / 1000);
);
} }
} }
} }
@@ -67,11 +67,10 @@ impl FailoverExecutor {
if origins.is_empty() { if origins.is_empty() {
if let Some(origin) = self.registry.route_with_fallback(path) { if let Some(origin) = self.registry.route_with_fallback(path) {
warn!( warn!("No healthy origins, using fallback origin {}", origin.id());
"No healthy origins, using fallback origin {}", return self
origin.id() .read_with_retry(&origin, &path.path, offset, size)
); .await;
return self.read_with_retry(&origin, &path.path, offset, size).await;
} }
return Err(Error::NoOriginAvailable); return Err(Error::NoOriginAvailable);
} }
@@ -81,7 +80,10 @@ impl FailoverExecutor {
for origin in origins { for origin in origins {
trace!(origin_id = %origin.id(), "Attempting read from origin"); trace!(origin_id = %origin.id(), "Attempting read from origin");
let start = std::time::Instant::now(); let start = std::time::Instant::now();
match self.read_with_retry(&origin, &path.path, offset, size).await { match self
.read_with_retry(&origin, &path.path, offset, size)
.await
{
Ok(data) => { Ok(data) => {
let latency = start.elapsed().as_millis() as u64; let latency = start.elapsed().as_millis() as u64;
self.registry.record_latency(origin.id(), latency); self.registry.record_latency(origin.id(), latency);
@@ -214,10 +216,8 @@ mod tests {
#[test] #[test]
fn test_custom_delays() { fn test_custom_delays() {
let config = RetryConfig::with_delays(vec![ let config =
Duration::from_millis(50), RetryConfig::with_delays(vec![Duration::from_millis(50), Duration::from_millis(100)]);
Duration::from_millis(100),
]);
assert_eq!(config.max_attempts, 2); assert_eq!(config.max_attempts, 2);
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(50)); assert_eq!(config.delay_for_attempt(0), Duration::from_millis(50));
@@ -349,10 +349,13 @@ mod tests {
let mut thresholds = HashMap::new(); let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 3); thresholds.insert(OriginType::Local, 3);
let monitor = HealthMonitor::new(Duration::from_secs(30)) let monitor =
.with_per_type_thresholds(thresholds); HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds);
let origin = Arc::new(LocalOrigin::new("missing", std::path::Path::new("/nonexistent"))); let origin = Arc::new(LocalOrigin::new(
"missing",
std::path::Path::new("/nonexistent"),
));
monitor.add_origin(origin); monitor.add_origin(origin);
monitor.check_now(&OriginId::from("missing")).await; monitor.check_now(&OriginId::from("missing")).await;
@@ -372,7 +375,10 @@ mod tests {
async fn test_local_origin_threshold_is_one() { async fn test_local_origin_threshold_is_one() {
let monitor = HealthMonitor::new(Duration::from_secs(30)); let monitor = HealthMonitor::new(Duration::from_secs(30));
let origin = Arc::new(LocalOrigin::new("missing", std::path::Path::new("/nonexistent"))); let origin = Arc::new(LocalOrigin::new(
"missing",
std::path::Path::new("/nonexistent"),
));
monitor.add_origin(origin); monitor.add_origin(origin);
monitor.check_now(&OriginId::from("missing")).await; monitor.check_now(&OriginId::from("missing")).await;
@@ -86,7 +86,7 @@ impl Router {
(priority, latency) (priority, latency)
}) })
.cloned(); .cloned();
if let Some(ref id) = selected { if let Some(ref id) = selected {
let priority = self.get_priority(id); let priority = self.get_priority(id);
let latency = self.latency_stats.get(id).map(|s| s.p50_ms).unwrap_or(0); let latency = self.latency_stats.get(id).map(|s| s.p50_ms).unwrap_or(0);
@@ -97,7 +97,7 @@ impl Router {
"Selected healthy origin" "Selected healthy origin"
); );
} }
selected selected
} }
@@ -141,7 +141,7 @@ impl Router {
(failures, priority) (failures, priority)
}) })
.cloned(); .cloned();
if let Some(ref id) = selected { if let Some(ref id) = selected {
let failures = health.failure_count(id).unwrap_or(u32::MAX); let failures = health.failure_count(id).unwrap_or(u32::MAX);
trace!( trace!(
@@ -151,7 +151,7 @@ impl Router {
"Selected least-bad unhealthy origin" "Selected least-bad unhealthy origin"
); );
} }
selected selected
} }
} }
@@ -47,5 +47,3 @@
mod implementation { mod implementation {
// Full S3 implementation would go here when aws-sdk-s3 is enabled // Full S3 implementation would go here when aws-sdk-s3 is enabled
} }
@@ -91,11 +91,13 @@ impl Origin for SmbOrigin {
} }
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> { async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
self.retry_on_disconnect(|| self.inner.read(path, offset, size)).await self.retry_on_disconnect(|| self.inner.read(path, offset, size))
.await
} }
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> { async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
self.retry_on_disconnect(|| self.inner.read_full(path)).await self.retry_on_disconnect(|| self.inner.read_full(path))
.await
} }
async fn exists(&self, path: &Path) -> Result<bool> { async fn exists(&self, path: &Path) -> Result<bool> {
@@ -55,9 +55,8 @@ impl NativePluginHost {
info!("Loading native plugin from {:?}", canonical); info!("Loading native plugin from {:?}", canonical);
let library = unsafe { let library = unsafe {
Library::new(&canonical).map_err(|e| { Library::new(&canonical)
PluginError::LoadFailed(format!("Failed to load library: {}", e)) .map_err(|e| PluginError::LoadFailed(format!("Failed to load library: {}", e)))?
})?
}; };
self.verify_api_version(&library)?; self.verify_api_version(&library)?;
@@ -190,9 +189,9 @@ impl NativePluginHost {
fn verify_api_version(&self, library: &Library) -> Result<()> { fn verify_api_version(&self, library: &Library) -> Result<()> {
let version_fn: Symbol<unsafe extern "C" fn() -> *const std::ffi::c_char> = unsafe { let version_fn: Symbol<unsafe extern "C" fn() -> *const std::ffi::c_char> = unsafe {
library library.get(b"musicfs_plugin_api_version").map_err(|_| {
.get(b"musicfs_plugin_api_version") PluginError::SymbolNotFound("musicfs_plugin_api_version".to_string())
.map_err(|_| PluginError::SymbolNotFound("musicfs_plugin_api_version".to_string()))? })?
}; };
let version_ptr = unsafe { version_fn() }; let version_ptr = unsafe { version_fn() };
@@ -203,10 +202,11 @@ impl NativePluginHost {
actual: "<invalid UTF-8>".to_string(), actual: "<invalid UTF-8>".to_string(),
})?; })?;
let plugin_version = Version::parse(version_str).map_err(|_| PluginError::VersionMismatch { let plugin_version =
expected: PLUGIN_API_VERSION.to_string(), Version::parse(version_str).map_err(|_| PluginError::VersionMismatch {
actual: version_str.to_string(), expected: PLUGIN_API_VERSION.to_string(),
})?; actual: version_str.to_string(),
})?;
let expected_version = Version::parse(PLUGIN_API_VERSION).unwrap(); let expected_version = Version::parse(PLUGIN_API_VERSION).unwrap();
@@ -95,11 +95,7 @@ pub trait OriginPlugin: Plugin {
/// ///
/// The config contains origin-specific settings (credentials, paths, etc). /// The config contains origin-specific settings (credentials, paths, etc).
/// Returns a boxed Origin that can be used by the OriginRouter. /// Returns a boxed Origin that can be used by the OriginRouter.
async fn create_origin( async fn create_origin(&self, id: &str, config: Value) -> Result<Box<dyn OriginInstance>>;
&self,
id: &str,
config: Value,
) -> Result<Box<dyn OriginInstance>>;
} }
/// Instance created by OriginPlugin /// Instance created by OriginPlugin
@@ -261,7 +261,12 @@ mod tests {
let store = CollectionStore::new(&db_path).unwrap(); let store = CollectionStore::new(&db_path).unwrap();
let collection = store let collection = store
.create("Jazz", CollectionQuery::Genre { genre: "Jazz".to_string() }) .create(
"Jazz",
CollectionQuery::Genre {
genre: "Jazz".to_string(),
},
)
.unwrap(); .unwrap();
assert_eq!(collection.name, "Jazz"); assert_eq!(collection.name, "Jazz");
@@ -279,7 +284,9 @@ mod tests {
let query = CollectionQuery::Compound { let query = CollectionQuery::Compound {
op: BoolOp::And, op: BoolOp::And,
children: vec![ children: vec![
CollectionQuery::Genre { genre: "Metal".to_string() }, CollectionQuery::Genre {
genre: "Metal".to_string(),
},
CollectionQuery::DateRange { CollectionQuery::DateRange {
field: "year".to_string(), field: "year".to_string(),
start: 1980, start: 1980,
@@ -306,6 +313,9 @@ mod tests {
assert!(CollectionQuery::RecentlyAdded { days: 30 }.is_dynamic()); assert!(CollectionQuery::RecentlyAdded { days: 30 }.is_dynamic());
assert!(CollectionQuery::RecentlyPlayed { days: 7 }.is_dynamic()); assert!(CollectionQuery::RecentlyPlayed { days: 7 }.is_dynamic());
assert!(CollectionQuery::MostPlayed { limit: 100 }.is_dynamic()); assert!(CollectionQuery::MostPlayed { limit: 100 }.is_dynamic());
assert!(!CollectionQuery::Genre { genre: "Rock".to_string() }.is_dynamic()); assert!(!CollectionQuery::Genre {
genre: "Rock".to_string()
}
.is_dynamic());
} }
} }
@@ -4,7 +4,7 @@ use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use tantivy::collector::TopDocs; use tantivy::collector::TopDocs;
use tantivy::query::{BooleanQuery, FuzzyTermQuery, Occur, Query, QueryParser}; use tantivy::query::{BooleanQuery, FuzzyTermQuery, Occur, Query, QueryParser};
use tantivy::schema::{Field, Schema, Value, STORED, TEXT, INDEXED}; use tantivy::schema::{Field, Schema, Value, INDEXED, STORED, TEXT};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term}; use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
@@ -109,8 +109,7 @@ impl SearchIndex {
"Search index corrupted, rebuilding from scratch" "Search index corrupted, rebuilding from scratch"
); );
if index_path.exists() { if index_path.exists() {
std::fs::remove_dir_all(index_path) std::fs::remove_dir_all(index_path).map_err(SearchError::Io)?;
.map_err(SearchError::Io)?;
} }
Self::open(index_path) Self::open(index_path)
} }
@@ -205,20 +204,21 @@ impl SearchIndex {
self.schema.composer, self.schema.composer,
]; ];
let query: Box<dyn Query> = if let Some((term, distance)) = Self::parse_fuzzy_query(query_str) { let query: Box<dyn Query> =
let subqueries: Vec<(Occur, Box<dyn Query>)> = default_fields if let Some((term, distance)) = Self::parse_fuzzy_query(query_str) {
.iter() let subqueries: Vec<(Occur, Box<dyn Query>)> = default_fields
.map(|&field| { .iter()
let term = Term::from_field_text(field, &term); .map(|&field| {
let fuzzy = FuzzyTermQuery::new(term, distance, true); let term = Term::from_field_text(field, &term);
(Occur::Should, Box::new(fuzzy) as Box<dyn Query>) let fuzzy = FuzzyTermQuery::new(term, distance, true);
}) (Occur::Should, Box::new(fuzzy) as Box<dyn Query>)
.collect(); })
Box::new(BooleanQuery::new(subqueries)) .collect();
} else { Box::new(BooleanQuery::new(subqueries))
let query_parser = QueryParser::for_index(&self.index, default_fields); } else {
query_parser.parse_query(query_str)? let query_parser = QueryParser::for_index(&self.index, default_fields);
}; query_parser.parse_query(query_str)?
};
let top_docs = searcher.search(&*query, &TopDocs::with_limit(limit))?; let top_docs = searcher.search(&*query, &TopDocs::with_limit(limit))?;
@@ -241,9 +241,18 @@ impl SearchIndex {
results.push(SearchHit { results.push(SearchHit {
file_id, file_id,
virtual_path, virtual_path,
artist: doc.get_first(self.schema.artist).and_then(|v| v.as_str()).map(String::from), artist: doc
album: doc.get_first(self.schema.album).and_then(|v| v.as_str()).map(String::from), .get_first(self.schema.artist)
title: doc.get_first(self.schema.title).and_then(|v| v.as_str()).map(String::from), .and_then(|v| v.as_str())
.map(String::from),
album: doc
.get_first(self.schema.album)
.and_then(|v| v.as_str())
.map(String::from),
title: doc
.get_first(self.schema.title)
.and_then(|v| v.as_str())
.map(String::from),
score, score,
}); });
} }
@@ -322,9 +331,15 @@ mod tests {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap(); let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap(); index
index.index_file(&make_file(2, "Metallica", "Master of Puppets", "Battery")).unwrap(); .index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
index.index_file(&make_file(3, "Iron Maiden", "Powerslave", "Aces High")).unwrap(); .unwrap();
index
.index_file(&make_file(2, "Metallica", "Master of Puppets", "Battery"))
.unwrap();
index
.index_file(&make_file(3, "Iron Maiden", "Powerslave", "Aces High"))
.unwrap();
index.commit().unwrap(); index.commit().unwrap();
let results = index.search("metallica", 10).unwrap(); let results = index.search("metallica", 10).unwrap();
@@ -340,7 +355,9 @@ mod tests {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap(); let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap(); index
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
.unwrap();
index.commit().unwrap(); index.commit().unwrap();
let results = index.search("metalica~1", 10).unwrap(); let results = index.search("metalica~1", 10).unwrap();
@@ -352,7 +369,9 @@ mod tests {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap(); let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap(); index
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
.unwrap();
index.commit().unwrap(); index.commit().unwrap();
let results = index.search("genre:Metal", 10).unwrap(); let results = index.search("genre:Metal", 10).unwrap();
@@ -364,7 +383,9 @@ mod tests {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap(); let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Test", "Album", "Song")).unwrap(); index
.index_file(&make_file(1, "Test", "Album", "Song"))
.unwrap();
index.commit().unwrap(); index.commit().unwrap();
assert_eq!(index.search("test", 10).unwrap().len(), 1); assert_eq!(index.search("test", 10).unwrap().len(), 1);
@@ -381,7 +402,9 @@ mod tests {
{ {
let index = SearchIndex::open(dir.path()).unwrap(); let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Artist", "Album", "Track")).unwrap(); index
.index_file(&make_file(1, "Artist", "Album", "Track"))
.unwrap();
index.commit().unwrap(); index.commit().unwrap();
} }
@@ -15,11 +15,7 @@ pub struct Indexer<M: MetadataLookup> {
} }
impl<M: MetadataLookup + 'static> Indexer<M> { impl<M: MetadataLookup + 'static> Indexer<M> {
pub fn new( pub fn new(index: Arc<SearchIndex>, event_bus: Arc<EventBus>, metadata_lookup: Arc<M>) -> Self {
index: Arc<SearchIndex>,
event_bus: Arc<EventBus>,
metadata_lookup: Arc<M>,
) -> Self {
Self { Self {
index, index,
event_bus, event_bus,
@@ -4,8 +4,7 @@ mod indexer;
mod query; mod query;
pub use collections::{ pub use collections::{
builtin_collections, BoolOp, CollectionError, CollectionQuery, CollectionStore, builtin_collections, BoolOp, CollectionError, CollectionQuery, CollectionStore, SmartCollection,
SmartCollection,
}; };
pub use index::{SearchError, SearchHit, SearchIndex}; pub use index::{SearchError, SearchHit, SearchIndex};
pub use indexer::{Indexer, IndexerHandle, MetadataLookup}; pub use indexer::{Indexer, IndexerHandle, MetadataLookup};
@@ -138,14 +138,21 @@ mod tests {
let shared = hashes1.intersection(&hashes2).count(); let shared = hashes1.intersection(&hashes2).count();
assert!(shared > 0, "CDC should produce stable boundaries, got {} chunks in original, {} after prepend", chunks1.len(), chunks2.len()); assert!(
shared > 0,
"CDC should produce stable boundaries, got {} chunks in original, {} after prepend",
chunks1.len(),
chunks2.len()
);
} }
#[test] #[test]
fn test_cdc_chunk_sizes() { fn test_cdc_chunk_sizes() {
let chunker = CdcChunker::default(); let chunker = CdcChunker::default();
let data: Vec<u8> = (0..1024 * 1024).map(|i| ((i * 17 + 31) % 256) as u8).collect(); let data: Vec<u8> = (0..1024 * 1024)
.map(|i| ((i * 17 + 31) % 256) as u8)
.collect();
let chunks = chunker.chunk(&data); let chunks = chunker.chunk(&data);
@@ -68,7 +68,7 @@ impl DeltaDetector {
) -> Result<ChangeSet, DeltaError> { ) -> Result<ChangeSet, DeltaError> {
let origin_id = origin.id().clone(); let origin_id = origin.id().clone();
info!(origin_id = %origin_id, "Starting delta detection"); info!(origin_id = %origin_id, "Starting delta detection");
let mut changes = ChangeSet::default(); let mut changes = ChangeSet::default();
let origin_files = self.scan_origin(origin).await?; let origin_files = self.scan_origin(origin).await?;
@@ -187,7 +187,11 @@ impl DeltaDetector {
.collect()) .collect())
} }
fn compute_diff(&self, old_chunks: &[ManifestChunk], new_chunks: &[ManifestChunk]) -> ManifestDiff { fn compute_diff(
&self,
old_chunks: &[ManifestChunk],
new_chunks: &[ManifestChunk],
) -> ManifestDiff {
let old_hashes: HashSet<_> = old_chunks.iter().map(|c| c.hash).collect(); let old_hashes: HashSet<_> = old_chunks.iter().map(|c| c.hash).collect();
let new_hashes: HashSet<_> = new_chunks.iter().map(|c| c.hash).collect(); let new_hashes: HashSet<_> = new_chunks.iter().map(|c| c.hash).collect();
@@ -34,7 +34,8 @@ impl OriginWatcher {
let origin_id_str = origin_id.to_string(); let origin_id_str = origin_id.to_string();
tokio::spawn( tokio::spawn(
async move { async move {
if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await { if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await
{
error!("Watcher error: {}", e); error!("Watcher error: {}", e);
} }
} }
@@ -126,7 +127,10 @@ impl OriginWatcher {
} }
EventKind::Remove(_) => { EventKind::Remove(_) => {
trace!(origin_id = %origin_id, path = ?relative, "File removed"); trace!(origin_id = %origin_id, path = ?relative, "File removed");
event_bus.publish(Event::FileRemoved { path: vpath, file_id: None }); event_bus.publish(Event::FileRemoved {
path: vpath,
file_id: None,
});
} }
EventKind::Modify(_) => { EventKind::Modify(_) => {
trace!(origin_id = %origin_id, path = ?relative, "File modified"); trace!(origin_id = %origin_id, path = ?relative, "File modified");
@@ -186,7 +190,8 @@ mod tests {
let event_bus = Arc::new(EventBus::default()); let event_bus = Arc::new(EventBus::default());
let mut rx = event_bus.subscribe(); let mut rx = event_bus.subscribe();
let watcher = OriginWatcher::new(OriginId::from("test"), dir.path().to_path_buf(), event_bus); let watcher =
OriginWatcher::new(OriginId::from("test"), dir.path().to_path_buf(), event_bus);
let handle = watcher.start(); let handle = watcher.start();
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
@@ -206,6 +211,8 @@ mod tests {
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.flac"))); assert!(OriginWatcher::is_audio_file(Path::new("/music/song.flac")));
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.MP3"))); assert!(OriginWatcher::is_audio_file(Path::new("/music/song.MP3")));
assert!(!OriginWatcher::is_audio_file(Path::new("/music/cover.jpg"))); assert!(!OriginWatcher::is_audio_file(Path::new("/music/cover.jpg")));
assert!(!OriginWatcher::is_audio_file(Path::new("/music/readme.txt"))); assert!(!OriginWatcher::is_audio_file(Path::new(
"/music/readme.txt"
)));
} }
} }
@@ -133,10 +133,7 @@ where
{ {
tokio::time::timeout(timeout, future) tokio::time::timeout(timeout, future)
.await .await
.expect(&format!( .expect(&format!("Operation did not complete within {:?}", timeout))
"Operation did not complete within {:?}",
timeout
))
} }
pub async fn assert_times_out<F, T>(future: F, timeout: Duration) pub async fn assert_times_out<F, T>(future: F, timeout: Duration)
@@ -168,8 +165,10 @@ mod tests {
#[test] #[test]
fn test_assert_io_error() { fn test_assert_io_error() {
let result: Result<(), Error> = let result: Result<(), Error> = Err(Error::Io(std::io::Error::new(
Err(Error::Io(std::io::Error::new(std::io::ErrorKind::Other, "test"))); std::io::ErrorKind::Other,
"test",
)));
assert_io_error(result); assert_io_error(result);
} }
@@ -188,8 +187,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_assert_completes_within() { async fn test_assert_completes_within() {
let result = let result = assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
assert_eq!(result, 42); assert_eq!(result, 42);
} }
@@ -1,8 +1,6 @@
use musicfs_cache::TreeBuilder; use musicfs_cache::TreeBuilder;
use musicfs_cas::{CasConfig, CasStore}; use musicfs_cas::{CasConfig, CasStore};
use musicfs_core::{ use musicfs_core::{AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath};
AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath,
};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::time::SystemTime; use std::time::SystemTime;
@@ -17,7 +17,11 @@ async fn require_toxiproxy() {
Ok(resp) => resp.status().is_success(), Ok(resp) => resp.status().is_success(),
Err(_) => false, Err(_) => false,
}; };
assert!(available, "Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d", TOXIPROXY_API); assert!(
available,
"Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d",
TOXIPROXY_API
);
} }
#[tokio::test] #[tokio::test]
@@ -41,10 +45,7 @@ async fn test_toxiproxy_latency_injection() {
toxicity: 1.0, toxicity: 1.0,
}; };
proxy proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
let start = std::time::Instant::now(); let start = std::time::Instant::now();
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await; let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
@@ -80,10 +81,7 @@ async fn test_toxiproxy_timeout_simulates_network_partition() {
toxicity: 1.0, toxicity: 1.0,
}; };
proxy proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
let result = tokio::time::timeout( let result = tokio::time::timeout(
Duration::from_secs(2), Duration::from_secs(2),
@@ -127,10 +125,7 @@ async fn test_toxiproxy_slow_close_throttles_responses() {
toxicity: 1.0, toxicity: 1.0,
}; };
proxy proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
let start = std::time::Instant::now(); let start = std::time::Instant::now();
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await; let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
@@ -144,5 +139,3 @@ async fn test_toxiproxy_slow_close_throttles_responses() {
proxy.delete().await.expect("Failed to cleanup proxy"); proxy.delete().await.expect("Failed to cleanup proxy");
} }
@@ -33,7 +33,10 @@ async fn setup_cas(dir: &Path) -> CasStore {
} }
fn create_faulty_origin(id: &str, dir: &TempDir, mode: FailMode) -> Arc<FaultyOrigin> { fn create_faulty_origin(id: &str, dir: &TempDir, mode: FailMode) -> Arc<FaultyOrigin> {
let inner = Arc::new(LocalOrigin::new(OriginId::from(id), dir.path().to_path_buf())); let inner = Arc::new(LocalOrigin::new(
OriginId::from(id),
dir.path().to_path_buf(),
));
Arc::new(FaultyOrigin::new(inner, mode)) Arc::new(FaultyOrigin::new(inner, mode))
} }
@@ -94,7 +97,9 @@ async fn test_tantivy_corruption_triggers_rebuild() {
{ {
let index = SearchIndex::open(&index_path).unwrap(); let index = SearchIndex::open(&index_path).unwrap();
index.index_file(&make_file_meta(1, "/a.flac", 1000)).unwrap(); index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap(); index.commit().unwrap();
} }
@@ -147,8 +152,11 @@ async fn test_cas_put_handles_enospc() {
let large_data = vec![0u8; 1000]; let large_data = vec![0u8; 1000];
let result = store.put(&large_data).await; let result = store.put(&large_data).await;
assert!(result.is_err(), "Issue 2.8: CasStore should pre-check space and reject oversized write"); assert!(
result.is_err(),
"Issue 2.8: CasStore should pre-check space and reject oversized write"
);
} }
/// Demonstrates the PROBLEM with std::sync::RwLock: after a writer panic, /// Demonstrates the PROBLEM with std::sync::RwLock: after a writer panic,
@@ -190,7 +198,10 @@ fn test_parking_lot_rwlock_survives_panic() {
let _ = handle.join(); let _ = handle.join();
assert!(tree.read().get(ROOT_INODE).is_some(), "parking_lot RwLock should survive writer panic"); assert!(
tree.read().get(ROOT_INODE).is_some(),
"parking_lot RwLock should survive writer panic"
);
} }
#[tokio::test] #[tokio::test]
@@ -200,12 +211,17 @@ async fn test_failover_on_primary_death() {
setup_test_file(&primary_dir, "test.txt", b"primary"); setup_test_file(&primary_dir, "test.txt", b"primary");
setup_test_file(&backup_dir, "test.txt", b"backup"); setup_test_file(&backup_dir, "test.txt", b"backup");
let primary = create_faulty_origin("primary", &primary_dir, FailMode::ReturnError(ErrorKind::ConnectionRefused)); let primary = create_faulty_origin(
"primary",
&primary_dir,
FailMode::ReturnError(ErrorKind::ConnectionRefused),
);
let backup = create_faulty_origin("backup", &backup_dir, FailMode::Healthy); let backup = create_faulty_origin("backup", &backup_dir, FailMode::Healthy);
let mut thresholds = HashMap::new(); let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 1); thresholds.insert(OriginType::Local, 1);
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds)); let monitor =
Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
let registry = Arc::new(OriginRegistry::new(monitor.clone())); let registry = Arc::new(OriginRegistry::new(monitor.clone()));
registry.register(primary.clone(), 1); registry.register(primary.clone(), 1);
@@ -231,21 +247,44 @@ async fn test_origin_recovery_resumes_routing() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
setup_test_file(&dir, "test.txt", b"content"); setup_test_file(&dir, "test.txt", b"content");
let faulty = create_faulty_origin("recovering", &dir, FailMode::ReturnError(ErrorKind::ConnectionRefused)); let faulty = create_faulty_origin(
"recovering",
&dir,
FailMode::ReturnError(ErrorKind::ConnectionRefused),
);
let mut thresholds = HashMap::new(); let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 1); thresholds.insert(OriginType::Local, 1);
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds)); let monitor =
Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
monitor.add_origin(faulty.clone()); monitor.add_origin(faulty.clone());
monitor.check_now(&OriginId::from("recovering")).await; monitor.check_now(&OriginId::from("recovering")).await;
assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().status, HealthStatus::Unhealthy); assert_eq!(
monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.status,
HealthStatus::Unhealthy
);
faulty.set_mode(FailMode::Healthy); faulty.set_mode(FailMode::Healthy);
monitor.check_now(&OriginId::from("recovering")).await; monitor.check_now(&OriginId::from("recovering")).await;
assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().status, HealthStatus::Healthy); assert_eq!(
assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().consecutive_failures, 0); monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.status,
HealthStatus::Healthy
);
assert_eq!(
monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.consecutive_failures,
0
);
} }
#[tokio::test] #[tokio::test]
@@ -262,9 +301,12 @@ async fn test_local_origin_health_check_has_timeout() {
monitor.check_now(&OriginId::from("slow")).await; monitor.check_now(&OriginId::from("slow")).await;
let elapsed = start.elapsed(); let elapsed = start.elapsed();
assert!(elapsed < Duration::from_secs(2), assert!(
"Issue 4.2.1: Health check should timeout in <2s, took {:?}", elapsed); elapsed < Duration::from_secs(2),
"Issue 4.2.1: Health check should timeout in <2s, took {:?}",
elapsed
);
let state = monitor.get_state(&OriginId::from("slow")).unwrap(); let state = monitor.get_state(&OriginId::from("slow")).unwrap();
assert_eq!(state.status, HealthStatus::Unhealthy); assert_eq!(state.status, HealthStatus::Unhealthy);
} }
@@ -288,7 +330,11 @@ async fn test_health_checks_run_in_parallel() {
monitor.check_all().await; monitor.check_all().await;
let elapsed = start.elapsed(); let elapsed = start.elapsed();
assert!(elapsed < Duration::from_millis(350), "Issue 4.2.2: check_all() should run in parallel (sequential would take ~600ms), took {:?}", elapsed); assert!(
elapsed < Duration::from_millis(350),
"Issue 4.2.2: check_all() should run in parallel (sequential would take ~600ms), took {:?}",
elapsed
);
} }
#[test] #[test]
@@ -298,9 +344,13 @@ fn test_tantivy_survives_uncommitted_crash() {
{ {
let index = SearchIndex::open(&index_path).unwrap(); let index = SearchIndex::open(&index_path).unwrap();
index.index_file(&make_file_meta(1, "/a.flac", 1000)).unwrap(); index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap(); index.commit().unwrap();
index.index_file(&make_file_meta(2, "/b.flac", 1000)).unwrap(); index
.index_file(&make_file_meta(2, "/b.flac", 1000))
.unwrap();
} }
let index = SearchIndex::open(&index_path).unwrap(); let index = SearchIndex::open(&index_path).unwrap();
@@ -329,10 +379,7 @@ async fn test_fd_exhaustion_handling() {
Ok(_store) => {} Ok(_store) => {}
Err(e) => { Err(e) => {
let msg = format!("{}", e); let msg = format!("{}", e);
assert!( assert!(!msg.contains("panic"), "Should not panic on fd exhaustion");
!msg.contains("panic"),
"Should not panic on fd exhaustion"
);
} }
} }
@@ -356,8 +403,11 @@ async fn test_corrupt_chunk_auto_refetched() {
setup_test_file(&origin_dir, "test.flac", test_content); setup_test_file(&origin_dir, "test.flac", test_content);
let store = Arc::new(setup_cas(dir.path()).await); let store = Arc::new(setup_cas(dir.path()).await);
let origin = Arc::new(LocalOrigin::new(OriginId::from("local"), origin_dir.path().to_path_buf())); let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone())); let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin); fetcher.register_origin(origin);
@@ -378,8 +428,13 @@ async fn test_corrupt_chunk_auto_refetched() {
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap(); let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
let chunk_hash = manifest.chunks[0].hash; let chunk_hash = manifest.chunks[0].hash;
let hex = chunk_hash.as_hex(); let hex = chunk_hash.as_hex();
let chunk_path = dir.path().join("chunks").join(&hex[0..2]).join(&hex[2..4]).join(&hex); let chunk_path = dir
.path()
.join("chunks")
.join(&hex[0..2])
.join(&hex[2..4])
.join(&hex);
let mut corrupted = std::fs::read(&chunk_path).unwrap(); let mut corrupted = std::fs::read(&chunk_path).unwrap();
corrupted[0] = corrupted[0].wrapping_add(1); corrupted[0] = corrupted[0].wrapping_add(1);
std::fs::write(&chunk_path, &corrupted).unwrap(); std::fs::write(&chunk_path, &corrupted).unwrap();
@@ -388,9 +443,16 @@ async fn test_corrupt_chunk_auto_refetched() {
reader.register_manifest(manifest); reader.register_manifest(manifest);
let result = reader.read(FileId(1), 0, test_content.len() as u32).await; let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
assert!(result.is_ok(), "Issue 6.4: Corrupted chunk should be auto-refetched from origin"); assert!(
assert_eq!(&result.unwrap()[..], test_content, "Data should match original after re-fetch"); result.is_ok(),
"Issue 6.4: Corrupted chunk should be auto-refetched from origin"
);
assert_eq!(
&result.unwrap()[..],
test_content,
"Data should match original after re-fetch"
);
} }
#[tokio::test] #[tokio::test]
@@ -404,8 +466,11 @@ async fn test_missing_chunk_triggers_origin_fetch() {
setup_test_file(&origin_dir, "test.flac", test_content); setup_test_file(&origin_dir, "test.flac", test_content);
let store = Arc::new(setup_cas(dir.path()).await); let store = Arc::new(setup_cas(dir.path()).await);
let origin = Arc::new(LocalOrigin::new(OriginId::from("local"), origin_dir.path().to_path_buf())); let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone())); let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin); fetcher.register_origin(origin);
@@ -426,17 +491,29 @@ async fn test_missing_chunk_triggers_origin_fetch() {
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap(); let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
let chunk_hash = manifest.chunks[0].hash; let chunk_hash = manifest.chunks[0].hash;
let hex = chunk_hash.as_hex(); let hex = chunk_hash.as_hex();
let chunk_path = dir.path().join("chunks").join(&hex[0..2]).join(&hex[2..4]).join(&hex); let chunk_path = dir
.path()
.join("chunks")
.join(&hex[0..2])
.join(&hex[2..4])
.join(&hex);
std::fs::remove_file(&chunk_path).unwrap(); std::fs::remove_file(&chunk_path).unwrap();
let reader = FileReader::with_fetcher(store, fetcher); let reader = FileReader::with_fetcher(store, fetcher);
reader.register_manifest(manifest); reader.register_manifest(manifest);
let result = reader.read(FileId(1), 0, test_content.len() as u32).await; let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
assert!(result.is_ok(), "Issue 6.4: Missing chunk should be re-fetched from origin"); assert!(
assert_eq!(&result.unwrap()[..], test_content, "Data should match original after re-fetch"); result.is_ok(),
"Issue 6.4: Missing chunk should be re-fetched from origin"
);
assert_eq!(
&result.unwrap()[..],
test_content,
"Data should match original after re-fetch"
);
} }
#[tokio::test] #[tokio::test]
@@ -449,15 +526,20 @@ async fn test_passthrough_mode_when_cache_disk_dead() {
let test_content = b"passthrough test data"; let test_content = b"passthrough test data";
setup_test_file(&origin_dir, "test.flac", test_content); setup_test_file(&origin_dir, "test.flac", test_content);
let store = Arc::new(CasStore::open(CasConfig { let store = Arc::new(
chunks_dir: dir.path().join("chunks"), CasStore::open(CasConfig {
max_size: 10, chunks_dir: dir.path().join("chunks"),
shard_levels: 2, max_size: 10,
}) shard_levels: 2,
.await })
.unwrap()); .await
.unwrap(),
let origin = Arc::new(LocalOrigin::new(OriginId::from("local"), origin_dir.path().to_path_buf())); );
let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone())); let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin); fetcher.register_origin(origin);
@@ -477,7 +559,10 @@ async fn test_passthrough_mode_when_cache_disk_dead() {
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap(); let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
assert!(!manifest.chunks.is_empty(), "Issue 6.6: Fetch should complete even when CAS write fails (passthrough mode)"); assert!(
!manifest.chunks.is_empty(),
"Issue 6.6: Fetch should complete even when CAS write fails (passthrough mode)"
);
} }
#[tokio::test] #[tokio::test]
@@ -522,12 +607,18 @@ fn test_pid_file_prevents_concurrent_mount() {
assert!(lock1.is_ok(), "Issue C9: First lock should succeed"); assert!(lock1.is_ok(), "Issue C9: First lock should succeed");
let lock2 = try_lock(&lock_path); let lock2 = try_lock(&lock_path);
assert!(lock2.is_err(), "Issue C9: Second lock should fail (already held)"); assert!(
lock2.is_err(),
"Issue C9: Second lock should fail (already held)"
);
drop(lock1); drop(lock1);
let lock3 = try_lock(&lock_path); let lock3 = try_lock(&lock_path);
assert!(lock3.is_ok(), "Issue C9: Third lock should succeed after first released"); assert!(
lock3.is_ok(),
"Issue C9: Third lock should succeed after first released"
);
} }
#[test] #[test]
@@ -556,7 +647,10 @@ fn test_stale_mount_check_function_exists() {
fn test_systemd_service_has_execstoppost() { fn test_systemd_service_has_execstoppost() {
let service_path = std::path::Path::new("../../dist/musicfs.service"); let service_path = std::path::Path::new("../../dist/musicfs.service");
if !service_path.exists() { if !service_path.exists() {
panic!("Issue 3.7: dist/musicfs.service does not exist at {:?}", service_path); panic!(
"Issue 3.7: dist/musicfs.service does not exist at {:?}",
service_path
);
} }
let content = std::fs::read_to_string(service_path).unwrap(); let content = std::fs::read_to_string(service_path).unwrap();
@@ -574,18 +668,27 @@ fn test_sd_notify_ready_sent() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let socket_path = dir.path().join("notify.sock"); let socket_path = dir.path().join("notify.sock");
let socket = UnixDatagram::bind(&socket_path).unwrap(); let socket = UnixDatagram::bind(&socket_path).unwrap();
socket.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); socket
.set_read_timeout(Some(Duration::from_secs(1)))
.unwrap();
std::env::set_var("NOTIFY_SOCKET", &socket_path); std::env::set_var("NOTIFY_SOCKET", &socket_path);
let result = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]); let result = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]);
assert!(result.is_ok(), "sd_notify should succeed when NOTIFY_SOCKET is set"); assert!(
result.is_ok(),
"sd_notify should succeed when NOTIFY_SOCKET is set"
);
let mut buf = [0u8; 256]; let mut buf = [0u8; 256];
let len = socket.recv(&mut buf).unwrap(); let len = socket.recv(&mut buf).unwrap();
let msg = std::str::from_utf8(&buf[..len]).unwrap(); let msg = std::str::from_utf8(&buf[..len]).unwrap();
assert!(msg.contains("READY=1"), "sd_notify should send READY=1, got: {}", msg); assert!(
msg.contains("READY=1"),
"sd_notify should send READY=1, got: {}",
msg
);
std::env::remove_var("NOTIFY_SOCKET"); std::env::remove_var("NOTIFY_SOCKET");
} }
@@ -615,7 +718,9 @@ async fn test_shutdown_flushes_tantivy() {
{ {
let index = SearchIndex::open(&idx_path).unwrap(); let index = SearchIndex::open(&idx_path).unwrap();
index.index_file(&make_file_meta(1, "/a.flac", 1000)).unwrap(); index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap(); index.commit().unwrap();
} }
@@ -678,7 +783,9 @@ async fn test_sigterm_triggers_shutdown() {
let musicfs_bin = std::env::var("CARGO_BIN_EXE_musicfs").ok(); let musicfs_bin = std::env::var("CARGO_BIN_EXE_musicfs").ok();
if musicfs_bin.is_none() { if musicfs_bin.is_none() {
eprintln!("Skipping test_sigterm_triggers_shutdown: musicfs binary not available in test context"); eprintln!(
"Skipping test_sigterm_triggers_shutdown: musicfs binary not available in test context"
);
return; return;
} }
@@ -690,7 +797,12 @@ async fn test_sigterm_triggers_shutdown() {
std::fs::create_dir_all(&origin).unwrap(); std::fs::create_dir_all(&origin).unwrap();
let mut child = Command::new(&bin_path) let mut child = Command::new(&bin_path)
.args(["mount", "--origin", origin.to_str().unwrap(), mountpoint.to_str().unwrap()]) .args([
"mount",
"--origin",
origin.to_str().unwrap(),
mountpoint.to_str().unwrap(),
])
.stdout(Stdio::null()) .stdout(Stdio::null())
.stderr(Stdio::null()) .stderr(Stdio::null())
.spawn(); .spawn();
@@ -716,7 +828,11 @@ async fn test_sigterm_triggers_shutdown() {
} }
} }
child.wait().unwrap() child.wait().unwrap()
}).await; })
.await;
assert!(exit_result.is_ok(), "Issue 2.1: Process should exit within 10s after SIGTERM"); assert!(
exit_result.is_ok(),
"Issue 2.1: Process should exit within 10s after SIGTERM"
);
} }
Generated
+80 -7
View File
@@ -1,5 +1,21 @@
{ {
"nodes": { "nodes": {
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1767039857,
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
"owner": "NixOS",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": { "flake-utils": {
"inputs": { "inputs": {
"systems": "systems" "systems": "systems"
@@ -18,19 +34,75 @@
"type": "github" "type": "github"
} }
}, },
"nixpkgs-old": { "git-hooks": {
"flake": false, "inputs": {
"flake-compat": "flake-compat",
"gitignore": "gitignore",
"nixpkgs": "nixpkgs"
},
"locked": { "locked": {
"lastModified": 1558254092, "lastModified": 1778507602,
"narHash": "sha256-5v6XuO9dOVpB3ZGNyDvLqOvCnzlyyvscTYbNiQouSZo=", "narHash": "sha256-kTwur1wV+01SdqskVMSo6JMEpg71ps3HpbFY2GsflKs=",
"owner": "cachix",
"repo": "git-hooks.nix",
"rev": "61ab0e80d9c7ab14c256b5b453d8b3fb0189ba0a",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "git-hooks.nix",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"git-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1709087332,
"narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1770073757,
"narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "a7e559a5504572008567383c3dc8e142fa7a8633", "rev": "47472570b1e607482890801aeaf29bfb749884f6",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "NixOS", "owner": "NixOS",
"ref": "nixos-18.09", "ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1778443072,
"narHash": "sha256-zi7/fsqM/kFdNuED//4WOCUtezGtKKqRNORjMvfwjnA=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "da5ad661ba4e5ef59ba743f0d112cbc30e474f32",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs", "repo": "nixpkgs",
"type": "github" "type": "github"
} }
@@ -38,7 +110,8 @@
"root": { "root": {
"inputs": { "inputs": {
"flake-utils": "flake-utils", "flake-utils": "flake-utils",
"nixpkgs-old": "nixpkgs-old" "git-hooks": "git-hooks",
"nixpkgs": "nixpkgs_2"
} }
}, },
"systems": { "systems": {
+61 -104
View File
@@ -1,115 +1,72 @@
{ {
description = "beetfs - FUSE filesystem for beets with metadata overlay (Python 2.7)"; description = "MusicFS - FUSE filesystem for music with metadata overlay";
inputs = { inputs = {
# Using nixos-18.09 - has all Python 2 packages: fuse, mutagen, jellyfish, munkres nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
# Mark as non-flake since 18.09 predates flakes
nixpkgs-old = {
url = "github:NixOS/nixpkgs/nixos-18.09";
flake = false;
};
flake-utils.url = "github:numtide/flake-utils"; flake-utils.url = "github:numtide/flake-utils";
git-hooks.url = "github:cachix/git-hooks.nix";
}; };
outputs = { self, nixpkgs-old, flake-utils }: outputs = { self, nixpkgs, flake-utils, git-hooks }:
flake-utils.lib.eachDefaultSystem (system: flake-utils.lib.eachDefaultSystem (system:
let let
pkgs = import nixpkgs-old { pkgs = import nixpkgs {
inherit system; inherit system;
}; };
# Build beets 1.4.9 for Python 2 (last Python 2 compatible version) pre-commit-check = git-hooks.lib.${system}.run {
# Using minimal deps - autotagger features (munkres, jellyfish) not needed for beetfs src = ./.;
beets-py2 = pkgs.python2Packages.buildPythonPackage rec { hooks = {
pname = "beets"; rustfmt = {
version = "1.4.9"; enable = true;
packageOverrides = {
# Use legacy setup.py install (not pip/wheel) cargo = pkgs.cargo;
format = "other"; rustfmt = pkgs.rustfmt;
src = pkgs.fetchFromGitHub {
owner = "beetbox";
repo = "beets";
rev = "v${version}";
sha256 = "sha256-KO5jtqxw82ylbSKsJgUGr3bfxUPH8vanf/+kv//CreM=";
};
# No patching needed - nixpkgs-18.09 has all required Python 2 packages
nativeBuildInputs = with pkgs.python2Packages; [ setuptools ];
buildPhase = "true"; # No build needed
installPhase = ''
# Direct copy to avoid dependency resolution
mkdir -p $out/lib/python2.7/site-packages
cp -r beets $out/lib/python2.7/site-packages/
cp -r beetsplug $out/lib/python2.7/site-packages/
# Create version file
echo "__version__ = '${version}'" >> $out/lib/python2.7/site-packages/beets/__init__.py
# Create beet command wrapper
mkdir -p $out/bin
cat > $out/bin/beet << 'EOF'
#!${pkgs.python2.interpreter}
import sys
from beets.ui import main
main()
EOF
chmod +x $out/bin/beet
'';
propagatedBuildInputs = with pkgs.python2Packages; [
pyyaml
mutagen
unidecode
enum34
six
musicbrainzngs
jellyfish
munkres
];
# Disable tests for faster builds
doCheck = false;
meta = {
description = "Music library manager and tagger";
homepage = "https://beets.io/";
}; };
}; };
clippy = {
pythonEnv = pkgs.python2.withPackages (ps: with ps; [ enable = true;
fuse packageOverrides = {
mutagen cargo = pkgs.cargo;
pyyaml clippy = pkgs.clippy;
enum34 };
six
jellyfish
munkres
unidecode
musicbrainzngs
] ++ [ beets-py2 ]);
in {
devShells.default = pkgs.mkShell {
buildInputs = [
pythonEnv
pkgs.fuse
pkgs.ffmpeg
pkgs.flac
];
shellHook = ''
unset PYTHONPATH
export PYTHONPATH="$PWD/beetsplug:$PWD/tests"
echo "beetfs development environment (Python 2.7)"
echo " Python: $(python --version 2>&1)"
echo " Run tests: cd tests && python -m unittest discover"
echo " Mount: beet mount <mountpoint>"
'';
}; };
} };
); };
in {
checks = {
inherit pre-commit-check;
};
devShells.default = pkgs.mkShell rec {
inherit (pre-commit-check) shellHook;
buildInputs = with pkgs; [
pre-commit
gitleaks
just
pkg-config
fuse3
sqlite
openssl
rustc
cargo
cargo-watch
cargo-nextest
cargo-criterion
rust-analyzer
clippy
rustfmt
clang
lld
protobuf
grpcurl
];
};
});
} }
-10
View File
@@ -1,10 +0,0 @@
[build]
rustflags = ["-C", "link-arg=-fuse-ld=lld"]
[target.x86_64-unknown-linux-gnu]
linker = "clang"
[alias]
t = "test"
c = "check"
b = "build"
-1
View File
@@ -1 +0,0 @@
target/
-23
View File
@@ -1,23 +0,0 @@
pkgname=musicfs
pkgver=0.1.0
pkgrel=1
pkgdesc="Metadata-Organized Music Filesystem"
arch=('x86_64')
url="https://github.com/yourusername/musicfs"
license=('MIT')
depends=('fuse3')
makedepends=('rust' 'cargo')
source=("$pkgname-$pkgver.tar.gz")
sha256sums=('SKIP')
build() {
cd "$srcdir/$pkgname-$pkgver"
cargo build --release --locked
}
package() {
cd "$srcdir/$pkgname-$pkgver"
install -Dm755 "target/release/musicfs" "$pkgdir/usr/bin/musicfs"
install -Dm644 "dist/musicfs.service" "$pkgdir/usr/lib/systemd/system/musicfs.service"
install -Dm644 "config.example.toml" "$pkgdir/etc/musicfs/config.example.toml"
}
-30
View File
@@ -1,30 +0,0 @@
mount_point = "/mnt/music"
cache_dir = "/var/cache/musicfs"
[logging]
log_dir = "/var/log/musicfs"
json_output = true
journald = true
level = "musicfs=info,warn"
trace_sample_rate = 1.0
[cache]
metadata_cache_mb = 100
content_cache_gb = 10
[health]
check_interval_secs = 30
timeout_ms = 5000
unhealthy_threshold = 3
[[origins]]
id = "local"
origin_type = "local"
priority = 1
path = "/srv/music"
[[origins]]
id = "nas"
origin_type = "nfs"
priority = 2
mount_point = "/mnt/nas/music"
-9
View File
@@ -1,9 +0,0 @@
/var/log/musicfs/*.log {
daily
rotate 30
compress
delaycompress
missingok
notifempty
create 0640 musicfs musicfs
}
-29
View File
@@ -1,29 +0,0 @@
[Unit]
Description=MusicFS - Metadata-Organized Music Filesystem
After=network.target
[Service]
Type=notify
ExecStart=/usr/bin/musicfs mount --config /etc/musicfs/config.toml /mnt/music
ExecStop=/usr/bin/musicfs shutdown
ExecStopPost=/usr/bin/fusermount -uz /mnt/music || true
Restart=on-failure
RestartSec=5
User=musicfs
Group=musicfs
Environment="RUST_LOG=musicfs=info,warn"
StandardOutput=journal
StandardError=journal
SyslogIdentifier=musicfs
RateLimitIntervalSec=30s
RateLimitBurst=1000
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/var/cache/musicfs /var/log/musicfs /mnt/music
PrivateTmp=true
[Install]
WantedBy=multi-user.target
-39
View File
@@ -1,39 +0,0 @@
Name: musicfs
Version: 0.1.0
Release: 1%{?dist}
Summary: Metadata-Organized Music Filesystem
License: MIT
URL: https://github.com/yourusername/musicfs
Source0: %{name}-%{version}.tar.gz
BuildRequires: rust >= 1.70
BuildRequires: cargo
BuildRequires: fuse3-devel
Requires: fuse3
%description
MusicFS is a virtual FUSE filesystem that organizes music files by metadata.
%prep
%autosetup
%build
cargo build --release --locked
%install
install -Dm755 target/release/musicfs %{buildroot}%{_bindir}/musicfs
install -Dm644 dist/musicfs.service %{buildroot}%{_unitdir}/musicfs.service
install -Dm644 config.example.toml %{buildroot}%{_sysconfdir}/musicfs/config.example.toml
%files
%license LICENSE
%doc README.md
%{_bindir}/musicfs
%{_unitdir}/musicfs.service
%config(noreplace) %{_sysconfdir}/musicfs/config.example.toml
%changelog
* Mon Jan 01 2024 MusicFS Team <team@example.com> - 0.1.0-1
- Initial package

Some files were not shown because too many files have changed in this diff Show More