Move the files around

This commit is contained in:
Alexander
2026-05-13 20:34:14 +02:00
parent 90e9683076
commit 305d027c8b
113 changed files with 650 additions and 3569 deletions
+33
View File
@@ -14,4 +14,37 @@ tests/*.log
# Nix
result
.cargo/
.direnv/
.pre-commit-config.yaml
dist/
###
# Rust
###
result-*
# Generated by Cargo
# will have compiled files and executables
debug
target
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
# Generated by cargo mutants
# Contains mutation testing data
**/mutants.out*/
# rustc will dump stack traces when hitting an internal compiler error to PWD
rustc-ice-*.txt
# RustRover
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
View File
View File
-7
View File
@@ -1,7 +0,0 @@
Organising a music library can be a hassle. With the wealth of online stores all providing music tagged in various formats, it can be a nightmare to unify them all.
This is where beetFs comes in. Derived from beets, beetFs presents a FUSE filesystem that is based on your tags.
Modifying the tags within the beetFs mountpoint will not change the data on the hard disk, merely update the beet database. When an application requests a music file from within the beetFs mountpoint, beetFs provides tag information from its own database, instead of from the original file, but music data from the on-disk location.
This enables completely transparent modification of tags within an audio file with no change to the underlying on-disk data.
-2
View File
@@ -1,2 +0,0 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
-1144
View File
File diff suppressed because it is too large Load Diff
@@ -48,9 +48,18 @@ impl ArtworkCache {
}
pub async fn store(&self, file_id: i64, artwork: &Artwork) -> Result<ChunkHash, ArtworkError> {
trace!(file_id = file_id, size_bytes = artwork.data.len(), "Storing artwork");
trace!(
file_id = file_id,
size_bytes = artwork.data.len(),
"Storing artwork"
);
if artwork.data.len() > MAX_ARTWORK_INPUT_SIZE {
warn!(file_id = file_id, size = artwork.data.len(), max = MAX_ARTWORK_INPUT_SIZE, "Artwork too large");
warn!(
file_id = file_id,
size = artwork.data.len(),
max = MAX_ARTWORK_INPUT_SIZE,
"Artwork too large"
);
return Err(ArtworkError::ImageTooLarge(artwork.data.len()));
}
@@ -35,8 +35,8 @@ impl Database {
pub fn open_with_integrity_check(path: &Path) -> Result<Self> {
debug!(?path, "Opening database with integrity check");
let conn = Connection::open(path)
.map_err(|e| Error::Database(format!("open failed: {}", e)))?;
let conn =
Connection::open(path).map_err(|e| Error::Database(format!("open failed: {}", e)))?;
let integrity: String = conn
.query_row("PRAGMA integrity_check(1)", [], |row| row.get(0))
@@ -45,7 +45,8 @@ impl Database {
if integrity != "ok" {
warn!(path = ?path, result = %integrity, "Database integrity check failed");
return Err(Error::DatabaseCorrupted(format!(
"integrity check failed: {}", integrity
"integrity check failed: {}",
integrity
)));
}
@@ -250,11 +251,9 @@ impl Database {
pub fn file_count(&self) -> Result<u64> {
let conn = self.conn.lock().unwrap();
conn.query_row("SELECT COUNT(*) FROM files", [], |row| {
row.get::<_, i64>(0)
})
.map(|c| c as u64)
.map_err(|e| Error::Database(format!("count failed: {}", e)))
conn.query_row("SELECT COUNT(*) FROM files", [], |row| row.get::<_, i64>(0))
.map(|c| c as u64)
.map_err(|e| Error::Database(format!("count failed: {}", e)))
}
pub fn update_content_hash(&self, id: FileId, hash: &ContentHash) -> Result<()> {
@@ -352,10 +351,7 @@ mod tests {
)
.unwrap();
let retrieved = db
.get_file_by_virtual_path(&virtual_path)
.unwrap()
.unwrap();
let retrieved = db.get_file_by_virtual_path(&virtual_path).unwrap().unwrap();
assert_eq!(retrieved.id, id);
assert_eq!(
retrieved.audio.as_ref().unwrap().title,
@@ -401,10 +397,7 @@ mod tests {
assert_eq!(db.file_count().unwrap(), 1);
let retrieved = db
.get_file_by_virtual_path(&virtual_path)
.unwrap()
.unwrap();
let retrieved = db.get_file_by_virtual_path(&virtual_path).unwrap().unwrap();
assert_eq!(
retrieved.audio.as_ref().unwrap().title,
Some("Updated".to_string())
@@ -94,7 +94,14 @@ mod tests {
};
cache
.store(&origin_id, real_path, &virtual_path, &meta, UNIX_EPOCH, 5000)
.store(
&origin_id,
real_path,
&virtual_path,
&meta,
UNIX_EPOCH,
5000,
)
.unwrap();
let retrieved = cache.lookup(&virtual_path).unwrap().unwrap();
@@ -63,13 +63,11 @@ impl PatternStore {
let sequence_counts = {
let mut map = HashMap::new();
let mut stmt = db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?;
let mut stmt =
db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?;
let rows = stmt.query_map([], |row| {
Ok((
(
FileId(row.get::<_, i64>(0)?),
FileId(row.get::<_, i64>(1)?),
),
(FileId(row.get::<_, i64>(0)?), FileId(row.get::<_, i64>(1)?)),
row.get::<_, u32>(2)?,
))
})?;
@@ -154,7 +152,11 @@ impl PatternStore {
.take(limit)
.map(|(id, _)| id)
.collect();
debug!(file_id = current.0, predictions = result.len(), "Predicted next files");
debug!(
file_id = current.0,
predictions = result.len(),
"Predicted next files"
);
result
}
@@ -102,13 +102,8 @@ impl PrefetchEngine {
pattern_store.predict_next(file_id, config.lookahead);
for predicted_id in predictions {
prefetch_file(
predicted_id,
&fetcher,
&in_flight,
&semaphore,
)
.await;
prefetch_file(predicted_id, &fetcher, &in_flight, &semaphore)
.await;
}
tokio::time::sleep(config.cooldown).await;
@@ -102,8 +102,7 @@ impl VirtualTree {
mtime: SystemTime::now(),
}),
);
tree.path_to_inode
.insert(VirtualPath::new("/"), ROOT_INODE);
tree.path_to_inode.insert(VirtualPath::new("/"), ROOT_INODE);
tree
}
@@ -161,13 +160,11 @@ impl VirtualTree {
fn find_parent_by_path_lookup(&self, inode: Inode) -> Option<Inode> {
for (path, &ino) in &self.path_to_inode {
if ino == inode {
return std::path::Path::new(path.as_str())
.parent()
.and_then(|p| {
self.path_to_inode
.get(&VirtualPath::new(p.to_string_lossy().into_owned()))
.copied()
});
return std::path::Path::new(path.as_str()).parent().and_then(|p| {
self.path_to_inode
.get(&VirtualPath::new(p.to_string_lossy().into_owned()))
.copied()
});
}
}
None
@@ -69,11 +69,7 @@ impl ContentFetcher {
.ok_or_else(|| FetchError::OriginNotFound(meta.real_path.origin_id.clone()))?
};
info!(
"Fetching file {:?} from origin {}",
file_id,
origin.id()
);
info!("Fetching file {:?} from origin {}", file_id, origin.id());
let data = origin
.read_full(&meta.real_path.path)
@@ -26,7 +26,12 @@ impl ChunkManifest {
rmp_serde::from_slice(data).ok()
}
pub fn from_db(file_id: FileId, total_size: u64, mtime: i64, chunk_blob: &[u8]) -> Option<Self> {
pub fn from_db(
file_id: FileId,
total_size: u64,
mtime: i64,
chunk_blob: &[u8],
) -> Option<Self> {
let chunks = Self::chunks_from_bytes(chunk_blob)?;
Some(Self {
file_id,
@@ -80,9 +85,7 @@ impl FileReader {
};
let manifest = fetcher.ensure_cached(file_id).await?;
self.manifests
.write()
.insert(file_id, manifest.clone());
self.manifests.write().insert(file_id, manifest.clone());
Ok(manifest)
}
@@ -126,7 +129,9 @@ impl FileReader {
self.manifests.write().insert(file_id, new_manifest);
self.store.get(&chunk_ref.hash).await?
} else {
return Err(ReaderError::Cas(CasError::NotFound(chunk_ref.hash.as_hex())));
return Err(ReaderError::Cas(CasError::NotFound(
chunk_ref.hash.as_hex(),
)));
}
}
Err(CasError::NotFound(_)) => {
@@ -136,7 +141,9 @@ impl FileReader {
self.manifests.write().insert(file_id, new_manifest);
self.store.get(&chunk_ref.hash).await?
} else {
return Err(ReaderError::Cas(CasError::NotFound(chunk_ref.hash.as_hex())));
return Err(ReaderError::Cas(CasError::NotFound(
chunk_ref.hash.as_hex(),
)));
}
}
Err(e) => return Err(ReaderError::Cas(e)),
@@ -58,8 +58,7 @@ impl CasStore {
Err(repair_err) => {
warn!(error = %repair_err, "sled repair failed, recreating index");
if index_path.exists() {
std::fs::remove_dir_all(&index_path)
.map_err(CasError::Io)?;
std::fs::remove_dir_all(&index_path).map_err(CasError::Io)?;
}
sled::open(&index_path)?
}
@@ -80,7 +79,9 @@ impl CasStore {
Self::calculate_size_recursive(dir).await
}
fn calculate_size_recursive(dir: &Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = u64> + Send + '_>> {
fn calculate_size_recursive(
dir: &Path,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = u64> + Send + '_>> {
Box::pin(async move {
let mut size = 0u64;
if let Ok(mut entries) = fs::read_dir(dir).await {
@@ -117,7 +117,10 @@ async fn test_fetcher_cache_miss_flow() {
let store = Arc::new(CasStore::open(config).await.unwrap());
let origin_id = OriginId::from("test-origin");
let origin = Arc::new(LocalOrigin::new(origin_id.clone(), origin_dir.path().to_path_buf()));
let origin = Arc::new(LocalOrigin::new(
origin_id.clone(),
origin_dir.path().to_path_buf(),
));
let fetcher = ContentFetcher::new(store.clone());
fetcher.register_origin(origin);
@@ -163,7 +166,10 @@ async fn test_reader_with_fetcher_integration() {
let store = Arc::new(CasStore::open(config).await.unwrap());
let origin_id = OriginId::from("local");
let origin = Arc::new(LocalOrigin::new(origin_id.clone(), origin_dir.path().to_path_buf()));
let origin = Arc::new(LocalOrigin::new(
origin_id.clone(),
origin_dir.path().to_path_buf(),
));
let fetcher = ContentFetcher::new(store.clone());
fetcher.register_origin(origin);
@@ -82,12 +82,8 @@ enum CacheCommands {
#[derive(Subcommand)]
enum OriginCommands {
List,
Health {
origin_id: String,
},
Rescan {
origin_id: String,
},
Health { origin_id: String },
Rescan { origin_id: String },
}
struct LockFile {
@@ -245,8 +241,7 @@ fn run_mount(
runtime.block_on(async {
let mut sigterm =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
let mut sigint =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
tokio::select! {
_ = sigterm.recv() => {
@@ -290,10 +285,7 @@ fn run_cache(command: CacheCommands) -> Result<()> {
println!("Cache stats: gRPC client integration pending");
}
CacheCommands::Clear { origin } => {
println!(
"Clearing cache for: {}",
origin.as_deref().unwrap_or("all")
);
println!("Clearing cache for: {}", origin.as_deref().unwrap_or("all"));
println!("gRPC client integration pending");
}
CacheCommands::Prefetch { paths } => {
@@ -364,8 +356,8 @@ fn init_logging(config: &LoggingConfig) -> Result<WorkerGuard> {
let stderr_layer = fmt::layer().with_writer(std::io::stderr).compact();
let filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new(&config.level));
let filter =
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&config.level));
let subscriber = tracing_subscriber::registry()
.with(filter)
@@ -488,10 +480,7 @@ fn build_virtual_path(path: &Path, audio: Option<&musicfs_core::AudioMeta>) -> V
if let Some(meta) = audio {
let artist = meta.artist.as_deref().unwrap_or("Unknown Artist");
let album = meta.album.as_deref().unwrap_or("Unknown Album");
let filename = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("track");
let filename = path.file_name().and_then(|n| n.to_str()).unwrap_or("track");
VirtualPath::new(&format!(
"/{}/{}/{}",
@@ -16,7 +16,10 @@ impl EventBus {
trace!(event = ?event, "Publishing event");
let receiver_count = self.sender.receiver_count();
if self.sender.send(event).is_err() && receiver_count > 0 {
debug!(receiver_count = receiver_count, "Event dropped, no active receivers");
debug!(
receiver_count = receiver_count,
"Event dropped, no active receivers"
);
}
}
@@ -22,9 +22,7 @@ impl Metrics {
}
pub fn uptime_secs(&self) -> u64 {
self.start_time
.map(|t| t.elapsed().as_secs())
.unwrap_or(0)
self.start_time.map(|t| t.elapsed().as_secs()).unwrap_or(0)
}
pub fn to_prometheus(&self) -> String {
@@ -55,11 +53,16 @@ impl Metrics {
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.99\"}} {:.6}\n\
musicfs_fuse_latency_seconds_sum{{op=\"{}\"}} {:.6}\n\
musicfs_fuse_latency_seconds_count{{op=\"{}\"}} {}\n",
op, quantiles.p50,
op, quantiles.p95,
op, quantiles.p99,
op, histogram.sum_secs(),
op, histogram.count(),
op,
quantiles.p50,
op,
quantiles.p95,
op,
quantiles.p99,
op,
histogram.sum_secs(),
op,
histogram.count(),
));
}
@@ -266,9 +269,7 @@ pub struct OriginHealthMetrics {
impl OriginHealthMetrics {
pub fn set_health(&self, origin_id: &str, healthy: bool) {
self.status
.write()
.insert(origin_id.to_string(), healthy);
self.status.write().insert(origin_id.to_string(), healthy);
}
}
@@ -46,7 +46,11 @@ impl MusicFs {
}
}
pub fn with_reader(tree: Arc<RwLock<VirtualTree>>, reader: Arc<FileReader>, runtime_handle: Handle) -> Self {
pub fn with_reader(
tree: Arc<RwLock<VirtualTree>>,
reader: Arc<FileReader>,
runtime_handle: Handle,
) -> Self {
Self {
tree,
reader: Some(reader),
@@ -287,7 +291,12 @@ impl Filesystem for MusicFs {
let tree = self.tree.read();
if let Some(children) = tree.readdir(ino) {
trace!(ino, offset, children_count = children.len(), "directory found");
trace!(
ino,
offset,
children_count = children.len(),
"directory found"
);
let parent_ino = tree.get_parent(ino).unwrap_or(ROOT_INODE);
let entries: Vec<(u64, FileType, &str)> = vec![
@@ -396,7 +405,13 @@ impl Filesystem for MusicFs {
match result {
Ok(Ok(data)) => {
trace!(ino, offset, size_bytes = size, bytes_read = data.len(), "read successful");
trace!(
ino,
offset,
size_bytes = size,
bytes_read = data.len(),
"read successful"
);
reply.data(&data);
}
Ok(Err(e)) => {
@@ -591,6 +606,8 @@ mod tests {
let tree_read = tree.read();
assert!(tree_read.get(ROOT_INODE).is_some());
assert!(tree_read.get_by_path(&VirtualPath::new("/Artist")).is_some());
assert!(tree_read
.get_by_path(&VirtualPath::new("/Artist"))
.is_some());
}
}
@@ -43,10 +43,7 @@ impl PrefetchOps {
}
}
pub fn start_engine(
&self,
event_bus: Arc<EventBus>,
) -> Option<musicfs_cache::PrefetchHandle> {
pub fn start_engine(&self, event_bus: Arc<EventBus>) -> Option<musicfs_cache::PrefetchHandle> {
self.engine
.as_ref()
.map(|e| e.clone().start(event_bus, self.pattern_store.clone()))
@@ -266,7 +263,8 @@ mod tests {
#[test]
fn test_prefetch_ops_new() {
let dir = TempDir::new().unwrap();
let pattern_store = Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
let pattern_store =
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
let _ops = PrefetchOps::new(pattern_store, 1000, 1000);
}
@@ -283,11 +281,18 @@ mod tests {
#[test]
fn test_hint_name_to_inode() {
let dir = TempDir::new().unwrap();
let pattern_store = Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
let pattern_store =
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
let ops = PrefetchOps::new(pattern_store, 1000, 1000);
assert_eq!(ops.hint_name_to_inode("hint_0001"), Some(PREFETCH_HINTS_BASE + 1));
assert_eq!(ops.hint_name_to_inode("hint_9999"), Some(PREFETCH_HINTS_BASE + 9999));
assert_eq!(
ops.hint_name_to_inode("hint_0001"),
Some(PREFETCH_HINTS_BASE + 1)
);
assert_eq!(
ops.hint_name_to_inode("hint_9999"),
Some(PREFETCH_HINTS_BASE + 9999)
);
assert_eq!(ops.hint_name_to_inode("invalid"), None);
}
}
@@ -160,16 +160,17 @@ impl SearchOps {
}
fn safe_symlink_target(&self, virtual_path: &str) -> Option<String> {
let normalized = Path::new(virtual_path)
.components()
.fold(std::path::PathBuf::new(), |mut acc, comp| {
let normalized = Path::new(virtual_path).components().fold(
std::path::PathBuf::new(),
|mut acc, comp| {
match comp {
std::path::Component::Normal(s) => acc.push(s),
std::path::Component::RootDir => acc.push("/"),
_ => {}
}
acc
});
},
);
let path_str = normalized.to_string_lossy();
if path_str.contains("..") {
@@ -198,7 +199,9 @@ impl SearchOps {
fn result_filename(&self, hit: &SearchHit, index: usize) -> String {
let artist = hit.artist.as_deref().unwrap_or("Unknown");
let title = hit.title.as_deref().unwrap_or("Unknown");
let ext = hit.virtual_path.as_str()
let ext = hit
.virtual_path
.as_str()
.rsplit('.')
.next()
.unwrap_or("flac");
@@ -35,7 +35,9 @@ impl MusicFs for SearchService {
}
if req.query.len() > 256 {
return Err(Status::invalid_argument("Query exceeds maximum length (256)"));
return Err(Status::invalid_argument(
"Query exceeds maximum length (256)",
));
}
let limit = req.limit.unwrap_or(100).min(10000) as usize;
@@ -228,10 +228,7 @@ impl MusicFs for MusicFsServer {
}
#[instrument(level = "info", skip(self, request), fields(method = "shutdown"))]
async fn shutdown(
&self,
request: Request<ShutdownRequest>,
) -> Result<Response<Empty>, Status> {
async fn shutdown(&self, request: Request<ShutdownRequest>) -> Result<Response<Empty>, Status> {
let req = request.into_inner();
info!(
graceful = req.graceful,
@@ -242,7 +239,11 @@ impl MusicFs for MusicFsServer {
Ok(Response::new(Empty {}))
}
#[instrument(level = "debug", skip(self, _request), fields(method = "get_cache_stats"))]
#[instrument(
level = "debug",
skip(self, _request),
fields(method = "get_cache_stats")
)]
async fn get_cache_stats(
&self,
_request: Request<Empty>,
@@ -339,7 +340,11 @@ impl MusicFs for MusicFsServer {
Ok(Response::new(OriginsResponse { origins: vec![] }))
}
#[instrument(level = "debug", skip(self, request), fields(method = "get_origin_health"))]
#[instrument(
level = "debug",
skip(self, request),
fields(method = "get_origin_health")
)]
async fn get_origin_health(
&self,
request: Request<OriginRequest>,
@@ -389,7 +394,11 @@ impl MusicFs for MusicFsServer {
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
#[instrument(level = "info", skip(self, request), fields(method = "subscribe_events"))]
#[instrument(
level = "info",
skip(self, request),
fields(method = "subscribe_events")
)]
async fn subscribe_events(
&self,
request: Request<EventFilter>,
@@ -52,8 +52,7 @@ impl MetadataParser {
if let Some(n_frames) = params.n_frames {
if let Some(sample_rate) = params.sample_rate {
audio_meta.duration_ms =
Some((n_frames as u64 * 1000) / sample_rate as u64);
audio_meta.duration_ms = Some((n_frames as u64 * 1000) / sample_rate as u64);
audio_meta.sample_rate = Some(sample_rate);
}
}
@@ -61,9 +60,8 @@ impl MetadataParser {
if let Some(bits_per_sample) = params.bits_per_sample {
if let Some(sample_rate) = params.sample_rate {
if let Some(channels) = params.channels {
audio_meta.bitrate = Some(
bits_per_sample * sample_rate * channels.count() as u32 / 1000,
);
audio_meta.bitrate =
Some(bits_per_sample * sample_rate * channels.count() as u32 / 1000);
}
}
}
@@ -67,11 +67,10 @@ impl FailoverExecutor {
if origins.is_empty() {
if let Some(origin) = self.registry.route_with_fallback(path) {
warn!(
"No healthy origins, using fallback origin {}",
origin.id()
);
return self.read_with_retry(&origin, &path.path, offset, size).await;
warn!("No healthy origins, using fallback origin {}", origin.id());
return self
.read_with_retry(&origin, &path.path, offset, size)
.await;
}
return Err(Error::NoOriginAvailable);
}
@@ -81,7 +80,10 @@ impl FailoverExecutor {
for origin in origins {
trace!(origin_id = %origin.id(), "Attempting read from origin");
let start = std::time::Instant::now();
match self.read_with_retry(&origin, &path.path, offset, size).await {
match self
.read_with_retry(&origin, &path.path, offset, size)
.await
{
Ok(data) => {
let latency = start.elapsed().as_millis() as u64;
self.registry.record_latency(origin.id(), latency);
@@ -214,10 +216,8 @@ mod tests {
#[test]
fn test_custom_delays() {
let config = RetryConfig::with_delays(vec![
Duration::from_millis(50),
Duration::from_millis(100),
]);
let config =
RetryConfig::with_delays(vec![Duration::from_millis(50), Duration::from_millis(100)]);
assert_eq!(config.max_attempts, 2);
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(50));
@@ -349,10 +349,13 @@ mod tests {
let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 3);
let monitor = HealthMonitor::new(Duration::from_secs(30))
.with_per_type_thresholds(thresholds);
let monitor =
HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds);
let origin = Arc::new(LocalOrigin::new("missing", std::path::Path::new("/nonexistent")));
let origin = Arc::new(LocalOrigin::new(
"missing",
std::path::Path::new("/nonexistent"),
));
monitor.add_origin(origin);
monitor.check_now(&OriginId::from("missing")).await;
@@ -372,7 +375,10 @@ mod tests {
async fn test_local_origin_threshold_is_one() {
let monitor = HealthMonitor::new(Duration::from_secs(30));
let origin = Arc::new(LocalOrigin::new("missing", std::path::Path::new("/nonexistent")));
let origin = Arc::new(LocalOrigin::new(
"missing",
std::path::Path::new("/nonexistent"),
));
monitor.add_origin(origin);
monitor.check_now(&OriginId::from("missing")).await;
@@ -47,5 +47,3 @@
mod implementation {
// Full S3 implementation would go here when aws-sdk-s3 is enabled
}
@@ -91,11 +91,13 @@ impl Origin for SmbOrigin {
}
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
self.retry_on_disconnect(|| self.inner.read(path, offset, size)).await
self.retry_on_disconnect(|| self.inner.read(path, offset, size))
.await
}
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
self.retry_on_disconnect(|| self.inner.read_full(path)).await
self.retry_on_disconnect(|| self.inner.read_full(path))
.await
}
async fn exists(&self, path: &Path) -> Result<bool> {
@@ -55,9 +55,8 @@ impl NativePluginHost {
info!("Loading native plugin from {:?}", canonical);
let library = unsafe {
Library::new(&canonical).map_err(|e| {
PluginError::LoadFailed(format!("Failed to load library: {}", e))
})?
Library::new(&canonical)
.map_err(|e| PluginError::LoadFailed(format!("Failed to load library: {}", e)))?
};
self.verify_api_version(&library)?;
@@ -190,9 +189,9 @@ impl NativePluginHost {
fn verify_api_version(&self, library: &Library) -> Result<()> {
let version_fn: Symbol<unsafe extern "C" fn() -> *const std::ffi::c_char> = unsafe {
library
.get(b"musicfs_plugin_api_version")
.map_err(|_| PluginError::SymbolNotFound("musicfs_plugin_api_version".to_string()))?
library.get(b"musicfs_plugin_api_version").map_err(|_| {
PluginError::SymbolNotFound("musicfs_plugin_api_version".to_string())
})?
};
let version_ptr = unsafe { version_fn() };
@@ -203,10 +202,11 @@ impl NativePluginHost {
actual: "<invalid UTF-8>".to_string(),
})?;
let plugin_version = Version::parse(version_str).map_err(|_| PluginError::VersionMismatch {
expected: PLUGIN_API_VERSION.to_string(),
actual: version_str.to_string(),
})?;
let plugin_version =
Version::parse(version_str).map_err(|_| PluginError::VersionMismatch {
expected: PLUGIN_API_VERSION.to_string(),
actual: version_str.to_string(),
})?;
let expected_version = Version::parse(PLUGIN_API_VERSION).unwrap();
@@ -95,11 +95,7 @@ pub trait OriginPlugin: Plugin {
///
/// The config contains origin-specific settings (credentials, paths, etc).
/// Returns a boxed Origin that can be used by the OriginRouter.
async fn create_origin(
&self,
id: &str,
config: Value,
) -> Result<Box<dyn OriginInstance>>;
async fn create_origin(&self, id: &str, config: Value) -> Result<Box<dyn OriginInstance>>;
}
/// Instance created by OriginPlugin
@@ -261,7 +261,12 @@ mod tests {
let store = CollectionStore::new(&db_path).unwrap();
let collection = store
.create("Jazz", CollectionQuery::Genre { genre: "Jazz".to_string() })
.create(
"Jazz",
CollectionQuery::Genre {
genre: "Jazz".to_string(),
},
)
.unwrap();
assert_eq!(collection.name, "Jazz");
@@ -279,7 +284,9 @@ mod tests {
let query = CollectionQuery::Compound {
op: BoolOp::And,
children: vec![
CollectionQuery::Genre { genre: "Metal".to_string() },
CollectionQuery::Genre {
genre: "Metal".to_string(),
},
CollectionQuery::DateRange {
field: "year".to_string(),
start: 1980,
@@ -306,6 +313,9 @@ mod tests {
assert!(CollectionQuery::RecentlyAdded { days: 30 }.is_dynamic());
assert!(CollectionQuery::RecentlyPlayed { days: 7 }.is_dynamic());
assert!(CollectionQuery::MostPlayed { limit: 100 }.is_dynamic());
assert!(!CollectionQuery::Genre { genre: "Rock".to_string() }.is_dynamic());
assert!(!CollectionQuery::Genre {
genre: "Rock".to_string()
}
.is_dynamic());
}
}
@@ -4,7 +4,7 @@ use std::path::Path;
use std::sync::Arc;
use tantivy::collector::TopDocs;
use tantivy::query::{BooleanQuery, FuzzyTermQuery, Occur, Query, QueryParser};
use tantivy::schema::{Field, Schema, Value, STORED, TEXT, INDEXED};
use tantivy::schema::{Field, Schema, Value, INDEXED, STORED, TEXT};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
use tracing::{debug, info, warn};
@@ -109,8 +109,7 @@ impl SearchIndex {
"Search index corrupted, rebuilding from scratch"
);
if index_path.exists() {
std::fs::remove_dir_all(index_path)
.map_err(SearchError::Io)?;
std::fs::remove_dir_all(index_path).map_err(SearchError::Io)?;
}
Self::open(index_path)
}
@@ -205,20 +204,21 @@ impl SearchIndex {
self.schema.composer,
];
let query: Box<dyn Query> = if let Some((term, distance)) = Self::parse_fuzzy_query(query_str) {
let subqueries: Vec<(Occur, Box<dyn Query>)> = default_fields
.iter()
.map(|&field| {
let term = Term::from_field_text(field, &term);
let fuzzy = FuzzyTermQuery::new(term, distance, true);
(Occur::Should, Box::new(fuzzy) as Box<dyn Query>)
})
.collect();
Box::new(BooleanQuery::new(subqueries))
} else {
let query_parser = QueryParser::for_index(&self.index, default_fields);
query_parser.parse_query(query_str)?
};
let query: Box<dyn Query> =
if let Some((term, distance)) = Self::parse_fuzzy_query(query_str) {
let subqueries: Vec<(Occur, Box<dyn Query>)> = default_fields
.iter()
.map(|&field| {
let term = Term::from_field_text(field, &term);
let fuzzy = FuzzyTermQuery::new(term, distance, true);
(Occur::Should, Box::new(fuzzy) as Box<dyn Query>)
})
.collect();
Box::new(BooleanQuery::new(subqueries))
} else {
let query_parser = QueryParser::for_index(&self.index, default_fields);
query_parser.parse_query(query_str)?
};
let top_docs = searcher.search(&*query, &TopDocs::with_limit(limit))?;
@@ -241,9 +241,18 @@ impl SearchIndex {
results.push(SearchHit {
file_id,
virtual_path,
artist: doc.get_first(self.schema.artist).and_then(|v| v.as_str()).map(String::from),
album: doc.get_first(self.schema.album).and_then(|v| v.as_str()).map(String::from),
title: doc.get_first(self.schema.title).and_then(|v| v.as_str()).map(String::from),
artist: doc
.get_first(self.schema.artist)
.and_then(|v| v.as_str())
.map(String::from),
album: doc
.get_first(self.schema.album)
.and_then(|v| v.as_str())
.map(String::from),
title: doc
.get_first(self.schema.title)
.and_then(|v| v.as_str())
.map(String::from),
score,
});
}
@@ -322,9 +331,15 @@ mod tests {
let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap();
index.index_file(&make_file(2, "Metallica", "Master of Puppets", "Battery")).unwrap();
index.index_file(&make_file(3, "Iron Maiden", "Powerslave", "Aces High")).unwrap();
index
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
.unwrap();
index
.index_file(&make_file(2, "Metallica", "Master of Puppets", "Battery"))
.unwrap();
index
.index_file(&make_file(3, "Iron Maiden", "Powerslave", "Aces High"))
.unwrap();
index.commit().unwrap();
let results = index.search("metallica", 10).unwrap();
@@ -340,7 +355,9 @@ mod tests {
let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap();
index
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
.unwrap();
index.commit().unwrap();
let results = index.search("metalica~1", 10).unwrap();
@@ -352,7 +369,9 @@ mod tests {
let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap();
index
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
.unwrap();
index.commit().unwrap();
let results = index.search("genre:Metal", 10).unwrap();
@@ -364,7 +383,9 @@ mod tests {
let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Test", "Album", "Song")).unwrap();
index
.index_file(&make_file(1, "Test", "Album", "Song"))
.unwrap();
index.commit().unwrap();
assert_eq!(index.search("test", 10).unwrap().len(), 1);
@@ -381,7 +402,9 @@ mod tests {
{
let index = SearchIndex::open(dir.path()).unwrap();
index.index_file(&make_file(1, "Artist", "Album", "Track")).unwrap();
index
.index_file(&make_file(1, "Artist", "Album", "Track"))
.unwrap();
index.commit().unwrap();
}
@@ -15,11 +15,7 @@ pub struct Indexer<M: MetadataLookup> {
}
impl<M: MetadataLookup + 'static> Indexer<M> {
pub fn new(
index: Arc<SearchIndex>,
event_bus: Arc<EventBus>,
metadata_lookup: Arc<M>,
) -> Self {
pub fn new(index: Arc<SearchIndex>, event_bus: Arc<EventBus>, metadata_lookup: Arc<M>) -> Self {
Self {
index,
event_bus,
@@ -4,8 +4,7 @@ mod indexer;
mod query;
pub use collections::{
builtin_collections, BoolOp, CollectionError, CollectionQuery, CollectionStore,
SmartCollection,
builtin_collections, BoolOp, CollectionError, CollectionQuery, CollectionStore, SmartCollection,
};
pub use index::{SearchError, SearchHit, SearchIndex};
pub use indexer::{Indexer, IndexerHandle, MetadataLookup};
@@ -138,14 +138,21 @@ mod tests {
let shared = hashes1.intersection(&hashes2).count();
assert!(shared > 0, "CDC should produce stable boundaries, got {} chunks in original, {} after prepend", chunks1.len(), chunks2.len());
assert!(
shared > 0,
"CDC should produce stable boundaries, got {} chunks in original, {} after prepend",
chunks1.len(),
chunks2.len()
);
}
#[test]
fn test_cdc_chunk_sizes() {
let chunker = CdcChunker::default();
let data: Vec<u8> = (0..1024 * 1024).map(|i| ((i * 17 + 31) % 256) as u8).collect();
let data: Vec<u8> = (0..1024 * 1024)
.map(|i| ((i * 17 + 31) % 256) as u8)
.collect();
let chunks = chunker.chunk(&data);
@@ -187,7 +187,11 @@ impl DeltaDetector {
.collect())
}
fn compute_diff(&self, old_chunks: &[ManifestChunk], new_chunks: &[ManifestChunk]) -> ManifestDiff {
fn compute_diff(
&self,
old_chunks: &[ManifestChunk],
new_chunks: &[ManifestChunk],
) -> ManifestDiff {
let old_hashes: HashSet<_> = old_chunks.iter().map(|c| c.hash).collect();
let new_hashes: HashSet<_> = new_chunks.iter().map(|c| c.hash).collect();
@@ -34,7 +34,8 @@ impl OriginWatcher {
let origin_id_str = origin_id.to_string();
tokio::spawn(
async move {
if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await {
if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await
{
error!("Watcher error: {}", e);
}
}
@@ -126,7 +127,10 @@ impl OriginWatcher {
}
EventKind::Remove(_) => {
trace!(origin_id = %origin_id, path = ?relative, "File removed");
event_bus.publish(Event::FileRemoved { path: vpath, file_id: None });
event_bus.publish(Event::FileRemoved {
path: vpath,
file_id: None,
});
}
EventKind::Modify(_) => {
trace!(origin_id = %origin_id, path = ?relative, "File modified");
@@ -186,7 +190,8 @@ mod tests {
let event_bus = Arc::new(EventBus::default());
let mut rx = event_bus.subscribe();
let watcher = OriginWatcher::new(OriginId::from("test"), dir.path().to_path_buf(), event_bus);
let watcher =
OriginWatcher::new(OriginId::from("test"), dir.path().to_path_buf(), event_bus);
let handle = watcher.start();
tokio::time::sleep(Duration::from_millis(100)).await;
@@ -206,6 +211,8 @@ mod tests {
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.flac")));
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.MP3")));
assert!(!OriginWatcher::is_audio_file(Path::new("/music/cover.jpg")));
assert!(!OriginWatcher::is_audio_file(Path::new("/music/readme.txt")));
assert!(!OriginWatcher::is_audio_file(Path::new(
"/music/readme.txt"
)));
}
}
@@ -133,10 +133,7 @@ where
{
tokio::time::timeout(timeout, future)
.await
.expect(&format!(
"Operation did not complete within {:?}",
timeout
))
.expect(&format!("Operation did not complete within {:?}", timeout))
}
pub async fn assert_times_out<F, T>(future: F, timeout: Duration)
@@ -168,8 +165,10 @@ mod tests {
#[test]
fn test_assert_io_error() {
let result: Result<(), Error> =
Err(Error::Io(std::io::Error::new(std::io::ErrorKind::Other, "test")));
let result: Result<(), Error> = Err(Error::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"test",
)));
assert_io_error(result);
}
@@ -188,8 +187,7 @@ mod tests {
#[tokio::test]
async fn test_assert_completes_within() {
let result =
assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
let result = assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
assert_eq!(result, 42);
}
@@ -1,8 +1,6 @@
use musicfs_cache::TreeBuilder;
use musicfs_cas::{CasConfig, CasStore};
use musicfs_core::{
AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath,
};
use musicfs_core::{AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath};
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use std::time::SystemTime;
@@ -17,7 +17,11 @@ async fn require_toxiproxy() {
Ok(resp) => resp.status().is_success(),
Err(_) => false,
};
assert!(available, "Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d", TOXIPROXY_API);
assert!(
available,
"Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d",
TOXIPROXY_API
);
}
#[tokio::test]
@@ -41,10 +45,7 @@ async fn test_toxiproxy_latency_injection() {
toxicity: 1.0,
};
proxy
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
let start = std::time::Instant::now();
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
@@ -80,10 +81,7 @@ async fn test_toxiproxy_timeout_simulates_network_partition() {
toxicity: 1.0,
};
proxy
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
let result = tokio::time::timeout(
Duration::from_secs(2),
@@ -127,10 +125,7 @@ async fn test_toxiproxy_slow_close_throttles_responses() {
toxicity: 1.0,
};
proxy
.add_toxic(&toxic)
.await
.expect("Failed to add toxic");
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
let start = std::time::Instant::now();
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
@@ -144,5 +139,3 @@ async fn test_toxiproxy_slow_close_throttles_responses() {
proxy.delete().await.expect("Failed to cleanup proxy");
}
@@ -33,7 +33,10 @@ async fn setup_cas(dir: &Path) -> CasStore {
}
fn create_faulty_origin(id: &str, dir: &TempDir, mode: FailMode) -> Arc<FaultyOrigin> {
let inner = Arc::new(LocalOrigin::new(OriginId::from(id), dir.path().to_path_buf()));
let inner = Arc::new(LocalOrigin::new(
OriginId::from(id),
dir.path().to_path_buf(),
));
Arc::new(FaultyOrigin::new(inner, mode))
}
@@ -94,7 +97,9 @@ async fn test_tantivy_corruption_triggers_rebuild() {
{
let index = SearchIndex::open(&index_path).unwrap();
index.index_file(&make_file_meta(1, "/a.flac", 1000)).unwrap();
index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap();
}
@@ -148,7 +153,10 @@ async fn test_cas_put_handles_enospc() {
let large_data = vec![0u8; 1000];
let result = store.put(&large_data).await;
assert!(result.is_err(), "Issue 2.8: CasStore should pre-check space and reject oversized write");
assert!(
result.is_err(),
"Issue 2.8: CasStore should pre-check space and reject oversized write"
);
}
/// Demonstrates the PROBLEM with std::sync::RwLock: after a writer panic,
@@ -190,7 +198,10 @@ fn test_parking_lot_rwlock_survives_panic() {
let _ = handle.join();
assert!(tree.read().get(ROOT_INODE).is_some(), "parking_lot RwLock should survive writer panic");
assert!(
tree.read().get(ROOT_INODE).is_some(),
"parking_lot RwLock should survive writer panic"
);
}
#[tokio::test]
@@ -200,12 +211,17 @@ async fn test_failover_on_primary_death() {
setup_test_file(&primary_dir, "test.txt", b"primary");
setup_test_file(&backup_dir, "test.txt", b"backup");
let primary = create_faulty_origin("primary", &primary_dir, FailMode::ReturnError(ErrorKind::ConnectionRefused));
let primary = create_faulty_origin(
"primary",
&primary_dir,
FailMode::ReturnError(ErrorKind::ConnectionRefused),
);
let backup = create_faulty_origin("backup", &backup_dir, FailMode::Healthy);
let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 1);
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
let monitor =
Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
let registry = Arc::new(OriginRegistry::new(monitor.clone()));
registry.register(primary.clone(), 1);
@@ -231,21 +247,44 @@ async fn test_origin_recovery_resumes_routing() {
let dir = TempDir::new().unwrap();
setup_test_file(&dir, "test.txt", b"content");
let faulty = create_faulty_origin("recovering", &dir, FailMode::ReturnError(ErrorKind::ConnectionRefused));
let faulty = create_faulty_origin(
"recovering",
&dir,
FailMode::ReturnError(ErrorKind::ConnectionRefused),
);
let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 1);
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
let monitor =
Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
monitor.add_origin(faulty.clone());
monitor.check_now(&OriginId::from("recovering")).await;
assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().status, HealthStatus::Unhealthy);
assert_eq!(
monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.status,
HealthStatus::Unhealthy
);
faulty.set_mode(FailMode::Healthy);
monitor.check_now(&OriginId::from("recovering")).await;
assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().status, HealthStatus::Healthy);
assert_eq!(monitor.get_state(&OriginId::from("recovering")).unwrap().consecutive_failures, 0);
assert_eq!(
monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.status,
HealthStatus::Healthy
);
assert_eq!(
monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.consecutive_failures,
0
);
}
#[tokio::test]
@@ -262,8 +301,11 @@ async fn test_local_origin_health_check_has_timeout() {
monitor.check_now(&OriginId::from("slow")).await;
let elapsed = start.elapsed();
assert!(elapsed < Duration::from_secs(2),
"Issue 4.2.1: Health check should timeout in <2s, took {:?}", elapsed);
assert!(
elapsed < Duration::from_secs(2),
"Issue 4.2.1: Health check should timeout in <2s, took {:?}",
elapsed
);
let state = monitor.get_state(&OriginId::from("slow")).unwrap();
assert_eq!(state.status, HealthStatus::Unhealthy);
@@ -288,7 +330,11 @@ async fn test_health_checks_run_in_parallel() {
monitor.check_all().await;
let elapsed = start.elapsed();
assert!(elapsed < Duration::from_millis(350), "Issue 4.2.2: check_all() should run in parallel (sequential would take ~600ms), took {:?}", elapsed);
assert!(
elapsed < Duration::from_millis(350),
"Issue 4.2.2: check_all() should run in parallel (sequential would take ~600ms), took {:?}",
elapsed
);
}
#[test]
@@ -298,9 +344,13 @@ fn test_tantivy_survives_uncommitted_crash() {
{
let index = SearchIndex::open(&index_path).unwrap();
index.index_file(&make_file_meta(1, "/a.flac", 1000)).unwrap();
index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap();
index.index_file(&make_file_meta(2, "/b.flac", 1000)).unwrap();
index
.index_file(&make_file_meta(2, "/b.flac", 1000))
.unwrap();
}
let index = SearchIndex::open(&index_path).unwrap();
@@ -329,10 +379,7 @@ async fn test_fd_exhaustion_handling() {
Ok(_store) => {}
Err(e) => {
let msg = format!("{}", e);
assert!(
!msg.contains("panic"),
"Should not panic on fd exhaustion"
);
assert!(!msg.contains("panic"), "Should not panic on fd exhaustion");
}
}
@@ -357,7 +404,10 @@ async fn test_corrupt_chunk_auto_refetched() {
let store = Arc::new(setup_cas(dir.path()).await);
let origin = Arc::new(LocalOrigin::new(OriginId::from("local"), origin_dir.path().to_path_buf()));
let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin);
@@ -378,7 +428,12 @@ async fn test_corrupt_chunk_auto_refetched() {
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
let chunk_hash = manifest.chunks[0].hash;
let hex = chunk_hash.as_hex();
let chunk_path = dir.path().join("chunks").join(&hex[0..2]).join(&hex[2..4]).join(&hex);
let chunk_path = dir
.path()
.join("chunks")
.join(&hex[0..2])
.join(&hex[2..4])
.join(&hex);
let mut corrupted = std::fs::read(&chunk_path).unwrap();
corrupted[0] = corrupted[0].wrapping_add(1);
@@ -389,8 +444,15 @@ async fn test_corrupt_chunk_auto_refetched() {
let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
assert!(result.is_ok(), "Issue 6.4: Corrupted chunk should be auto-refetched from origin");
assert_eq!(&result.unwrap()[..], test_content, "Data should match original after re-fetch");
assert!(
result.is_ok(),
"Issue 6.4: Corrupted chunk should be auto-refetched from origin"
);
assert_eq!(
&result.unwrap()[..],
test_content,
"Data should match original after re-fetch"
);
}
#[tokio::test]
@@ -405,7 +467,10 @@ async fn test_missing_chunk_triggers_origin_fetch() {
let store = Arc::new(setup_cas(dir.path()).await);
let origin = Arc::new(LocalOrigin::new(OriginId::from("local"), origin_dir.path().to_path_buf()));
let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin);
@@ -426,7 +491,12 @@ async fn test_missing_chunk_triggers_origin_fetch() {
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
let chunk_hash = manifest.chunks[0].hash;
let hex = chunk_hash.as_hex();
let chunk_path = dir.path().join("chunks").join(&hex[0..2]).join(&hex[2..4]).join(&hex);
let chunk_path = dir
.path()
.join("chunks")
.join(&hex[0..2])
.join(&hex[2..4])
.join(&hex);
std::fs::remove_file(&chunk_path).unwrap();
@@ -435,8 +505,15 @@ async fn test_missing_chunk_triggers_origin_fetch() {
let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
assert!(result.is_ok(), "Issue 6.4: Missing chunk should be re-fetched from origin");
assert_eq!(&result.unwrap()[..], test_content, "Data should match original after re-fetch");
assert!(
result.is_ok(),
"Issue 6.4: Missing chunk should be re-fetched from origin"
);
assert_eq!(
&result.unwrap()[..],
test_content,
"Data should match original after re-fetch"
);
}
#[tokio::test]
@@ -449,15 +526,20 @@ async fn test_passthrough_mode_when_cache_disk_dead() {
let test_content = b"passthrough test data";
setup_test_file(&origin_dir, "test.flac", test_content);
let store = Arc::new(CasStore::open(CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 10,
shard_levels: 2,
})
.await
.unwrap());
let store = Arc::new(
CasStore::open(CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 10,
shard_levels: 2,
})
.await
.unwrap(),
);
let origin = Arc::new(LocalOrigin::new(OriginId::from("local"), origin_dir.path().to_path_buf()));
let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin);
@@ -477,7 +559,10 @@ async fn test_passthrough_mode_when_cache_disk_dead() {
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
assert!(!manifest.chunks.is_empty(), "Issue 6.6: Fetch should complete even when CAS write fails (passthrough mode)");
assert!(
!manifest.chunks.is_empty(),
"Issue 6.6: Fetch should complete even when CAS write fails (passthrough mode)"
);
}
#[tokio::test]
@@ -522,12 +607,18 @@ fn test_pid_file_prevents_concurrent_mount() {
assert!(lock1.is_ok(), "Issue C9: First lock should succeed");
let lock2 = try_lock(&lock_path);
assert!(lock2.is_err(), "Issue C9: Second lock should fail (already held)");
assert!(
lock2.is_err(),
"Issue C9: Second lock should fail (already held)"
);
drop(lock1);
let lock3 = try_lock(&lock_path);
assert!(lock3.is_ok(), "Issue C9: Third lock should succeed after first released");
assert!(
lock3.is_ok(),
"Issue C9: Third lock should succeed after first released"
);
}
#[test]
@@ -556,7 +647,10 @@ fn test_stale_mount_check_function_exists() {
fn test_systemd_service_has_execstoppost() {
let service_path = std::path::Path::new("../../dist/musicfs.service");
if !service_path.exists() {
panic!("Issue 3.7: dist/musicfs.service does not exist at {:?}", service_path);
panic!(
"Issue 3.7: dist/musicfs.service does not exist at {:?}",
service_path
);
}
let content = std::fs::read_to_string(service_path).unwrap();
@@ -574,18 +668,27 @@ fn test_sd_notify_ready_sent() {
let dir = TempDir::new().unwrap();
let socket_path = dir.path().join("notify.sock");
let socket = UnixDatagram::bind(&socket_path).unwrap();
socket.set_read_timeout(Some(Duration::from_secs(1))).unwrap();
socket
.set_read_timeout(Some(Duration::from_secs(1)))
.unwrap();
std::env::set_var("NOTIFY_SOCKET", &socket_path);
let result = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]);
assert!(result.is_ok(), "sd_notify should succeed when NOTIFY_SOCKET is set");
assert!(
result.is_ok(),
"sd_notify should succeed when NOTIFY_SOCKET is set"
);
let mut buf = [0u8; 256];
let len = socket.recv(&mut buf).unwrap();
let msg = std::str::from_utf8(&buf[..len]).unwrap();
assert!(msg.contains("READY=1"), "sd_notify should send READY=1, got: {}", msg);
assert!(
msg.contains("READY=1"),
"sd_notify should send READY=1, got: {}",
msg
);
std::env::remove_var("NOTIFY_SOCKET");
}
@@ -615,7 +718,9 @@ async fn test_shutdown_flushes_tantivy() {
{
let index = SearchIndex::open(&idx_path).unwrap();
index.index_file(&make_file_meta(1, "/a.flac", 1000)).unwrap();
index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap();
}
@@ -678,7 +783,9 @@ async fn test_sigterm_triggers_shutdown() {
let musicfs_bin = std::env::var("CARGO_BIN_EXE_musicfs").ok();
if musicfs_bin.is_none() {
eprintln!("Skipping test_sigterm_triggers_shutdown: musicfs binary not available in test context");
eprintln!(
"Skipping test_sigterm_triggers_shutdown: musicfs binary not available in test context"
);
return;
}
@@ -690,7 +797,12 @@ async fn test_sigterm_triggers_shutdown() {
std::fs::create_dir_all(&origin).unwrap();
let mut child = Command::new(&bin_path)
.args(["mount", "--origin", origin.to_str().unwrap(), mountpoint.to_str().unwrap()])
.args([
"mount",
"--origin",
origin.to_str().unwrap(),
mountpoint.to_str().unwrap(),
])
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn();
@@ -716,7 +828,11 @@ async fn test_sigterm_triggers_shutdown() {
}
}
child.wait().unwrap()
}).await;
})
.await;
assert!(exit_result.is_ok(), "Issue 2.1: Process should exit within 10s after SIGTERM");
assert!(
exit_result.is_ok(),
"Issue 2.1: Process should exit within 10s after SIGTERM"
);
}
Generated
+80 -7
View File
@@ -1,5 +1,21 @@
{
"nodes": {
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1767039857,
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
"owner": "NixOS",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
@@ -18,19 +34,75 @@
"type": "github"
}
},
"nixpkgs-old": {
"flake": false,
"git-hooks": {
"inputs": {
"flake-compat": "flake-compat",
"gitignore": "gitignore",
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1558254092,
"narHash": "sha256-5v6XuO9dOVpB3ZGNyDvLqOvCnzlyyvscTYbNiQouSZo=",
"lastModified": 1778507602,
"narHash": "sha256-kTwur1wV+01SdqskVMSo6JMEpg71ps3HpbFY2GsflKs=",
"owner": "cachix",
"repo": "git-hooks.nix",
"rev": "61ab0e80d9c7ab14c256b5b453d8b3fb0189ba0a",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "git-hooks.nix",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"git-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1709087332,
"narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1770073757,
"narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "a7e559a5504572008567383c3dc8e142fa7a8633",
"rev": "47472570b1e607482890801aeaf29bfb749884f6",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-18.09",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1778443072,
"narHash": "sha256-zi7/fsqM/kFdNuED//4WOCUtezGtKKqRNORjMvfwjnA=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "da5ad661ba4e5ef59ba743f0d112cbc30e474f32",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
@@ -38,7 +110,8 @@
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs-old": "nixpkgs-old"
"git-hooks": "git-hooks",
"nixpkgs": "nixpkgs_2"
}
},
"systems": {
+61 -104
View File
@@ -1,115 +1,72 @@
{
description = "beetfs - FUSE filesystem for beets with metadata overlay (Python 2.7)";
description = "MusicFS - FUSE filesystem for music with metadata overlay";
inputs = {
# Using nixos-18.09 - has all Python 2 packages: fuse, mutagen, jellyfish, munkres
# Mark as non-flake since 18.09 predates flakes
nixpkgs-old = {
url = "github:NixOS/nixpkgs/nixos-18.09";
flake = false;
};
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
flake-utils.url = "github:numtide/flake-utils";
git-hooks.url = "github:cachix/git-hooks.nix";
};
outputs = { self, nixpkgs-old, flake-utils }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs-old {
inherit system;
};
outputs = { self, nixpkgs, flake-utils, git-hooks }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs {
inherit system;
};
# Build beets 1.4.9 for Python 2 (last Python 2 compatible version)
# Using minimal deps - autotagger features (munkres, jellyfish) not needed for beetfs
beets-py2 = pkgs.python2Packages.buildPythonPackage rec {
pname = "beets";
version = "1.4.9";
# Use legacy setup.py install (not pip/wheel)
format = "other";
src = pkgs.fetchFromGitHub {
owner = "beetbox";
repo = "beets";
rev = "v${version}";
sha256 = "sha256-KO5jtqxw82ylbSKsJgUGr3bfxUPH8vanf/+kv//CreM=";
};
# No patching needed - nixpkgs-18.09 has all required Python 2 packages
nativeBuildInputs = with pkgs.python2Packages; [ setuptools ];
buildPhase = "true"; # No build needed
installPhase = ''
# Direct copy to avoid dependency resolution
mkdir -p $out/lib/python2.7/site-packages
cp -r beets $out/lib/python2.7/site-packages/
cp -r beetsplug $out/lib/python2.7/site-packages/
# Create version file
echo "__version__ = '${version}'" >> $out/lib/python2.7/site-packages/beets/__init__.py
# Create beet command wrapper
mkdir -p $out/bin
cat > $out/bin/beet << 'EOF'
#!${pkgs.python2.interpreter}
import sys
from beets.ui import main
main()
EOF
chmod +x $out/bin/beet
'';
propagatedBuildInputs = with pkgs.python2Packages; [
pyyaml
mutagen
unidecode
enum34
six
musicbrainzngs
jellyfish
munkres
];
# Disable tests for faster builds
doCheck = false;
meta = {
description = "Music library manager and tagger";
homepage = "https://beets.io/";
pre-commit-check = git-hooks.lib.${system}.run {
src = ./.;
hooks = {
rustfmt = {
enable = true;
packageOverrides = {
cargo = pkgs.cargo;
rustfmt = pkgs.rustfmt;
};
};
pythonEnv = pkgs.python2.withPackages (ps: with ps; [
fuse
mutagen
pyyaml
enum34
six
jellyfish
munkres
unidecode
musicbrainzngs
] ++ [ beets-py2 ]);
in {
devShells.default = pkgs.mkShell {
buildInputs = [
pythonEnv
pkgs.fuse
pkgs.ffmpeg
pkgs.flac
];
shellHook = ''
unset PYTHONPATH
export PYTHONPATH="$PWD/beetsplug:$PWD/tests"
echo "beetfs development environment (Python 2.7)"
echo " Python: $(python --version 2>&1)"
echo " Run tests: cd tests && python -m unittest discover"
echo " Mount: beet mount <mountpoint>"
'';
clippy = {
enable = true;
packageOverrides = {
cargo = pkgs.cargo;
clippy = pkgs.clippy;
};
};
}
);
};
};
in {
checks = {
inherit pre-commit-check;
};
devShells.default = pkgs.mkShell rec {
inherit (pre-commit-check) shellHook;
buildInputs = with pkgs; [
pre-commit
gitleaks
just
pkg-config
fuse3
sqlite
openssl
rustc
cargo
cargo-watch
cargo-nextest
cargo-criterion
rust-analyzer
clippy
rustfmt
clang
lld
protobuf
grpcurl
];
};
});
}
-10
View File
@@ -1,10 +0,0 @@
[build]
rustflags = ["-C", "link-arg=-fuse-ld=lld"]
[target.x86_64-unknown-linux-gnu]
linker = "clang"
[alias]
t = "test"
c = "check"
b = "build"
-1
View File
@@ -1 +0,0 @@
target/
-23
View File
@@ -1,23 +0,0 @@
pkgname=musicfs
pkgver=0.1.0
pkgrel=1
pkgdesc="Metadata-Organized Music Filesystem"
arch=('x86_64')
url="https://github.com/yourusername/musicfs"
license=('MIT')
depends=('fuse3')
makedepends=('rust' 'cargo')
source=("$pkgname-$pkgver.tar.gz")
sha256sums=('SKIP')
build() {
cd "$srcdir/$pkgname-$pkgver"
cargo build --release --locked
}
package() {
cd "$srcdir/$pkgname-$pkgver"
install -Dm755 "target/release/musicfs" "$pkgdir/usr/bin/musicfs"
install -Dm644 "dist/musicfs.service" "$pkgdir/usr/lib/systemd/system/musicfs.service"
install -Dm644 "config.example.toml" "$pkgdir/etc/musicfs/config.example.toml"
}
-30
View File
@@ -1,30 +0,0 @@
mount_point = "/mnt/music"
cache_dir = "/var/cache/musicfs"
[logging]
log_dir = "/var/log/musicfs"
json_output = true
journald = true
level = "musicfs=info,warn"
trace_sample_rate = 1.0
[cache]
metadata_cache_mb = 100
content_cache_gb = 10
[health]
check_interval_secs = 30
timeout_ms = 5000
unhealthy_threshold = 3
[[origins]]
id = "local"
origin_type = "local"
priority = 1
path = "/srv/music"
[[origins]]
id = "nas"
origin_type = "nfs"
priority = 2
mount_point = "/mnt/nas/music"
-9
View File
@@ -1,9 +0,0 @@
/var/log/musicfs/*.log {
daily
rotate 30
compress
delaycompress
missingok
notifempty
create 0640 musicfs musicfs
}
-29
View File
@@ -1,29 +0,0 @@
[Unit]
Description=MusicFS - Metadata-Organized Music Filesystem
After=network.target
[Service]
Type=notify
ExecStart=/usr/bin/musicfs mount --config /etc/musicfs/config.toml /mnt/music
ExecStop=/usr/bin/musicfs shutdown
ExecStopPost=/usr/bin/fusermount -uz /mnt/music || true
Restart=on-failure
RestartSec=5
User=musicfs
Group=musicfs
Environment="RUST_LOG=musicfs=info,warn"
StandardOutput=journal
StandardError=journal
SyslogIdentifier=musicfs
RateLimitIntervalSec=30s
RateLimitBurst=1000
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/var/cache/musicfs /var/log/musicfs /mnt/music
PrivateTmp=true
[Install]
WantedBy=multi-user.target
-39
View File
@@ -1,39 +0,0 @@
Name: musicfs
Version: 0.1.0
Release: 1%{?dist}
Summary: Metadata-Organized Music Filesystem
License: MIT
URL: https://github.com/yourusername/musicfs
Source0: %{name}-%{version}.tar.gz
BuildRequires: rust >= 1.70
BuildRequires: cargo
BuildRequires: fuse3-devel
Requires: fuse3
%description
MusicFS is a virtual FUSE filesystem that organizes music files by metadata.
%prep
%autosetup
%build
cargo build --release --locked
%install
install -Dm755 target/release/musicfs %{buildroot}%{_bindir}/musicfs
install -Dm644 dist/musicfs.service %{buildroot}%{_unitdir}/musicfs.service
install -Dm644 config.example.toml %{buildroot}%{_sysconfdir}/musicfs/config.example.toml
%files
%license LICENSE
%doc README.md
%{_bindir}/musicfs
%{_unitdir}/musicfs.service
%config(noreplace) %{_sysconfdir}/musicfs/config.example.toml
%changelog
* Mon Jan 01 2024 MusicFS Team <team@example.com> - 0.1.0-1
- Initial package

Some files were not shown because too many files have changed in this diff Show More