fujin/MusicFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Start anew

Browse Source
This commit is contained in:
Alexander
2026-06-18 23:15:11 +02:00
parent 05b83364df
commit 83ff042b91
109 changed files with 0 additions and 30964 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/musicfs-cache/Cargo.toml
-25
View File
@@ -1,25 +0,0 @@
[package]
name = "musicfs-cache"
version.workspace = true
edition.workspace = true
[dependencies]
musicfs-core = { path = "../musicfs-core" }
musicfs-cas = { path = "../musicfs-cas" }
musicfs-metadata = { path = "../musicfs-metadata" }
bytes.workspace = true
rusqlite = { workspace = true, features = ["bundled"] }
sled.workspace = true
tokio.workspace = true
tracing.workspace = true
thiserror.workspace = true
serde.workspace = true
rmp-serde.workspace = true
serde_json.workspace = true
image.workspace = true
lofty = "0.24"
parking_lot.workspace = true
chrono.workspace = true
[dev-dependencies]
tempfile.workspace = true
crates/musicfs-cache/src/artwork.rs
-213
View File
@@ -1,213 +0,0 @@
use image::ImageFormat;
use musicfs_cas::CasStore;
use musicfs_core::ChunkHash;
use musicfs_metadata::artwork::{ArtSize, ArtType, Artwork};
use std::io::Cursor;
use std::path::Path;
use std::sync::Arc;
use tracing::{debug, info, trace, warn};
const MAX_ARTWORK_INPUT_SIZE: usize = 10 * 1024 * 1024;
pub struct ArtworkCache {
store: Arc<CasStore>,
db_path: std::path::PathBuf,
}
#[derive(Debug)]
pub struct CachedArtwork {
pub file_id: i64,
pub art_type: String,
pub chunk_hash: ChunkHash,
pub width: u32,
pub height: u32,
}
impl ArtworkCache {
pub fn new(store: Arc<CasStore>, db_path: &Path) -> Result<Self, ArtworkError> {
let db = rusqlite::Connection::open(db_path)?;
db.execute(
"CREATE TABLE IF NOT EXISTS artwork (
id INTEGER PRIMARY KEY,
file_id INTEGER NOT NULL,
art_type TEXT NOT NULL,
chunk_hash TEXT NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
UNIQUE(file_id, art_type)
)",
[],
)?;
info!(path = ?db_path, "Artwork cache opened");
Ok(Self {
store,
db_path: db_path.to_path_buf(),
})
}
pub async fn store(&self, file_id: i64, artwork: &Artwork) -> Result<ChunkHash, ArtworkError> {
trace!(
file_id = file_id,
size_bytes = artwork.data.len(),
"Storing artwork"
);
if artwork.data.len() > MAX_ARTWORK_INPUT_SIZE {
warn!(
file_id = file_id,
size = artwork.data.len(),
max = MAX_ARTWORK_INPUT_SIZE,
"Artwork too large"
);
return Err(ArtworkError::ImageTooLarge(artwork.data.len()));
}
let hash = self.store.put(&artwork.data).await?;
let art_type_str = match artwork.art_type {
ArtType::Front => "front",
ArtType::Back => "back",
ArtType::Other => "other",
};
let db_path = self.db_path.clone();
let art_type_clone = art_type_str.to_string();
let hash_hex = hash.to_hex();
let width = artwork.width;
let height = artwork.height;
tokio::task::spawn_blocking(move || {
let db = rusqlite::Connection::open(&db_path)?;
db.execute(
"INSERT OR REPLACE INTO artwork
(file_id, art_type, chunk_hash, width, height)
VALUES (?1, ?2, ?3, ?4, ?5)",
rusqlite::params![file_id, art_type_clone, hash_hex, width, height],
)?;
Ok::<_, ArtworkError>(())
})
.await
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))??;
debug!("Cached artwork for file {}", file_id);
Ok(hash)
}
pub async fn get(
&self,
file_id: i64,
art_type: &str,
size: ArtSize,
) -> Result<Option<Vec<u8>>, ArtworkError> {
trace!(file_id = file_id, art_type = %art_type, "Getting artwork");
let db_path = self.db_path.clone();
let art_type_clone = art_type.to_string();
let hash_hex: Option<String> = tokio::task::spawn_blocking(move || {
let db = rusqlite::Connection::open(&db_path)?;
db.query_row(
"SELECT chunk_hash FROM artwork WHERE file_id = ?1 AND art_type = ?2",
rusqlite::params![file_id, art_type_clone],
|row| row.get(0),
)
.ok()
.ok_or(ArtworkError::NotFound)
})
.await
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))?
.ok();
match hash_hex {
Some(hex) => {
trace!(file_id = file_id, "Artwork cache hit");
let hash = ChunkHash::from_hex(&hex).ok_or(ArtworkError::InvalidHash)?;
let data = self.store.get(&hash).await?;
match size {
ArtSize::Full => Ok(Some(data.to_vec())),
ArtSize::Thumbnail | ArtSize::Medium => {
let resized = self.resize_on_demand(&data, size)?;
Ok(Some(resized))
}
}
}
None => {
trace!(file_id = file_id, "Artwork cache miss");
Ok(None)
}
}
}
pub async fn has(&self, file_id: i64, art_type: &str) -> Result<bool, ArtworkError> {
let db_path = self.db_path.clone();
let art_type_clone = art_type.to_string();
tokio::task::spawn_blocking(move || {
let db = rusqlite::Connection::open(&db_path)?;
let count: i64 = db.query_row(
"SELECT COUNT(*) FROM artwork WHERE file_id = ?1 AND art_type = ?2",
rusqlite::params![file_id, art_type_clone],
|row| row.get(0),
)?;
Ok(count > 0)
})
.await
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))?
}
fn resize_on_demand(&self, data: &[u8], size: ArtSize) -> Result<Vec<u8>, ArtworkError> {
let max_dim = size.max_dimension().unwrap_or(300);
let img = image::load_from_memory(data).map_err(|_| ArtworkError::InvalidImage)?;
if img.width() <= max_dim && img.height() <= max_dim {
return Ok(data.to_vec());
}
let resized = img.thumbnail(max_dim, max_dim);
let mut output = Vec::new();
let mut cursor = Cursor::new(&mut output);
resized
.write_to(&mut cursor, ImageFormat::Jpeg)
.map_err(|_| ArtworkError::ResizeFailed)?;
Ok(output)
}
}
#[derive(Debug, thiserror::Error)]
pub enum ArtworkError {
#[error("database error: {0}")]
Database(#[from] rusqlite::Error),
#[error("CAS error: {0}")]
Cas(#[from] musicfs_cas::CasError),
#[error("invalid hash")]
InvalidHash,
#[error("artwork not found")]
NotFound,
#[error("image too large: {0} bytes (max 10MB)")]
ImageTooLarge(usize),
#[error("invalid image data")]
InvalidImage,
#[error("resize failed")]
ResizeFailed,
#[error("spawn_blocking error: {0}")]
SpawnBlocking(String),
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_max_artwork_size() {
assert_eq!(MAX_ARTWORK_INPUT_SIZE, 10 * 1024 * 1024);
}
}
crates/musicfs-cache/src/db.rs
-1792
View File
File diff suppressed because it is too large Load Diff
crates/musicfs-cache/src/eviction.rs
-155
View File
@@ -1,155 +0,0 @@
use musicfs_cas::CasStore;
use musicfs_core::ChunkHash;
use parking_lot::RwLock;
use std::collections::BTreeMap;
use std::time::Instant;
use tracing::info;
pub trait EvictionPolicy: Send + Sync {
fn record_access(&self, hash: ChunkHash);
fn select_victims(&self, count: usize) -> Vec<ChunkHash>;
fn remove(&self, hash: &ChunkHash);
}
pub struct LruEviction {
access_times: RwLock<BTreeMap<Instant, ChunkHash>>,
hash_to_time: RwLock<std::collections::HashMap<ChunkHash, Instant>>,
}
impl LruEviction {
pub fn new() -> Self {
Self {
access_times: RwLock::new(BTreeMap::new()),
hash_to_time: RwLock::new(std::collections::HashMap::new()),
}
}
pub async fn evict_to_target(
&self,
store: &CasStore,
target_size: u64,
) -> Result<u64, EvictionError> {
let mut bytes_freed = 0u64;
while store.current_size() > target_size {
let victims = self.select_victims(10);
if victims.is_empty() {
break;
}
for hash in victims {
if let Ok(data) = store.get(&hash).await {
bytes_freed += data.len() as u64;
store.delete(&hash).await?;
self.remove(&hash);
}
}
}
if bytes_freed > 0 {
info!("Evicted {} bytes from cache", bytes_freed);
}
Ok(bytes_freed)
}
}
impl Default for LruEviction {
fn default() -> Self {
Self::new()
}
}
impl EvictionPolicy for LruEviction {
fn record_access(&self, hash: ChunkHash) {
let now = Instant::now();
let mut times = self.access_times.write();
let mut h2t = self.hash_to_time.write();
if let Some(old_time) = h2t.remove(&hash) {
times.remove(&old_time);
}
times.insert(now, hash);
h2t.insert(hash, now);
}
fn select_victims(&self, count: usize) -> Vec<ChunkHash> {
let times = self.access_times.read();
times.values().take(count).copied().collect()
}
fn remove(&self, hash: &ChunkHash) {
let mut times = self.access_times.write();
let mut h2t = self.hash_to_time.write();
if let Some(time) = h2t.remove(hash) {
times.remove(&time);
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum EvictionError {
#[error("CAS error: {0}")]
Cas(#[from] musicfs_cas::CasError),
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_lru_access_order() {
let lru = LruEviction::new();
let h1 = ChunkHash::from_bytes(b"chunk1");
let h2 = ChunkHash::from_bytes(b"chunk2");
let h3 = ChunkHash::from_bytes(b"chunk3");
lru.record_access(h1);
std::thread::sleep(std::time::Duration::from_millis(1));
lru.record_access(h2);
std::thread::sleep(std::time::Duration::from_millis(1));
lru.record_access(h3);
let victims = lru.select_victims(2);
assert_eq!(victims.len(), 2);
assert_eq!(victims[0], h1);
assert_eq!(victims[1], h2);
}
#[test]
fn test_lru_reaccess_updates_order() {
let lru = LruEviction::new();
let h1 = ChunkHash::from_bytes(b"chunk1");
let h2 = ChunkHash::from_bytes(b"chunk2");
lru.record_access(h1);
std::thread::sleep(std::time::Duration::from_millis(1));
lru.record_access(h2);
std::thread::sleep(std::time::Duration::from_millis(1));
lru.record_access(h1);
let victims = lru.select_victims(1);
assert_eq!(victims[0], h2);
}
#[test]
fn test_lru_remove() {
let lru = LruEviction::new();
let h1 = ChunkHash::from_bytes(b"chunk1");
let h2 = ChunkHash::from_bytes(b"chunk2");
lru.record_access(h1);
lru.record_access(h2);
lru.remove(&h1);
let victims = lru.select_victims(10);
assert_eq!(victims.len(), 1);
assert_eq!(victims[0], h2);
}
}
crates/musicfs-cache/src/format_handler.rs
-103
View File
@@ -1,103 +0,0 @@
use crate::FormatLayout;
use musicfs_core::AudioMeta;
use std::collections::HashMap;
use std::sync::Arc;
/// Error types for format handling operations
#[derive(Debug, thiserror::Error)]
pub enum FormatError {
#[error("Unsupported format")]
UnsupportedFormat,
#[error("Invalid data: {0}")]
InvalidData(String),
#[error("Synthesis failed: {0}")]
SynthesisFailed(String),
}
/// Trait for format-specific metadata handling.
///
/// Implementations handle:
/// 1. Analyzing original files to find audio boundaries
/// 2. Synthesizing new headers from database metadata
pub trait FormatHandler: Send + Sync + 'static {
/// Unique identifier for this handler
fn id(&self) -> &'static str;
/// Human-readable name
fn name(&self) -> &'static str;
/// File extensions this handler supports
fn extensions(&self) -> &[&'static str];
/// MIME types this handler supports
fn mime_types(&self) -> &[&'static str];
/// Analyze file bytes to determine audio layout
fn analyze(
&self,
data: &[u8],
file_size: u64,
) -> std::result::Result<FormatLayout, FormatError>;
/// Synthesize header bytes from metadata. Called on every read().
fn synthesize(
&self,
metadata: &AudioMeta,
layout: &FormatLayout,
) -> std::result::Result<Vec<u8>, FormatError>;
/// Extract metadata from header bytes (for initial ingest)
fn extract(&self, data: &[u8]) -> std::result::Result<AudioMeta, FormatError>;
/// Estimate header size without full synthesis (for getattr)
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
10 * 1024 // 10KB default
}
}
/// Registry for format handlers
pub struct FormatHandlerRegistry {
handlers: HashMap<String, Arc<dyn FormatHandler>>,
extension_map: HashMap<String, String>,
}
impl FormatHandlerRegistry {
/// Create empty registry
pub fn new() -> Self {
Self {
handlers: HashMap::new(),
extension_map: HashMap::new(),
}
}
/// Register a format handler
pub fn register(&mut self, handler: Arc<dyn FormatHandler>) {
let id = handler.id().to_string();
// Map extensions to handler ID
for ext in handler.extensions() {
self.extension_map.insert(ext.to_string(), id.clone());
}
self.handlers.insert(id, handler);
}
/// Get handler by file extension
pub fn get_by_extension(&self, ext: &str) -> Option<Arc<dyn FormatHandler>> {
let id = self.extension_map.get(ext)?;
self.handlers.get(id).cloned()
}
/// Get handler by format ID
pub fn get_by_format(&self, format: &str) -> Option<Arc<dyn FormatHandler>> {
self.handlers.get(format).cloned()
}
}
impl Default for FormatHandlerRegistry {
fn default() -> Self {
Self::new()
}
}
crates/musicfs-cache/src/format_layout.rs
-22
View File
@@ -1,22 +0,0 @@
use musicfs_core::AudioFormat;
use serde::{Deserialize, Serialize};
/// Describes the byte layout of an audio file for overlay splicing.
///
/// This struct tracks where the audio data begins and ends in the origin file,
/// allowing the OverlayReader to splice synthetic headers with original audio.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FormatLayout {
/// Byte offset where audio data begins in the origin file
pub audio_start: u64,
/// Byte offset where audio data ends in the origin file
pub audio_end: u64,
/// Audio format (from musicfs-core)
pub format: AudioFormat,
/// Format-specific data (e.g., FLAC STREAMINFO block, MP4 stco offsets)
/// Stored as raw bytes, interpreted by format handlers
pub format_data: Option<Vec<u8>>,
}
crates/musicfs-cache/src/handlers/flac.rs
-886
View File
@@ -1,886 +0,0 @@
//! FLAC format handler for metadata synthesis.
//!
//! FLAC files use Vorbis comments for metadata. The file structure is:
//! - "fLaC" marker (4 bytes)
//! - STREAMINFO block (mandatory, 38 bytes total: 4 header + 34 data)
//! - Optional metadata blocks (VORBIS_COMMENT, PICTURE, PADDING, etc.)
//! - Audio frames
//!
//! CRITICAL: STREAMINFO must be preserved from the original file as it contains
//! MD5 checksum, sample count, and audio properties that must match the audio data.
use crate::{FormatError, FormatHandler, FormatLayout};
use lofty::config::ParseOptions;
use lofty::file::AudioFile;
use lofty::flac::FlacFile;
use lofty::ogg::VorbisComments;
use lofty::tag::Accessor;
use musicfs_core::{AudioFormat, AudioMeta};
use std::borrow::Cow;
use std::io::Cursor;
/// FLAC stream marker: "fLaC" in ASCII
const FLAC_MARKER: &[u8; 4] = b"fLaC";
/// FLAC metadata block types
const BLOCK_TYPE_STREAMINFO: u8 = 0;
const BLOCK_TYPE_VORBIS_COMMENT: u8 = 4;
/// STREAMINFO block data size (always 34 bytes)
const STREAMINFO_DATA_SIZE: usize = 34;
/// Metadata block header size (1 byte type/flags + 3 bytes length)
const BLOCK_HEADER_SIZE: usize = 4;
/// Full STREAMINFO block size (header + data)
const STREAMINFO_BLOCK_SIZE: usize = BLOCK_HEADER_SIZE + STREAMINFO_DATA_SIZE;
pub struct FlacHandler;
impl FlacHandler {
pub fn new() -> Self {
Self
}
/// Parse FLAC metadata block header.
/// Returns (is_last, block_type, block_size).
fn parse_block_header(data: &[u8]) -> Option<(bool, u8, usize)> {
if data.len() < BLOCK_HEADER_SIZE {
return None;
}
let is_last = (data[0] & 0x80) != 0;
let block_type = data[0] & 0x7F;
let block_size =
((data[1] as usize) << 16) | ((data[2] as usize) << 8) | (data[3] as usize);
Some((is_last, block_type, block_size))
}
/// Write a metadata block header.
fn write_block_header(is_last: bool, block_type: u8, size: usize) -> [u8; 4] {
let type_byte = if is_last {
block_type | 0x80
} else {
block_type
};
[
type_byte,
((size >> 16) & 0xFF) as u8,
((size >> 8) & 0xFF) as u8,
(size & 0xFF) as u8,
]
}
/// Build Vorbis comments from AudioMeta.
fn build_vorbis_comments(metadata: &AudioMeta) -> VorbisComments {
let mut tag = VorbisComments::default();
// Basic fields (using Accessor trait)
if let Some(ref title) = metadata.title {
tag.set_title(title.clone());
}
if let Some(ref artist) = metadata.artist {
tag.set_artist(artist.clone());
}
if let Some(ref album) = metadata.album {
tag.set_album(album.clone());
}
if let Some(ref genre) = metadata.genre {
tag.set_genre(genre.clone());
}
// Album artist
if let Some(ref album_artist) = metadata.album_artist {
tag.insert("ALBUMARTIST".to_string(), album_artist.clone());
}
// Year/Date
if let Some(ref date) = metadata.date {
tag.insert("DATE".to_string(), date.clone());
} else if let Some(year) = metadata.year {
tag.insert("DATE".to_string(), year.to_string());
}
// Track/Disc numbers
if let Some(track) = metadata.track {
tag.insert("TRACKNUMBER".to_string(), track.to_string());
}
if let Some(track_total) = metadata.track_total {
tag.insert("TRACKTOTAL".to_string(), track_total.to_string());
}
if let Some(disc) = metadata.disc {
tag.insert("DISCNUMBER".to_string(), disc.to_string());
}
if let Some(disc_total) = metadata.disc_total {
tag.insert("DISCTOTAL".to_string(), disc_total.to_string());
}
// Extended metadata
if let Some(ref composer) = metadata.composer {
tag.insert("COMPOSER".to_string(), composer.clone());
}
if let Some(ref comment) = metadata.comment {
tag.insert("COMMENT".to_string(), comment.clone());
}
if let Some(ref lyrics) = metadata.lyrics {
tag.insert("LYRICS".to_string(), lyrics.clone());
}
if let Some(ref copyright) = metadata.copyright {
tag.insert("COPYRIGHT".to_string(), copyright.clone());
}
if let Some(compilation) = metadata.compilation {
tag.insert(
"COMPILATION".to_string(),
if compilation { "1" } else { "0" }.to_string(),
);
}
// Sort fields
if let Some(ref title_sort) = metadata.title_sort {
tag.insert("TITLESORT".to_string(), title_sort.clone());
}
if let Some(ref artist_sort) = metadata.artist_sort {
tag.insert("ARTISTSORT".to_string(), artist_sort.clone());
}
if let Some(ref album_sort) = metadata.album_sort {
tag.insert("ALBUMSORT".to_string(), album_sort.clone());
}
if let Some(ref album_artist_sort) = metadata.album_artist_sort {
tag.insert("ALBUMARTISTSORT".to_string(), album_artist_sort.clone());
}
// MusicBrainz IDs
if let Some(ref mb_recording_id) = metadata.mb_recording_id {
tag.insert("MUSICBRAINZ_TRACKID".to_string(), mb_recording_id.clone());
}
if let Some(ref mb_album_id) = metadata.mb_album_id {
tag.insert("MUSICBRAINZ_ALBUMID".to_string(), mb_album_id.clone());
}
if let Some(ref mb_artist_id) = metadata.mb_artist_id {
tag.insert("MUSICBRAINZ_ARTISTID".to_string(), mb_artist_id.clone());
}
if let Some(ref mb_album_artist_id) = metadata.mb_album_artist_id {
tag.insert(
"MUSICBRAINZ_ALBUMARTISTID".to_string(),
mb_album_artist_id.clone(),
);
}
if let Some(ref mb_release_group_id) = metadata.mb_release_group_id {
tag.insert(
"MUSICBRAINZ_RELEASEGROUPID".to_string(),
mb_release_group_id.clone(),
);
}
// ReplayGain
if let Some(gain) = metadata.replaygain_track_gain {
tag.insert(
"REPLAYGAIN_TRACK_GAIN".to_string(),
format!("{:.2} dB", gain),
);
}
if let Some(peak) = metadata.replaygain_track_peak {
tag.insert("REPLAYGAIN_TRACK_PEAK".to_string(), format!("{:.6}", peak));
}
if let Some(gain) = metadata.replaygain_album_gain {
tag.insert(
"REPLAYGAIN_ALBUM_GAIN".to_string(),
format!("{:.2} dB", gain),
);
}
if let Some(peak) = metadata.replaygain_album_peak {
tag.insert("REPLAYGAIN_ALBUM_PEAK".to_string(), format!("{:.6}", peak));
}
// Encoder
if let Some(ref encoder) = metadata.encoder {
tag.insert("ENCODER".to_string(), encoder.clone());
}
tag
}
/// Serialize Vorbis comments to bytes (without block header).
/// Format: vendor_length (4 LE) + vendor + comment_count (4 LE) + comments
fn serialize_vorbis_comments(tag: &VorbisComments) -> Vec<u8> {
let vendor = tag.vendor();
let vendor = if vendor.is_empty() { "musicfs" } else { vendor };
let mut data = Vec::new();
// Vendor string (little-endian length + UTF-8 string)
let vendor_bytes = vendor.as_bytes();
data.extend_from_slice(&(vendor_bytes.len() as u32).to_le_bytes());
data.extend_from_slice(vendor_bytes);
// Collect all comments
let comments: Vec<_> = tag.items().collect();
data.extend_from_slice(&(comments.len() as u32).to_le_bytes());
for (key, value) in comments {
let comment = format!("{}={}", key, value);
let comment_bytes = comment.as_bytes();
data.extend_from_slice(&(comment_bytes.len() as u32).to_le_bytes());
data.extend_from_slice(comment_bytes);
}
data
}
/// Extract metadata from Vorbis comments tag.
fn extract_from_vorbis_comments(tag: &VorbisComments) -> AudioMeta {
let mut meta = AudioMeta::default();
meta.format = AudioFormat::Flac;
// Basic fields (using Accessor trait)
meta.title = tag.title().map(|c: Cow<'_, str>| c.into_owned());
meta.artist = tag.artist().map(|c: Cow<'_, str>| c.into_owned());
meta.album = tag.album().map(|c: Cow<'_, str>| c.into_owned());
meta.genre = tag.genre().map(|c: Cow<'_, str>| c.into_owned());
// Album artist
meta.album_artist = tag.get("ALBUMARTIST").map(String::from);
// Date/Year
meta.date = tag.get("DATE").map(String::from);
if let Some(ref date) = meta.date {
if let Some(year_str) = date.split('-').next() {
meta.year = year_str.parse().ok();
}
}
// Track/Disc numbers
meta.track = tag.get("TRACKNUMBER").and_then(|s| s.parse().ok());
meta.track_total = tag.get("TRACKTOTAL").and_then(|s| s.parse().ok());
meta.disc = tag.get("DISCNUMBER").and_then(|s| s.parse().ok());
meta.disc_total = tag.get("DISCTOTAL").and_then(|s| s.parse().ok());
// Extended metadata
meta.composer = tag.get("COMPOSER").map(String::from);
meta.comment = tag.get("COMMENT").map(String::from);
meta.lyrics = tag.get("LYRICS").map(String::from);
meta.copyright = tag.get("COPYRIGHT").map(String::from);
meta.compilation = tag
.get("COMPILATION")
.map(|s| s == "1" || s.eq_ignore_ascii_case("true"));
// Sort fields
meta.title_sort = tag.get("TITLESORT").map(String::from);
meta.artist_sort = tag.get("ARTISTSORT").map(String::from);
meta.album_sort = tag.get("ALBUMSORT").map(String::from);
meta.album_artist_sort = tag.get("ALBUMARTISTSORT").map(String::from);
// MusicBrainz IDs
meta.mb_recording_id = tag.get("MUSICBRAINZ_TRACKID").map(String::from);
meta.mb_album_id = tag.get("MUSICBRAINZ_ALBUMID").map(String::from);
meta.mb_artist_id = tag.get("MUSICBRAINZ_ARTISTID").map(String::from);
meta.mb_album_artist_id = tag.get("MUSICBRAINZ_ALBUMARTISTID").map(String::from);
meta.mb_release_group_id = tag.get("MUSICBRAINZ_RELEASEGROUPID").map(String::from);
// ReplayGain
meta.replaygain_track_gain = tag
.get("REPLAYGAIN_TRACK_GAIN")
.and_then(|s| Self::parse_replaygain_value(s));
meta.replaygain_track_peak = tag
.get("REPLAYGAIN_TRACK_PEAK")
.and_then(|s| s.parse().ok());
meta.replaygain_album_gain = tag
.get("REPLAYGAIN_ALBUM_GAIN")
.and_then(|s| Self::parse_replaygain_value(s));
meta.replaygain_album_peak = tag
.get("REPLAYGAIN_ALBUM_PEAK")
.and_then(|s| s.parse().ok());
// Encoder
meta.encoder = tag.get("ENCODER").map(String::from);
meta
}
/// Parse ReplayGain value, stripping optional "dB" suffix.
fn parse_replaygain_value(value: &str) -> Option<f32> {
value
.trim()
.trim_end_matches(" dB")
.trim_end_matches("dB")
.parse()
.ok()
}
}
impl Default for FlacHandler {
fn default() -> Self {
Self::new()
}
}
impl FormatHandler for FlacHandler {
fn id(&self) -> &'static str {
"flac"
}
fn name(&self) -> &'static str {
"FLAC"
}
fn extensions(&self) -> &[&'static str] {
&["flac"]
}
fn mime_types(&self) -> &[&'static str] {
&["audio/flac", "audio/x-flac"]
}
fn analyze(&self, data: &[u8], file_size: u64) -> Result<FormatLayout, FormatError> {
// Verify FLAC marker
if data.len() < FLAC_MARKER.len() || &data[0..4] != FLAC_MARKER {
return Err(FormatError::InvalidData("Not a FLAC file".to_string()));
}
let mut offset = FLAC_MARKER.len();
let mut streaminfo_data: Option<Vec<u8>> = None;
// Parse metadata blocks to find audio_start and extract STREAMINFO
loop {
if offset + BLOCK_HEADER_SIZE > data.len() {
return Err(FormatError::InvalidData(
"Truncated FLAC metadata".to_string(),
));
}
let (is_last, block_type, block_size) = Self::parse_block_header(&data[offset..])
.ok_or_else(|| FormatError::InvalidData("Invalid block header".to_string()))?;
// Extract STREAMINFO block data (without header)
if block_type == BLOCK_TYPE_STREAMINFO {
if block_size != STREAMINFO_DATA_SIZE {
return Err(FormatError::InvalidData(format!(
"Invalid STREAMINFO size: {} (expected {})",
block_size, STREAMINFO_DATA_SIZE
)));
}
let data_start = offset + BLOCK_HEADER_SIZE;
let data_end = data_start + block_size;
if data_end > data.len() {
return Err(FormatError::InvalidData(
"Truncated STREAMINFO block".to_string(),
));
}
streaminfo_data = Some(data[data_start..data_end].to_vec());
}
offset += BLOCK_HEADER_SIZE + block_size;
if is_last {
break;
}
}
let streaminfo = streaminfo_data
.ok_or_else(|| FormatError::InvalidData("Missing STREAMINFO block".to_string()))?;
Ok(FormatLayout {
audio_start: offset as u64,
audio_end: file_size,
format: AudioFormat::Flac,
format_data: Some(streaminfo),
})
}
fn synthesize(
&self,
metadata: &AudioMeta,
layout: &FormatLayout,
) -> Result<Vec<u8>, FormatError> {
// STREAMINFO must be preserved from original
let streaminfo_data = layout.format_data.as_ref().ok_or_else(|| {
FormatError::SynthesisFailed("Missing STREAMINFO data in layout".to_string())
})?;
if streaminfo_data.len() != STREAMINFO_DATA_SIZE {
return Err(FormatError::SynthesisFailed(format!(
"Invalid STREAMINFO size: {} (expected {})",
streaminfo_data.len(),
STREAMINFO_DATA_SIZE
)));
}
// Build Vorbis comments
let vorbis_tag = Self::build_vorbis_comments(metadata);
let vorbis_data = Self::serialize_vorbis_comments(&vorbis_tag);
// Calculate total header size
let total_size =
FLAC_MARKER.len() + STREAMINFO_BLOCK_SIZE + BLOCK_HEADER_SIZE + vorbis_data.len();
let mut buffer = Vec::with_capacity(total_size);
// Write FLAC marker
buffer.extend_from_slice(FLAC_MARKER);
// Write STREAMINFO block (not last)
let streaminfo_header =
Self::write_block_header(false, BLOCK_TYPE_STREAMINFO, STREAMINFO_DATA_SIZE);
buffer.extend_from_slice(&streaminfo_header);
buffer.extend_from_slice(streaminfo_data);
// Write VORBIS_COMMENT block (last)
let vorbis_header =
Self::write_block_header(true, BLOCK_TYPE_VORBIS_COMMENT, vorbis_data.len());
buffer.extend_from_slice(&vorbis_header);
buffer.extend_from_slice(&vorbis_data);
Ok(buffer)
}
fn extract(&self, data: &[u8]) -> Result<AudioMeta, FormatError> {
let mut cursor = Cursor::new(data);
let flac_file = FlacFile::read_from(&mut cursor, ParseOptions::new())
.map_err(|e| FormatError::InvalidData(e.to_string()))?;
let tag = flac_file
.vorbis_comments()
.ok_or_else(|| FormatError::InvalidData("No Vorbis comments found".to_string()))?;
Ok(Self::extract_from_vorbis_comments(tag))
}
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
// fLaC (4) + STREAMINFO (38) + VORBIS_COMMENT header (4) + typical comments (~4KB)
8192
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_test_meta() -> AudioMeta {
AudioMeta {
title: Some("Test Title".to_string()),
artist: Some("Test Artist".to_string()),
album: Some("Test Album".to_string()),
album_artist: Some("Test Album Artist".to_string()),
genre: Some("Rock".to_string()),
year: Some(2024),
track: Some(5),
track_total: Some(12),
disc: Some(1),
disc_total: Some(2),
format: AudioFormat::Flac,
date: Some("2024-03-15".to_string()),
composer: Some("Test Composer".to_string()),
comment: Some("Test Comment".to_string()),
lyrics: Some("Test Lyrics\nLine 2".to_string()),
copyright: Some("2024 Test Copyright".to_string()),
compilation: Some(false),
title_sort: Some("Title, Test".to_string()),
artist_sort: Some("Artist, Test".to_string()),
album_sort: Some("Album, Test".to_string()),
album_artist_sort: Some("Album Artist, Test".to_string()),
mb_recording_id: Some("rec-12345".to_string()),
mb_album_id: Some("alb-12345".to_string()),
mb_artist_id: Some("art-12345".to_string()),
mb_album_artist_id: Some("albart-12345".to_string()),
mb_release_group_id: Some("rg-12345".to_string()),
replaygain_track_gain: Some(-6.5),
replaygain_track_peak: Some(0.987654),
replaygain_album_gain: Some(-5.2),
replaygain_album_peak: Some(0.999999),
encoder: Some("FLAC 1.4.0".to_string()),
..Default::default()
}
}
/// Create a minimal valid FLAC file header for testing.
fn make_minimal_flac_header() -> Vec<u8> {
let mut data = Vec::new();
// FLAC marker
data.extend_from_slice(b"fLaC");
// STREAMINFO block (last=true for minimal file)
// Header: type=0 (STREAMINFO), last=1, size=34
data.push(0x80); // 0x80 = last flag set, type 0
data.push(0x00);
data.push(0x00);
data.push(0x22); // 34 bytes
// STREAMINFO data (34 bytes) - minimal valid values
// min_block_size (16 bits) = 4096
data.push(0x10);
data.push(0x00);
// max_block_size (16 bits) = 4096
data.push(0x10);
data.push(0x00);
// min_frame_size (24 bits) = 0 (unknown)
data.push(0x00);
data.push(0x00);
data.push(0x00);
// max_frame_size (24 bits) = 0 (unknown)
data.push(0x00);
data.push(0x00);
data.push(0x00);
// sample_rate (20 bits) = 44100, channels-1 (3 bits) = 1, bits-1 (5 bits) = 15
// 44100 = 0xAC44, channels=2 (1), bits=16 (15)
// Packed: SSSS SSSS SSSS SSSS SSSS CCCC CBBB BB
// 0xAC44 << 12 | (1 << 9) | (15 << 4) = ...
// Let's use simpler encoding:
// Byte 0-1: sample_rate high 16 bits of 20
// Byte 2: sample_rate low 4 bits | channels 3 bits | bits high 1 bit
// Byte 3: bits low 4 bits | total_samples high 4 bits
// Actually the format is:
// 20 bits sample rate, 3 bits channels-1, 5 bits bits-1, 36 bits total samples
// 44100 = 0x0AC44
data.push(0x0A); // sample_rate bits 19-12
data.push(0xC4); // sample_rate bits 11-4
data.push(0x42); // sample_rate bits 3-0 (0x4), channels-1 (0x1=stereo), bits-1 high bit (0)
data.push(0xF0); // bits-1 low 4 bits (0xF=15, so 16 bits), total_samples high 4 bits (0)
// total_samples (remaining 32 bits) = 0
data.push(0x00);
data.push(0x00);
data.push(0x00);
data.push(0x00);
// MD5 signature (128 bits = 16 bytes)
data.extend_from_slice(&[0u8; 16]);
data
}
/// Create a FLAC header with Vorbis comments for testing extract().
fn make_flac_with_vorbis_comments() -> Vec<u8> {
let mut data = Vec::new();
// FLAC marker
data.extend_from_slice(b"fLaC");
// STREAMINFO block (not last)
data.push(0x00); // type=0, last=0
data.push(0x00);
data.push(0x00);
data.push(0x22); // 34 bytes
// STREAMINFO data (34 bytes)
data.push(0x10);
data.push(0x00);
data.push(0x10);
data.push(0x00);
data.extend_from_slice(&[0u8; 6]); // frame sizes
data.push(0x0A);
data.push(0xC4);
data.push(0x42);
data.push(0xF0);
data.extend_from_slice(&[0u8; 4]); // total samples
data.extend_from_slice(&[0u8; 16]); // MD5
// VORBIS_COMMENT block (last)
// Vendor: "test"
// Comments: TITLE=Test Song, ARTIST=Test Artist
let vendor = b"test";
let comments = [
b"TITLE=Test Song".as_slice(),
b"ARTIST=Test Artist".as_slice(),
b"ALBUM=Test Album".as_slice(),
b"TRACKNUMBER=3".as_slice(),
b"REPLAYGAIN_TRACK_GAIN=-5.50 dB".as_slice(),
];
let mut vorbis_data = Vec::new();
// Vendor length (LE)
vorbis_data.extend_from_slice(&(vendor.len() as u32).to_le_bytes());
vorbis_data.extend_from_slice(vendor);
// Comment count (LE)
vorbis_data.extend_from_slice(&(comments.len() as u32).to_le_bytes());
for comment in &comments {
vorbis_data.extend_from_slice(&(comment.len() as u32).to_le_bytes());
vorbis_data.extend_from_slice(*comment);
}
// VORBIS_COMMENT header
data.push(0x84); // type=4, last=1
data.push(((vorbis_data.len() >> 16) & 0xFF) as u8);
data.push(((vorbis_data.len() >> 8) & 0xFF) as u8);
data.push((vorbis_data.len() & 0xFF) as u8);
data.extend_from_slice(&vorbis_data);
data
}
#[test]
fn test_id_and_name() {
let handler = FlacHandler::new();
assert_eq!(handler.id(), "flac");
assert_eq!(handler.name(), "FLAC");
}
#[test]
fn test_extensions_and_mime_types() {
let handler = FlacHandler::new();
assert_eq!(handler.extensions(), &["flac"]);
assert_eq!(handler.mime_types(), &["audio/flac", "audio/x-flac"]);
}
#[test]
fn test_estimate_header_size() {
let handler = FlacHandler::new();
let meta = AudioMeta::default();
assert_eq!(handler.estimate_header_size(&meta), 8192);
}
#[test]
fn test_analyze_valid_flac() {
let handler = FlacHandler::new();
let data = make_minimal_flac_header();
let file_size = data.len() as u64 + 1000; // Pretend there's audio data
let result = handler.analyze(&data, file_size);
assert!(result.is_ok(), "analyze failed: {:?}", result.err());
let layout = result.unwrap();
assert_eq!(layout.audio_start, 42); // 4 (marker) + 38 (STREAMINFO)
assert_eq!(layout.audio_end, file_size);
assert_eq!(layout.format, AudioFormat::Flac);
assert!(layout.format_data.is_some());
assert_eq!(
layout.format_data.as_ref().unwrap().len(),
STREAMINFO_DATA_SIZE
);
}
#[test]
fn test_analyze_invalid_marker() {
let handler = FlacHandler::new();
let data = b"ID3\x04\x00\x00"; // MP3 header, not FLAC
let result = handler.analyze(data, 1000);
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), FormatError::InvalidData(_)));
}
#[test]
fn test_analyze_truncated() {
let handler = FlacHandler::new();
let data = b"fLaC"; // Just the marker, no blocks
let result = handler.analyze(data, 4);
assert!(result.is_err());
}
#[test]
fn test_synthesize_creates_valid_flac_header() {
let handler = FlacHandler::new();
let meta = make_test_meta();
// Create layout with STREAMINFO
let original_data = make_minimal_flac_header();
let layout = handler
.analyze(&original_data, original_data.len() as u64)
.unwrap();
let result = handler.synthesize(&meta, &layout);
assert!(result.is_ok(), "synthesize failed: {:?}", result.err());
let bytes = result.unwrap();
// Verify FLAC marker
assert!(bytes.len() >= 4);
assert_eq!(&bytes[0..4], b"fLaC");
// Verify STREAMINFO block header
assert_eq!(bytes[4] & 0x7F, BLOCK_TYPE_STREAMINFO); // Type 0
assert_eq!(bytes[4] & 0x80, 0); // Not last
// Verify STREAMINFO size
let streaminfo_size =
((bytes[5] as usize) << 16) | ((bytes[6] as usize) << 8) | (bytes[7] as usize);
assert_eq!(streaminfo_size, STREAMINFO_DATA_SIZE);
// Verify VORBIS_COMMENT block follows
let vorbis_offset = 4 + 4 + STREAMINFO_DATA_SIZE;
assert_eq!(bytes[vorbis_offset] & 0x7F, BLOCK_TYPE_VORBIS_COMMENT);
assert_eq!(bytes[vorbis_offset] & 0x80, 0x80); // Is last
}
#[test]
fn test_synthesize_preserves_streaminfo() {
let handler = FlacHandler::new();
let meta = AudioMeta::default();
// Create layout with specific STREAMINFO
let original_data = make_minimal_flac_header();
let layout = handler
.analyze(&original_data, original_data.len() as u64)
.unwrap();
let original_streaminfo = layout.format_data.as_ref().unwrap().clone();
let synthesized = handler.synthesize(&meta, &layout).unwrap();
// Extract STREAMINFO from synthesized header
let streaminfo_start = 4 + 4; // After marker and header
let streaminfo_end = streaminfo_start + STREAMINFO_DATA_SIZE;
let synthesized_streaminfo = &synthesized[streaminfo_start..streaminfo_end];
assert_eq!(synthesized_streaminfo, original_streaminfo.as_slice());
}
#[test]
fn test_synthesize_missing_streaminfo() {
let handler = FlacHandler::new();
let meta = AudioMeta::default();
let layout = FormatLayout {
audio_start: 42,
audio_end: 1000,
format: AudioFormat::Flac,
format_data: None, // Missing STREAMINFO
};
let result = handler.synthesize(&meta, &layout);
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
FormatError::SynthesisFailed(_)
));
}
#[test]
fn test_extract_from_flac() {
let handler = FlacHandler::new();
let data = make_flac_with_vorbis_comments();
let result = handler.extract(&data);
assert!(result.is_ok(), "extract failed: {:?}", result.err());
let meta = result.unwrap();
assert_eq!(meta.title, Some("Test Song".to_string()));
assert_eq!(meta.artist, Some("Test Artist".to_string()));
assert_eq!(meta.album, Some("Test Album".to_string()));
assert_eq!(meta.track, Some(3));
assert_eq!(meta.format, AudioFormat::Flac);
// Check ReplayGain parsing
let gain = meta.replaygain_track_gain.unwrap();
assert!((gain - (-5.5)).abs() < 0.01);
}
#[test]
fn test_build_and_extract_vorbis_comments() {
let original_meta = make_test_meta();
let tag = FlacHandler::build_vorbis_comments(&original_meta);
let extracted = FlacHandler::extract_from_vorbis_comments(&tag);
assert_eq!(extracted.title, original_meta.title);
assert_eq!(extracted.artist, original_meta.artist);
assert_eq!(extracted.album, original_meta.album);
assert_eq!(extracted.album_artist, original_meta.album_artist);
assert_eq!(extracted.genre, original_meta.genre);
assert_eq!(extracted.track, original_meta.track);
assert_eq!(extracted.track_total, original_meta.track_total);
assert_eq!(extracted.disc, original_meta.disc);
assert_eq!(extracted.disc_total, original_meta.disc_total);
assert_eq!(extracted.composer, original_meta.composer);
assert_eq!(extracted.comment, original_meta.comment);
assert_eq!(extracted.lyrics, original_meta.lyrics);
assert_eq!(extracted.copyright, original_meta.copyright);
assert_eq!(extracted.compilation, original_meta.compilation);
assert_eq!(extracted.title_sort, original_meta.title_sort);
assert_eq!(extracted.artist_sort, original_meta.artist_sort);
assert_eq!(extracted.album_sort, original_meta.album_sort);
assert_eq!(extracted.album_artist_sort, original_meta.album_artist_sort);
assert_eq!(extracted.mb_recording_id, original_meta.mb_recording_id);
assert_eq!(extracted.mb_album_id, original_meta.mb_album_id);
assert_eq!(extracted.mb_artist_id, original_meta.mb_artist_id);
assert_eq!(
extracted.mb_album_artist_id,
original_meta.mb_album_artist_id
);
assert_eq!(
extracted.mb_release_group_id,
original_meta.mb_release_group_id
);
assert_eq!(extracted.encoder, original_meta.encoder);
// ReplayGain values (with tolerance for formatting)
let orig_track_gain = original_meta.replaygain_track_gain.unwrap();
let ext_track_gain = extracted.replaygain_track_gain.unwrap();
assert!((orig_track_gain - ext_track_gain).abs() < 0.01);
let orig_track_peak = original_meta.replaygain_track_peak.unwrap();
let ext_track_peak = extracted.replaygain_track_peak.unwrap();
assert!((orig_track_peak - ext_track_peak).abs() < 0.0001);
}
#[test]
fn test_parse_replaygain_value() {
assert_eq!(FlacHandler::parse_replaygain_value("-6.50 dB"), Some(-6.50));
assert_eq!(FlacHandler::parse_replaygain_value("-6.50dB"), Some(-6.50));
assert_eq!(FlacHandler::parse_replaygain_value("-6.50"), Some(-6.50));
assert_eq!(FlacHandler::parse_replaygain_value(" 3.2 dB "), Some(3.2));
assert_eq!(FlacHandler::parse_replaygain_value("invalid"), None);
}
#[test]
fn test_parse_block_header() {
// Not last, type 0, size 34
let header = [0x00, 0x00, 0x00, 0x22];
let (is_last, block_type, size) = FlacHandler::parse_block_header(&header).unwrap();
assert!(!is_last);
assert_eq!(block_type, 0);
assert_eq!(size, 34);
// Last, type 4, size 256
let header = [0x84, 0x00, 0x01, 0x00];
let (is_last, block_type, size) = FlacHandler::parse_block_header(&header).unwrap();
assert!(is_last);
assert_eq!(block_type, 4);
assert_eq!(size, 256);
}
#[test]
fn test_write_block_header() {
let header = FlacHandler::write_block_header(false, 0, 34);
assert_eq!(header, [0x00, 0x00, 0x00, 0x22]);
let header = FlacHandler::write_block_header(true, 4, 256);
assert_eq!(header, [0x84, 0x00, 0x01, 0x00]);
}
#[test]
fn test_empty_metadata_produces_minimal_vorbis() {
let handler = FlacHandler::new();
let meta = AudioMeta::default();
let original_data = make_minimal_flac_header();
let layout = handler
.analyze(&original_data, original_data.len() as u64)
.unwrap();
let result = handler.synthesize(&meta, &layout);
assert!(result.is_ok());
let bytes = result.unwrap();
// Should have: fLaC (4) + STREAMINFO (38) + VORBIS_COMMENT (header + minimal data)
assert!(bytes.len() >= 42 + 4 + 8); // At least vendor string overhead
}
#[test]
fn test_round_trip_synthesize_analyze() {
let handler = FlacHandler::new();
let meta = make_test_meta();
// Create initial layout
let original_data = make_minimal_flac_header();
let layout = handler
.analyze(&original_data, original_data.len() as u64)
.unwrap();
// Synthesize new header
let synthesized = handler.synthesize(&meta, &layout).unwrap();
// Analyze synthesized header
let new_layout = handler
.analyze(&synthesized, synthesized.len() as u64)
.unwrap();
// STREAMINFO should be preserved
assert_eq!(new_layout.format_data, layout.format_data);
assert_eq!(new_layout.format, AudioFormat::Flac);
}
}
crates/musicfs-cache/src/handlers/id3v2.rs
-631
View File
@@ -1,631 +0,0 @@
use crate::{FormatError, FormatHandler, FormatLayout};
use lofty::config::{ParseOptions, WriteOptions};
use lofty::file::AudioFile;
use lofty::id3::v2::{
CommentFrame, Frame, FrameId, Id3v2Tag, TextInformationFrame, UnsynchronizedTextFrame,
};
use lofty::mpeg::MpegFile;
use lofty::tag::{Accessor, TagExt};
use lofty::TextEncoding;
use musicfs_core::{AudioFormat, AudioMeta};
use std::borrow::Cow;
use std::io::Cursor;
const ID3V2_HEADER_SIZE: usize = 10;
const ID3V1_TAG_SIZE: usize = 128;
pub struct Id3v2Handler;
impl Id3v2Handler {
pub fn new() -> Self {
Self
}
fn parse_id3v2_header(data: &[u8]) -> Option<usize> {
if data.len() < ID3V2_HEADER_SIZE {
return None;
}
if &data[0..3] != b"ID3" {
return None;
}
let size = syncsafe_decode(&data[6..10]);
Some(ID3V2_HEADER_SIZE + size)
}
fn has_id3v1_tag(data: &[u8], file_size: u64) -> bool {
if file_size < ID3V1_TAG_SIZE as u64 {
return false;
}
let tag_start = (file_size as usize).saturating_sub(ID3V1_TAG_SIZE);
if tag_start >= data.len() {
return false;
}
&data[tag_start..tag_start + 3] == b"TAG"
}
fn set_text_frame(tag: &mut Id3v2Tag, frame_id: &'static str, value: &str) {
let id = FrameId::Valid(Cow::Borrowed(frame_id));
let frame = Frame::Text(TextInformationFrame::new(
id,
TextEncoding::UTF8,
value.to_string(),
));
tag.insert(frame);
}
fn set_track_disc_frame(
tag: &mut Id3v2Tag,
frame_id: &'static str,
num: u32,
total: Option<u32>,
) {
let value = match total {
Some(t) => format!("{}/{}", num, t),
None => num.to_string(),
};
Self::set_text_frame(tag, frame_id, &value);
}
fn set_comment_frame(tag: &mut Id3v2Tag, value: &str) {
let frame = Frame::Comment(CommentFrame::new(
TextEncoding::UTF8,
*b"eng",
String::new(),
value.to_string(),
));
tag.insert(frame);
}
fn set_lyrics_frame(tag: &mut Id3v2Tag, value: &str) {
let frame = Frame::UnsynchronizedText(UnsynchronizedTextFrame::new(
TextEncoding::UTF8,
*b"eng",
String::new(),
value.to_string(),
));
tag.insert(frame);
}
fn build_tag_from_meta(metadata: &AudioMeta) -> Id3v2Tag {
let mut tag = Id3v2Tag::new();
if let Some(ref title) = metadata.title {
tag.set_title(title.clone());
}
if let Some(ref artist) = metadata.artist {
tag.set_artist(artist.clone());
}
if let Some(ref album) = metadata.album {
tag.set_album(album.clone());
}
if let Some(ref album_artist) = metadata.album_artist {
Self::set_text_frame(&mut tag, "TPE2", album_artist);
}
if let Some(year) = metadata.year {
Self::set_text_frame(&mut tag, "TDRC", &year.to_string());
}
if let Some(ref genre) = metadata.genre {
tag.set_genre(genre.clone());
}
if let Some(track) = metadata.track {
Self::set_track_disc_frame(&mut tag, "TRCK", track, metadata.track_total);
}
if let Some(disc) = metadata.disc {
Self::set_track_disc_frame(&mut tag, "TPOS", disc, metadata.disc_total);
}
if let Some(ref date) = metadata.date {
Self::set_text_frame(&mut tag, "TDRC", date);
}
if let Some(ref composer) = metadata.composer {
Self::set_text_frame(&mut tag, "TCOM", composer);
}
if let Some(ref comment) = metadata.comment {
Self::set_comment_frame(&mut tag, comment);
}
if let Some(ref lyrics) = metadata.lyrics {
Self::set_lyrics_frame(&mut tag, lyrics);
}
if let Some(ref copyright) = metadata.copyright {
Self::set_text_frame(&mut tag, "TCOP", copyright);
}
if let Some(compilation) = metadata.compilation {
Self::set_text_frame(&mut tag, "TCMP", if compilation { "1" } else { "0" });
}
if let Some(ref title_sort) = metadata.title_sort {
Self::set_text_frame(&mut tag, "TSOT", title_sort);
}
if let Some(ref artist_sort) = metadata.artist_sort {
Self::set_text_frame(&mut tag, "TSOP", artist_sort);
}
if let Some(ref album_sort) = metadata.album_sort {
Self::set_text_frame(&mut tag, "TSOA", album_sort);
}
if let Some(ref album_artist_sort) = metadata.album_artist_sort {
Self::set_text_frame(&mut tag, "TSO2", album_artist_sort);
}
if let Some(ref mb_recording_id) = metadata.mb_recording_id {
tag.insert_user_text(
"MusicBrainz Recording Id".to_string(),
mb_recording_id.clone(),
);
}
if let Some(ref mb_album_id) = metadata.mb_album_id {
tag.insert_user_text("MusicBrainz Album Id".to_string(), mb_album_id.clone());
}
if let Some(ref mb_artist_id) = metadata.mb_artist_id {
tag.insert_user_text("MusicBrainz Artist Id".to_string(), mb_artist_id.clone());
}
if let Some(ref mb_album_artist_id) = metadata.mb_album_artist_id {
tag.insert_user_text(
"MusicBrainz Album Artist Id".to_string(),
mb_album_artist_id.clone(),
);
}
if let Some(ref mb_release_group_id) = metadata.mb_release_group_id {
tag.insert_user_text(
"MusicBrainz Release Group Id".to_string(),
mb_release_group_id.clone(),
);
}
if let Some(gain) = metadata.replaygain_track_gain {
tag.insert_user_text(
"REPLAYGAIN_TRACK_GAIN".to_string(),
format!("{:.2} dB", gain),
);
}
if let Some(peak) = metadata.replaygain_track_peak {
tag.insert_user_text("REPLAYGAIN_TRACK_PEAK".to_string(), format!("{:.6}", peak));
}
if let Some(gain) = metadata.replaygain_album_gain {
tag.insert_user_text(
"REPLAYGAIN_ALBUM_GAIN".to_string(),
format!("{:.2} dB", gain),
);
}
if let Some(peak) = metadata.replaygain_album_peak {
tag.insert_user_text("REPLAYGAIN_ALBUM_PEAK".to_string(), format!("{:.6}", peak));
}
if let Some(ref encoder) = metadata.encoder {
Self::set_text_frame(&mut tag, "TSSE", encoder);
}
tag
}
fn extract_text_frame(tag: &Id3v2Tag, frame_id: &str) -> Option<String> {
let id = FrameId::new(frame_id).ok()?;
tag.get_text(&id).map(|s| s.to_string())
}
fn parse_track_disc(value: &str) -> (Option<u32>, Option<u32>) {
let parts: Vec<&str> = value.split('/').collect();
let num = parts.first().and_then(|s| s.parse().ok());
let total = parts.get(1).and_then(|s| s.parse().ok());
(num, total)
}
fn parse_replaygain_value(value: &str) -> Option<f32> {
value
.trim()
.trim_end_matches(" dB")
.trim_end_matches("dB")
.parse()
.ok()
}
fn extract_from_tag(tag: &Id3v2Tag) -> AudioMeta {
let mut meta = AudioMeta::default();
meta.format = AudioFormat::Mp3;
meta.title = tag.title().map(|c: Cow<'_, str>| c.into_owned());
meta.artist = tag.artist().map(|c: Cow<'_, str>| c.into_owned());
meta.album = tag.album().map(|c: Cow<'_, str>| c.into_owned());
meta.album_artist = Self::extract_text_frame(tag, "TPE2");
meta.genre = tag.genre().map(|c: Cow<'_, str>| c.into_owned());
if let Some(track_str) = Self::extract_text_frame(tag, "TRCK") {
let (track, track_total) = Self::parse_track_disc(&track_str);
meta.track = track;
meta.track_total = track_total;
} else {
meta.track = tag.track();
meta.track_total = tag.track_total();
}
if let Some(disc_str) = Self::extract_text_frame(tag, "TPOS") {
let (disc, disc_total) = Self::parse_track_disc(&disc_str);
meta.disc = disc;
meta.disc_total = disc_total;
} else {
meta.disc = tag.disk();
meta.disc_total = tag.disk_total();
}
meta.date = Self::extract_text_frame(tag, "TDRC");
if let Some(ref date) = meta.date {
if let Some(year_str) = date.split('-').next() {
meta.year = year_str.parse().ok();
}
}
meta.composer = Self::extract_text_frame(tag, "TCOM");
meta.comment = tag.comment().map(|c: Cow<'_, str>| c.into_owned());
if let Some(uslt) = tag.unsync_text().next() {
meta.lyrics = Some(uslt.content.to_string());
}
meta.copyright = Self::extract_text_frame(tag, "TCOP");
if let Some(tcmp) = Self::extract_text_frame(tag, "TCMP") {
meta.compilation = Some(tcmp == "1");
}
meta.title_sort = Self::extract_text_frame(tag, "TSOT");
meta.artist_sort = Self::extract_text_frame(tag, "TSOP");
meta.album_sort = Self::extract_text_frame(tag, "TSOA");
meta.album_artist_sort = Self::extract_text_frame(tag, "TSO2");
meta.mb_recording_id = tag
.get_user_text("MusicBrainz Recording Id")
.map(String::from);
meta.mb_album_id = tag.get_user_text("MusicBrainz Album Id").map(String::from);
meta.mb_artist_id = tag.get_user_text("MusicBrainz Artist Id").map(String::from);
meta.mb_album_artist_id = tag
.get_user_text("MusicBrainz Album Artist Id")
.map(String::from);
meta.mb_release_group_id = tag
.get_user_text("MusicBrainz Release Group Id")
.map(String::from);
if let Some(gain_str) = tag.get_user_text("REPLAYGAIN_TRACK_GAIN") {
meta.replaygain_track_gain = Self::parse_replaygain_value(gain_str);
}
if let Some(peak_str) = tag.get_user_text("REPLAYGAIN_TRACK_PEAK") {
meta.replaygain_track_peak = peak_str.parse::<f32>().ok();
}
if let Some(gain_str) = tag.get_user_text("REPLAYGAIN_ALBUM_GAIN") {
meta.replaygain_album_gain = Self::parse_replaygain_value(gain_str);
}
if let Some(peak_str) = tag.get_user_text("REPLAYGAIN_ALBUM_PEAK") {
meta.replaygain_album_peak = peak_str.parse::<f32>().ok();
}
meta.encoder = Self::extract_text_frame(tag, "TSSE");
meta
}
}
impl Default for Id3v2Handler {
fn default() -> Self {
Self::new()
}
}
impl FormatHandler for Id3v2Handler {
fn id(&self) -> &'static str {
"id3v2"
}
fn name(&self) -> &'static str {
"ID3v2 (MP3)"
}
fn extensions(&self) -> &[&'static str] {
&["mp3"]
}
fn mime_types(&self) -> &[&'static str] {
&["audio/mpeg"]
}
fn analyze(&self, data: &[u8], file_size: u64) -> Result<FormatLayout, FormatError> {
let audio_start = Self::parse_id3v2_header(data).unwrap_or(0) as u64;
let audio_end = if Self::has_id3v1_tag(data, file_size) {
file_size - ID3V1_TAG_SIZE as u64
} else {
file_size
};
Ok(FormatLayout {
audio_start,
audio_end,
format: AudioFormat::Mp3,
format_data: None,
})
}
fn synthesize(
&self,
metadata: &AudioMeta,
_layout: &FormatLayout,
) -> Result<Vec<u8>, FormatError> {
let tag = Self::build_tag_from_meta(metadata);
let mut buffer = Cursor::new(Vec::new());
let write_options = WriteOptions::new().preferred_padding(1024);
tag.dump_to(&mut buffer, write_options)
.map_err(|e| FormatError::SynthesisFailed(e.to_string()))?;
Ok(buffer.into_inner())
}
fn extract(&self, data: &[u8]) -> Result<AudioMeta, FormatError> {
let mut cursor = Cursor::new(data);
let mpeg_file = MpegFile::read_from(&mut cursor, ParseOptions::new())
.map_err(|e| FormatError::InvalidData(e.to_string()))?;
let tag = mpeg_file
.id3v2()
.ok_or_else(|| FormatError::InvalidData("No ID3v2 tag found".to_string()))?;
Ok(Self::extract_from_tag(tag))
}
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
4096 + 1024
}
}
fn syncsafe_decode(bytes: &[u8]) -> usize {
((bytes[0] as usize) << 21)
| ((bytes[1] as usize) << 14)
| ((bytes[2] as usize) << 7)
| (bytes[3] as usize)
}
#[cfg(test)]
mod tests {
use super::*;
fn make_test_meta() -> AudioMeta {
AudioMeta {
title: Some("Test Title".to_string()),
artist: Some("Test Artist".to_string()),
album: Some("Test Album".to_string()),
album_artist: Some("Test Album Artist".to_string()),
genre: Some("Rock".to_string()),
year: Some(2024),
track: Some(5),
track_total: Some(12),
disc: Some(1),
disc_total: Some(2),
format: AudioFormat::Mp3,
date: Some("2024-03-15".to_string()),
composer: Some("Test Composer".to_string()),
comment: Some("Test Comment".to_string()),
lyrics: Some("Test Lyrics\nLine 2".to_string()),
copyright: Some("2024 Test Copyright".to_string()),
compilation: Some(false),
title_sort: Some("Title, Test".to_string()),
artist_sort: Some("Artist, Test".to_string()),
album_sort: Some("Album, Test".to_string()),
album_artist_sort: Some("Album Artist, Test".to_string()),
mb_recording_id: Some("rec-12345".to_string()),
mb_album_id: Some("alb-12345".to_string()),
mb_artist_id: Some("art-12345".to_string()),
mb_album_artist_id: Some("albart-12345".to_string()),
mb_release_group_id: Some("rg-12345".to_string()),
replaygain_track_gain: Some(-6.5),
replaygain_track_peak: Some(0.987654),
replaygain_album_gain: Some(-5.2),
replaygain_album_peak: Some(0.999999),
encoder: Some("LAME 3.100".to_string()),
..Default::default()
}
}
#[test]
fn test_id_and_name() {
let handler = Id3v2Handler::new();
assert_eq!(handler.id(), "id3v2");
assert_eq!(handler.name(), "ID3v2 (MP3)");
}
#[test]
fn test_extensions_and_mime_types() {
let handler = Id3v2Handler::new();
assert_eq!(handler.extensions(), &["mp3"]);
assert_eq!(handler.mime_types(), &["audio/mpeg"]);
}
#[test]
fn test_estimate_header_size() {
let handler = Id3v2Handler::new();
let meta = AudioMeta::default();
assert_eq!(handler.estimate_header_size(&meta), 5120);
}
#[test]
fn test_synthesize_creates_valid_id3v2() {
let handler = Id3v2Handler::new();
let meta = make_test_meta();
let layout = FormatLayout {
audio_start: 0,
audio_end: 1000,
format: AudioFormat::Mp3,
format_data: None,
};
let result = handler.synthesize(&meta, &layout);
assert!(result.is_ok());
let bytes = result.unwrap();
assert!(bytes.len() >= 10);
assert_eq!(&bytes[0..3], b"ID3");
assert_eq!(bytes[3], 0x04);
}
#[test]
fn test_analyze_no_id3v2() {
let handler = Id3v2Handler::new();
let data = vec![0xFF, 0xFB, 0x90, 0x00];
let file_size = 1000;
let result = handler.analyze(&data, file_size);
assert!(result.is_ok());
let layout = result.unwrap();
assert_eq!(layout.audio_start, 0);
assert_eq!(layout.audio_end, 1000);
assert_eq!(layout.format, AudioFormat::Mp3);
}
#[test]
fn test_analyze_with_id3v2() {
let handler = Id3v2Handler::new();
let mut data = vec![b'I', b'D', b'3', 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64];
data.extend(vec![0u8; 100]);
let file_size = data.len() as u64;
let result = handler.analyze(&data, file_size);
assert!(result.is_ok());
let layout = result.unwrap();
assert_eq!(layout.audio_start, 110);
assert_eq!(layout.audio_end, file_size);
}
#[test]
fn test_analyze_with_id3v1() {
let handler = Id3v2Handler::new();
let mut data = vec![0xFF, 0xFB, 0x90, 0x00];
data.extend(vec![0u8; 100]);
data.extend(b"TAG");
data.extend(vec![0u8; 125]);
let file_size = data.len() as u64;
let result = handler.analyze(&data, file_size);
assert!(result.is_ok());
let layout = result.unwrap();
assert_eq!(layout.audio_start, 0);
assert_eq!(layout.audio_end, file_size - 128);
}
#[test]
fn test_syncsafe_decode() {
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x00, 0x7F]), 127);
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x01, 0x00]), 128);
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x00, 0x64]), 100);
}
#[test]
fn test_parse_track_disc() {
assert_eq!(Id3v2Handler::parse_track_disc("5/12"), (Some(5), Some(12)));
assert_eq!(Id3v2Handler::parse_track_disc("5"), (Some(5), None));
assert_eq!(Id3v2Handler::parse_track_disc(""), (None, None));
}
#[test]
fn test_parse_replaygain_value() {
assert_eq!(
Id3v2Handler::parse_replaygain_value("-6.50 dB"),
Some(-6.50)
);
assert_eq!(Id3v2Handler::parse_replaygain_value("-6.50dB"), Some(-6.50));
assert_eq!(Id3v2Handler::parse_replaygain_value("-6.50"), Some(-6.50));
assert_eq!(Id3v2Handler::parse_replaygain_value("invalid"), None);
}
#[test]
fn test_empty_metadata_produces_empty_tag() {
let handler = Id3v2Handler::new();
let meta = AudioMeta::default();
let layout = FormatLayout {
audio_start: 0,
audio_end: 1000,
format: AudioFormat::Mp3,
format_data: None,
};
let result = handler.synthesize(&meta, &layout);
assert!(result.is_ok());
let bytes = result.unwrap();
assert!(bytes.is_empty());
}
#[test]
fn test_minimal_metadata_produces_valid_tag() {
let handler = Id3v2Handler::new();
let mut meta = AudioMeta::default();
meta.title = Some("Test".to_string());
let layout = FormatLayout {
audio_start: 0,
audio_end: 1000,
format: AudioFormat::Mp3,
format_data: None,
};
let result = handler.synthesize(&meta, &layout);
assert!(result.is_ok());
let bytes = result.unwrap();
assert!(bytes.len() >= 10);
assert_eq!(&bytes[0..3], b"ID3");
assert_eq!(bytes[3], 0x04);
}
#[test]
fn test_build_and_extract_tag() {
let original_meta = make_test_meta();
let tag = Id3v2Handler::build_tag_from_meta(&original_meta);
let extracted = Id3v2Handler::extract_from_tag(&tag);
assert_eq!(extracted.title, original_meta.title);
assert_eq!(extracted.artist, original_meta.artist);
assert_eq!(extracted.album, original_meta.album);
assert_eq!(extracted.album_artist, original_meta.album_artist);
assert_eq!(extracted.genre, original_meta.genre);
assert_eq!(extracted.track, original_meta.track);
assert_eq!(extracted.track_total, original_meta.track_total);
assert_eq!(extracted.disc, original_meta.disc);
assert_eq!(extracted.disc_total, original_meta.disc_total);
assert_eq!(extracted.composer, original_meta.composer);
assert_eq!(extracted.comment, original_meta.comment);
assert_eq!(extracted.lyrics, original_meta.lyrics);
assert_eq!(extracted.copyright, original_meta.copyright);
assert_eq!(extracted.compilation, original_meta.compilation);
assert_eq!(extracted.title_sort, original_meta.title_sort);
assert_eq!(extracted.artist_sort, original_meta.artist_sort);
assert_eq!(extracted.album_sort, original_meta.album_sort);
assert_eq!(extracted.album_artist_sort, original_meta.album_artist_sort);
assert_eq!(extracted.mb_recording_id, original_meta.mb_recording_id);
assert_eq!(extracted.mb_album_id, original_meta.mb_album_id);
assert_eq!(extracted.mb_artist_id, original_meta.mb_artist_id);
assert_eq!(
extracted.mb_album_artist_id,
original_meta.mb_album_artist_id
);
assert_eq!(
extracted.mb_release_group_id,
original_meta.mb_release_group_id
);
assert_eq!(extracted.encoder, original_meta.encoder);
let orig_track_gain = original_meta.replaygain_track_gain.unwrap();
let ext_track_gain = extracted.replaygain_track_gain.unwrap();
assert!((orig_track_gain - ext_track_gain).abs() < 0.01);
let orig_track_peak = original_meta.replaygain_track_peak.unwrap();
let ext_track_peak = extracted.replaygain_track_peak.unwrap();
assert!((orig_track_peak - ext_track_peak).abs() < 0.0001);
}
}
crates/musicfs-cache/src/handlers/mod.rs
-12
View File
@@ -1,12 +0,0 @@
//! Format-specific metadata handlers for audio file synthesis.
//!
//! Each handler implements the `FormatHandler` trait to support:
//! - Analyzing original files to find audio boundaries
//! - Synthesizing new headers from database metadata
//! - Extracting metadata from existing files
mod flac;
mod id3v2;
pub use flac::FlacHandler;
pub use id3v2::Id3v2Handler;
crates/musicfs-cache/src/lib.rs
-26
View File
@@ -1,26 +0,0 @@
mod artwork;
mod db;
mod eviction;
mod format_handler;
mod format_layout;
pub mod handlers;
mod metadata;
mod overlay;
mod patterns;
mod prefetch;
mod tree;
pub use artwork::{ArtworkCache, ArtworkError, CachedArtwork};
pub use db::{Database, EnrichmentUpdate, TrashedFile, TrashedFilter};
pub use eviction::{EvictionError, EvictionPolicy, LruEviction};
pub use format_handler::{FormatError, FormatHandler, FormatHandlerRegistry};
pub use format_layout::FormatLayout;
pub use handlers::{FlacHandler, Id3v2Handler};
pub use metadata::MetadataCache;
pub use overlay::{OverlayError, OverlayReader};
pub use patterns::{AccessContext, AccessPattern, PatternError, PatternStore};
pub use prefetch::{PrefetchConfig, PrefetchEngine, PrefetchHandle};
pub use tree::{
DirNode, FileNode, Inode, RefreshPolicy, RemoveError, RenameError, TreeBuilder, VirtualNode,
VirtualTree, ROOT_INODE,
};
crates/musicfs-cache/src/metadata.rs
-138
View File
@@ -1,138 +0,0 @@
use crate::db::Database;
use musicfs_core::{AudioMeta, FileMeta, OriginId, Result, VirtualPath};
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tracing::trace;
pub struct MetadataCache {
db: Arc<Database>,
}
impl MetadataCache {
pub fn new(db: Arc<Database>) -> Self {
Self { db }
}
pub fn store(
&self,
origin_id: &OriginId,
real_path: &Path,
virtual_path: &VirtualPath,
audio_meta: &AudioMeta,
origin_mtime: SystemTime,
origin_size: u64,
) -> Result<()> {
self.db.upsert_file(
origin_id,
real_path,
virtual_path,
audio_meta,
origin_mtime,
origin_size,
)?;
Ok(())
}
pub fn lookup(&self, path: &VirtualPath) -> Result<Option<FileMeta>> {
let result = self.db.get_file_by_virtual_path(path)?;
let hit = result.is_some();
trace!(path = path.as_str(), hit, "metadata cache lookup");
Ok(result)
}
pub fn is_fresh(
&self,
origin_id: &OriginId,
real_path: &Path,
current_mtime: SystemTime,
) -> Result<bool> {
if let Some(cached_mtime) = self.db.get_mtime_by_real_path(origin_id, real_path)? {
let current_secs = current_mtime
.duration_since(UNIX_EPOCH)
.unwrap_or(Duration::ZERO)
.as_secs();
let cached_secs = cached_mtime
.duration_since(UNIX_EPOCH)
.unwrap_or(Duration::ZERO)
.as_secs();
let hit = current_secs == cached_secs;
trace!(path = ?real_path, hit, "metadata freshness check");
Ok(hit)
} else {
trace!(path = ?real_path, hit = false, "metadata freshness check");
Ok(false)
}
}
pub fn invalidate(&self, path: &VirtualPath) -> Result<()> {
if let Some(meta) = self.db.get_file_by_virtual_path(path)? {
self.db.delete_file(meta.id)?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use musicfs_core::AudioFormat;
#[test]
fn test_metadata_cache_store_and_lookup() {
let db = Arc::new(Database::open_memory().unwrap());
let cache = MetadataCache::new(db);
let origin_id = OriginId::from("local");
let real_path = Path::new("/music/song.flac");
let virtual_path = VirtualPath::new("/Artist/Album/Song.flac");
let meta = AudioMeta {
title: Some("Song".to_string()),
artist: Some("Artist".to_string()),
format: AudioFormat::Flac,
..Default::default()
};
cache
.store(
&origin_id,
real_path,
&virtual_path,
&meta,
UNIX_EPOCH,
5000,
)
.unwrap();
let retrieved = cache.lookup(&virtual_path).unwrap().unwrap();
assert_eq!(
retrieved.audio.as_ref().unwrap().title,
Some("Song".to_string())
);
}
#[test]
fn test_metadata_cache_invalidate() {
let db = Arc::new(Database::open_memory().unwrap());
let cache = MetadataCache::new(db);
let virtual_path = VirtualPath::new("/Test.flac");
cache
.store(
&OriginId::from("local"),
Path::new("/test.flac"),
&virtual_path,
&AudioMeta::default(),
UNIX_EPOCH,
100,
)
.unwrap();
assert!(cache.lookup(&virtual_path).unwrap().is_some());
cache.invalidate(&virtual_path).unwrap();
assert!(cache.lookup(&virtual_path).unwrap().is_none());
}
}
crates/musicfs-cache/src/overlay.rs
-467
View File
@@ -1,467 +0,0 @@
//! OverlayReader: On-the-fly metadata overlay with header/audio splice logic.
//!
//! This module provides the core read path for metadata overlay. It synthesizes
//! headers on-the-fly from database metadata and splices them with original audio
//! data from the CAS.
use crate::{Database, FormatError, FormatHandlerRegistry};
use bytes::{Bytes, BytesMut};
use musicfs_cas::{FileReader, ReaderError};
use musicfs_core::{AudioFormat, FileId};
use std::sync::Arc;
use tracing::{debug, trace};
/// Error types for overlay operations
#[derive(Debug, thiserror::Error)]
pub enum OverlayError {
#[error("Database error: {0}")]
Database(#[from] musicfs_core::Error),
#[error("Format handler error: {0}")]
Handler(#[from] FormatError),
#[error("CAS error: {0}")]
Cas(#[from] ReaderError),
#[error("File not found: {0:?}")]
NotFound(FileId),
#[error("No handler for format: {0:?}")]
NoHandler(AudioFormat),
}
/// OverlayReader provides on-the-fly metadata overlay for audio files.
///
/// It synthesizes headers from database metadata and splices them with
/// original audio data from the CAS, presenting a virtual file that
/// reflects the current metadata state.
pub struct OverlayReader {
db: Arc<Database>,
registry: Arc<FormatHandlerRegistry>,
cas_reader: Arc<FileReader>,
}
impl OverlayReader {
/// Create a new OverlayReader with the given dependencies.
pub fn new(
db: Arc<Database>,
registry: Arc<FormatHandlerRegistry>,
cas_reader: Arc<FileReader>,
) -> Self {
Self {
db,
registry,
cas_reader,
}
}
/// Read bytes from a virtual file with metadata overlay.
///
/// This method implements the three-region splice logic:
/// - Region 1: Synthetic header (offset < header_len)
/// - Region 2: Audio data from CAS (offset >= header_len)
/// - Region 3: Boundary crossing (spans header/audio)
///
/// If no format_layout exists for the file, delegates directly to CAS reader.
pub async fn read(
&self,
file_id: FileId,
offset: u64,
size: u32,
) -> Result<Bytes, OverlayError> {
// Get format layout - if None, passthrough to CAS
let layout = match self.db.get_format_layout(file_id)? {
Some(layout) => layout,
None => {
trace!(file_id = ?file_id, "No format_layout, passthrough to CAS");
return Ok(self.cas_reader.read(file_id, offset, size).await?);
}
};
// Get metadata for synthesis
let metadata = self.db.get_file_metadata_row(file_id)?;
// Get handler for this format (handler IDs are lowercase)
let format_id = format!("{:?}", layout.format).to_lowercase();
let handler = self
.registry
.get_by_format(&format_id)
.ok_or_else(|| OverlayError::NoHandler(layout.format))?;
// Synthesize header on-the-fly
let header = handler.synthesize(&metadata, &layout)?;
let header_len = header.len() as u64;
let audio_len = layout.audio_end - layout.audio_start;
let virtual_size = header_len + audio_len;
trace!(
file_id = ?file_id,
header_len,
audio_len,
virtual_size,
offset,
size,
"Overlay read"
);
// Handle EOF
if offset >= virtual_size {
return Ok(Bytes::new());
}
let virtual_end = (offset + size as u64).min(virtual_size);
let mut result = BytesMut::with_capacity((virtual_end - offset) as usize);
// Region 1: Synthetic header
if offset < header_len {
let end = virtual_end.min(header_len);
result.extend_from_slice(&header[offset as usize..end as usize]);
trace!(
file_id = ?file_id,
start = offset,
end,
bytes = end - offset,
"Read from synthetic header"
);
}
// Region 2: Origin audio data (from CAS)
if virtual_end > header_len {
let audio_start_in_virtual = header_len.max(offset);
let audio_offset_in_origin = layout.audio_start + (audio_start_in_virtual - header_len);
let audio_bytes_needed = (virtual_end - audio_start_in_virtual) as u32;
trace!(
file_id = ?file_id,
audio_offset_in_origin,
audio_bytes_needed,
"Read from CAS audio"
);
let audio = self
.cas_reader
.read(file_id, audio_offset_in_origin, audio_bytes_needed)
.await?;
result.extend_from_slice(&audio);
}
debug!(
file_id = ?file_id,
offset,
size,
returned = result.len(),
"Overlay read complete"
);
Ok(result.freeze())
}
/// Estimate the virtual size of a file for getattr.
///
/// Returns the estimated size based on format layout. If no layout exists,
/// returns None to indicate the caller should use the original file size.
pub fn estimate_virtual_size(&self, file_id: FileId) -> Result<Option<u64>, OverlayError> {
// Get format layout - if None, return None to indicate passthrough
let layout = match self.db.get_format_layout(file_id)? {
Some(layout) => layout,
None => return Ok(None),
};
// Get metadata for header size estimation
let metadata = self.db.get_file_metadata_row(file_id)?;
let format_id = format!("{:?}", layout.format).to_lowercase();
let handler = self
.registry
.get_by_format(&format_id)
.ok_or_else(|| OverlayError::NoHandler(layout.format))?;
// Estimate header size
let estimated_header = handler.estimate_header_size(&metadata) as u64;
let audio_len = layout.audio_end - layout.audio_start;
let virtual_size = estimated_header + audio_len;
trace!(
file_id = ?file_id,
estimated_header,
audio_len,
virtual_size,
"Estimated virtual size"
);
Ok(Some(virtual_size))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::handlers::FlacHandler;
use crate::FormatLayout;
use musicfs_cas::{CasConfig, CasStore, ChunkManifest, ChunkRef};
use musicfs_core::{AudioFormat, AudioMeta, OriginId, VirtualPath};
use std::path::Path;
use std::time::UNIX_EPOCH;
use tempfile::TempDir;
fn make_test_metadata() -> AudioMeta {
AudioMeta {
title: Some("Test Track".to_string()),
artist: Some("Test Artist".to_string()),
album: Some("Test Album".to_string()),
track: Some(1),
format: AudioFormat::Flac,
sample_rate: Some(44100),
bits_per_sample: Some(16),
channels: Some(2),
..Default::default()
}
}
fn make_test_layout() -> FormatLayout {
// Simulate a file with minimal FLAC header, audio from 42 to 102442 (100KB audio)
// STREAMINFO data (34 bytes) - minimal valid values for FLAC synthesis
let streaminfo_data = vec![
0x10, 0x00, // min_block_size = 4096
0x10, 0x00, // max_block_size = 4096
0x00, 0x00, 0x00, // min_frame_size = 0
0x00, 0x00, 0x00, // max_frame_size = 0
0x0A, 0xC4, 0x42, 0xF0, // sample_rate=44100, channels=2, bits=16
0x00, 0x00, 0x00, 0x00, // total_samples
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MD5 (16 bytes)
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
];
FormatLayout {
audio_start: 42, // fLaC (4) + STREAMINFO block (38)
audio_end: 42 + 100 * 1024, // 100KB audio
format: AudioFormat::Flac,
format_data: Some(streaminfo_data),
}
}
async fn setup_test_env() -> (
TempDir,
Arc<Database>,
Arc<FormatHandlerRegistry>,
Arc<FileReader>,
FileId,
) {
let dir = TempDir::new().unwrap();
// Setup database
let db = Arc::new(Database::open_memory().unwrap());
// Setup registry with FLAC handler
let mut registry = FormatHandlerRegistry::new();
registry.register(Arc::new(FlacHandler::new()));
let registry = Arc::new(registry);
// Setup CAS store and reader
let cas_config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
// Create test audio data (simulating 100KB of audio)
let audio_data: Vec<u8> = (0..100 * 1024).map(|i| (i % 256) as u8).collect();
let hash = store.put(&audio_data).await.unwrap();
let reader = Arc::new(FileReader::new(store));
// Register manifest for the test file
// The manifest represents the ORIGINAL file in CAS, with audio starting at offset 42
reader.register_manifest(ChunkManifest {
file_id: FileId(1),
total_size: 42 + 100 * 1024, // Original file size (42 byte header + 100KB audio)
mtime: 0,
chunks: vec![ChunkRef {
hash,
offset: 42, // Audio starts at offset 42 in the original file
size: audio_data.len() as u32,
}],
});
let file_id = db
.upsert_file_with_layout(
&OriginId::from("test"),
Path::new("/test.flac"),
&VirtualPath::new("/Test Artist/Test Album/01 - Test Track.flac"),
&make_test_metadata(),
UNIX_EPOCH,
42 + 100 * 1024,
Some(&make_test_layout()),
None,
)
.unwrap();
(dir, db, registry, reader, file_id)
}
#[tokio::test]
async fn test_read_header_region() {
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
let overlay = OverlayReader::new(db, registry, reader);
// Read first 100 bytes (should be from synthetic header)
let result = overlay.read(file_id, 0, 100).await.unwrap();
// Should return data (synthetic header)
assert!(!result.is_empty());
assert!(result.len() <= 100);
// FLAC files start with "fLaC" magic
assert_eq!(&result[0..4], b"fLaC");
}
#[tokio::test]
async fn test_read_audio_region() {
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
let overlay = OverlayReader::new(db.clone(), registry.clone(), reader.clone());
// First, get the actual header size by reading it
let _header_result = overlay.read(file_id, 0, 64 * 1024).await.unwrap();
// Get the layout to know where audio starts in virtual file
let layout = db.get_format_layout(file_id).unwrap().unwrap();
let metadata = db.get_file_metadata_row(file_id).unwrap();
let handler = registry.get_by_format("flac").unwrap();
let header = handler.synthesize(&metadata, &layout).unwrap();
let header_len = header.len() as u64;
// Read from well into the audio region
let audio_offset = header_len + 1000;
let result = overlay.read(file_id, audio_offset, 1000).await.unwrap();
// Should return audio data
assert!(!result.is_empty());
}
#[tokio::test]
async fn test_read_boundary() {
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
let overlay = OverlayReader::new(db.clone(), registry.clone(), reader.clone());
// Get the actual header size
let layout = db.get_format_layout(file_id).unwrap().unwrap();
let metadata = db.get_file_metadata_row(file_id).unwrap();
let handler = registry.get_by_format("flac").unwrap();
let header = handler.synthesize(&metadata, &layout).unwrap();
let header_len = header.len() as u64;
// Read across the header/audio boundary
let boundary_offset = header_len - 50;
let result = overlay.read(file_id, boundary_offset, 100).await.unwrap();
// Should return 100 bytes spanning both regions
assert_eq!(result.len(), 100);
// First 50 bytes should be from header
assert_eq!(&result[0..50], &header[(header_len - 50) as usize..]);
}
#[tokio::test]
async fn test_passthrough() {
let dir = TempDir::new().unwrap();
let db = Arc::new(Database::open_memory().unwrap());
let registry = Arc::new(FormatHandlerRegistry::new());
let cas_config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
let test_data = b"Hello, World! This is test data.";
let hash = store.put(test_data).await.unwrap();
// Insert file WITHOUT format_layout first to get the file_id
let file_id = db
.upsert_file(
&OriginId::from("test"),
Path::new("/test.txt"),
&VirtualPath::new("/test.txt"),
&AudioMeta::default(),
UNIX_EPOCH,
test_data.len() as u64,
)
.unwrap();
let reader = Arc::new(FileReader::new(store));
// Register manifest with the actual file_id from database
reader.register_manifest(ChunkManifest {
file_id,
total_size: test_data.len() as u64,
mtime: 0,
chunks: vec![ChunkRef {
hash,
offset: 0,
size: test_data.len() as u32,
}],
});
let overlay = OverlayReader::new(db, registry, reader);
let result = overlay
.read(file_id, 0, test_data.len() as u32)
.await
.unwrap();
assert_eq!(&result[..], test_data);
}
#[tokio::test]
async fn test_estimate_virtual_size() {
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
let overlay = OverlayReader::new(db, registry, reader);
// Should return estimated size
let size = overlay.estimate_virtual_size(file_id).unwrap();
assert!(size.is_some());
let virtual_size = size.unwrap();
// Virtual size should be header + audio (100KB audio)
assert!(virtual_size > 100 * 1024);
}
#[tokio::test]
async fn test_estimate_virtual_size_passthrough() {
let dir = TempDir::new().unwrap();
let db = Arc::new(Database::open_memory().unwrap());
let registry = Arc::new(FormatHandlerRegistry::new());
let cas_config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
let reader = Arc::new(FileReader::new(store));
// Insert file WITHOUT format_layout
let file_id = db
.upsert_file(
&OriginId::from("test"),
Path::new("/test.txt"),
&VirtualPath::new("/test.txt"),
&AudioMeta::default(),
UNIX_EPOCH,
1000,
)
.unwrap();
let overlay = OverlayReader::new(db, registry, reader);
// Should return None for passthrough
let size = overlay.estimate_virtual_size(file_id).unwrap();
assert!(size.is_none());
}
#[tokio::test]
async fn test_read_eof() {
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
let overlay = OverlayReader::new(db, registry, reader);
// Read past EOF
let result = overlay.read(file_id, 1_000_000, 100).await.unwrap();
assert!(result.is_empty());
}
}
crates/musicfs-cache/src/patterns.rs
-291
View File
@@ -1,291 +0,0 @@
use musicfs_core::FileId;
use parking_lot::{Mutex, RwLock};
use std::collections::HashMap;
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::{debug, info, trace};
#[derive(Debug, Clone)]
pub struct AccessPattern {
pub file_id: FileId,
pub timestamp: SystemTime,
pub context: AccessContext,
pub hour_of_day: u8,
}
#[derive(Debug, Clone, Default)]
pub struct AccessContext {
pub album_id: Option<i64>,
pub track_number: Option<u32>,
pub artist: Option<String>,
}
pub struct PatternStore {
db: Mutex<rusqlite::Connection>,
sequence_counts: RwLock<HashMap<(FileId, FileId), u32>>,
time_patterns: RwLock<HashMap<u8, Vec<FileId>>>,
max_history: usize,
}
impl PatternStore {
pub fn new(db_path: &Path, max_history: usize) -> Result<Self, PatternError> {
let db = rusqlite::Connection::open(db_path)?;
db.execute(
"CREATE TABLE IF NOT EXISTS access_log (
id INTEGER PRIMARY KEY,
file_id INTEGER NOT NULL,
access_time INTEGER NOT NULL,
hour_of_day INTEGER NOT NULL
)",
[],
)?;
db.execute(
"CREATE INDEX IF NOT EXISTS idx_access_log_file ON access_log(file_id)",
[],
)?;
db.execute(
"CREATE INDEX IF NOT EXISTS idx_access_log_time ON access_log(access_time)",
[],
)?;
db.execute(
"CREATE TABLE IF NOT EXISTS sequence_counts (
from_file_id INTEGER NOT NULL,
to_file_id INTEGER NOT NULL,
count INTEGER NOT NULL DEFAULT 1,
PRIMARY KEY (from_file_id, to_file_id)
)",
[],
)?;
let sequence_counts = {
let mut map = HashMap::new();
let mut stmt =
db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?;
let rows = stmt.query_map([], |row| {
Ok((
(FileId(row.get::<_, i64>(0)?), FileId(row.get::<_, i64>(1)?)),
row.get::<_, u32>(2)?,
))
})?;
for row in rows {
let (key, count) = row?;
map.insert(key, count);
}
map
};
let store = Self {
db: Mutex::new(db),
sequence_counts: RwLock::new(sequence_counts),
time_patterns: RwLock::new(HashMap::new()),
max_history,
};
let sequence_count = store.sequence_counts.read().len();
info!(path = ?db_path, sequence_count = sequence_count, max_history = max_history, "Pattern store opened");
Ok(store)
}
pub fn record(&self, file_id: FileId, _context: AccessContext) -> Result<(), PatternError> {
trace!(file_id = file_id.0, "Recording access pattern");
let now = SystemTime::now();
let timestamp = now.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64;
let hour = (timestamp / 3600 % 24) as u8;
let db = self.db.lock();
db.execute(
"INSERT INTO access_log (file_id, access_time, hour_of_day) VALUES (?1, ?2, ?3)",
rusqlite::params![file_id.0, timestamp, hour],
)?;
{
let mut time_patterns = self.time_patterns.write();
time_patterns.entry(hour).or_default().push(file_id);
}
let prev_file_id: Option<i64> = db
.query_row(
"SELECT file_id FROM access_log WHERE id = (SELECT MAX(id) - 1 FROM access_log)",
[],
|row| row.get(0),
)
.ok();
if let Some(prev_id) = prev_file_id {
let prev = FileId(prev_id);
{
let mut sequences = self.sequence_counts.write();
*sequences.entry((prev, file_id)).or_insert(0) += 1;
}
db.execute(
"INSERT INTO sequence_counts (from_file_id, to_file_id, count)
VALUES (?1, ?2, 1)
ON CONFLICT(from_file_id, to_file_id) DO UPDATE SET count = count + 1",
rusqlite::params![prev_id, file_id.0],
)?;
}
let cutoff = timestamp - (self.max_history as i64 * 86400);
db.execute("DELETE FROM access_log WHERE access_time < ?1", [cutoff])?;
Ok(())
}
pub fn predict_next(&self, current: FileId, limit: usize) -> Vec<FileId> {
let sequences = self.sequence_counts.read();
let mut predictions: Vec<_> = sequences
.iter()
.filter(|((from, _), count)| *from == current && **count >= 2)
.map(|((_, to), count)| (*to, *count))
.collect();
predictions.sort_by(|a, b| b.1.cmp(&a.1));
let result: Vec<FileId> = predictions
.into_iter()
.take(limit)
.map(|(id, _)| id)
.collect();
debug!(
file_id = current.0,
predictions = result.len(),
"Predicted next files"
);
result
}
pub fn predict_for_time(&self, hour: u8, limit: usize) -> Vec<FileId> {
let time_patterns = self.time_patterns.read();
time_patterns
.get(&hour)
.map(|files| files.iter().rev().take(limit).copied().collect())
.unwrap_or_default()
}
pub fn recently_played(&self, days: u32) -> Result<Vec<FileId>, PatternError> {
let cutoff = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs() as i64
- (days as i64 * 86400);
let db = self.db.lock();
let mut stmt = db.prepare(
"SELECT DISTINCT file_id FROM access_log WHERE access_time >= ?1 ORDER BY access_time DESC",
)?;
let files: Vec<FileId> = stmt
.query_map([cutoff], |row| Ok(FileId(row.get(0)?)))?
.filter_map(|r| r.ok())
.collect();
Ok(files)
}
pub fn most_played(&self, limit: u32) -> Result<Vec<FileId>, PatternError> {
let db = self.db.lock();
let mut stmt = db.prepare(
"SELECT file_id, COUNT(*) as play_count FROM access_log
GROUP BY file_id ORDER BY play_count DESC LIMIT ?1",
)?;
let files: Vec<FileId> = stmt
.query_map([limit], |row| Ok(FileId(row.get(0)?)))?
.filter_map(|r| r.ok())
.collect();
Ok(files)
}
}
#[derive(Debug, thiserror::Error)]
pub enum PatternError {
#[error("database error: {0}")]
Database(#[from] rusqlite::Error),
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[test]
fn test_pattern_prediction() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("patterns.db");
let store = PatternStore::new(&db_path, 30).unwrap();
let ctx = AccessContext::default();
for _ in 0..5 {
store.record(FileId(1), ctx.clone()).unwrap();
store.record(FileId(2), ctx.clone()).unwrap();
store.record(FileId(3), ctx.clone()).unwrap();
}
let predictions = store.predict_next(FileId(1), 3);
assert!(!predictions.is_empty());
assert_eq!(predictions[0], FileId(2));
}
#[test]
fn test_pattern_persistence() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("patterns.db");
let ctx = AccessContext::default();
{
let store = PatternStore::new(&db_path, 30).unwrap();
for _ in 0..3 {
store.record(FileId(1), ctx.clone()).unwrap();
store.record(FileId(2), ctx.clone()).unwrap();
}
}
{
let store = PatternStore::new(&db_path, 30).unwrap();
let predictions = store.predict_next(FileId(1), 3);
assert!(!predictions.is_empty());
assert_eq!(predictions[0], FileId(2));
}
}
#[test]
fn test_recently_played() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("patterns.db");
let store = PatternStore::new(&db_path, 30).unwrap();
let ctx = AccessContext::default();
store.record(FileId(100), ctx.clone()).unwrap();
store.record(FileId(200), ctx.clone()).unwrap();
let recent = store.recently_played(7).unwrap();
assert!(recent.contains(&FileId(100)));
assert!(recent.contains(&FileId(200)));
}
#[test]
fn test_most_played() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("patterns.db");
let store = PatternStore::new(&db_path, 30).unwrap();
let ctx = AccessContext::default();
for _ in 0..5 {
store.record(FileId(1), ctx.clone()).unwrap();
}
for _ in 0..2 {
store.record(FileId(2), ctx.clone()).unwrap();
}
let most = store.most_played(10).unwrap();
assert_eq!(most[0], FileId(1));
}
}
crates/musicfs-cache/src/prefetch.rs
-197
View File
@@ -1,197 +0,0 @@
use crate::patterns::{AccessContext, PatternStore};
use musicfs_cas::ContentFetcher;
use musicfs_core::{Event, EventBus, FileId};
use parking_lot::Mutex as ParkingMutex;
use std::collections::HashSet;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Semaphore;
use tokio::task::JoinHandle;
use tracing::{debug, info, warn};
const DEFAULT_PREFETCH_LOOKAHEAD: usize = 3;
const DEFAULT_MAX_CONCURRENT: usize = 2;
const DEFAULT_COOLDOWN_MS: u64 = 100;
#[derive(Debug, Clone)]
pub struct PrefetchConfig {
pub lookahead: usize,
pub max_concurrent: usize,
pub cooldown: Duration,
pub enabled: bool,
}
impl Default for PrefetchConfig {
fn default() -> Self {
Self {
lookahead: DEFAULT_PREFETCH_LOOKAHEAD,
max_concurrent: DEFAULT_MAX_CONCURRENT,
cooldown: Duration::from_millis(DEFAULT_COOLDOWN_MS),
enabled: true,
}
}
}
pub struct PrefetchEngine {
config: PrefetchConfig,
fetcher: Arc<ContentFetcher>,
in_flight: Arc<ParkingMutex<HashSet<FileId>>>,
semaphore: Arc<Semaphore>,
running: Arc<AtomicBool>,
}
pub struct PrefetchHandle {
handle: JoinHandle<()>,
running: Arc<AtomicBool>,
}
impl PrefetchHandle {
pub async fn stop(self) {
self.running.store(false, Ordering::SeqCst);
let _ = self.handle.await;
}
}
impl PrefetchEngine {
pub fn new(
config: PrefetchConfig,
_pattern_store: Arc<PatternStore>,
fetcher: Arc<ContentFetcher>,
) -> Self {
let semaphore = Arc::new(Semaphore::new(config.max_concurrent));
Self {
config,
fetcher,
in_flight: Arc::new(ParkingMutex::new(HashSet::new())),
semaphore,
running: Arc::new(AtomicBool::new(false)),
}
}
pub fn start(
self: Arc<Self>,
event_bus: Arc<EventBus>,
pattern_store: Arc<PatternStore>,
) -> PrefetchHandle {
self.running.store(true, Ordering::SeqCst);
let running = self.running.clone();
let config = self.config.clone();
let fetcher = self.fetcher.clone();
let in_flight = self.in_flight.clone();
let semaphore = self.semaphore.clone();
let running_inner = running.clone();
let handle = tokio::spawn(async move {
let mut rx = event_bus.subscribe();
while running_inner.load(Ordering::SeqCst) {
match tokio::time::timeout(Duration::from_secs(1), rx.recv()).await {
Ok(Ok(event)) => {
if let Event::FileAccessed { file_id, .. } = event {
if config.enabled {
let ctx = AccessContext::default();
if let Err(e) = pattern_store.record(file_id, ctx) {
warn!("Failed to record access pattern: {}", e);
continue;
}
let predictions =
pattern_store.predict_next(file_id, config.lookahead);
for predicted_id in predictions {
prefetch_file(predicted_id, &fetcher, &in_flight, &semaphore)
.await;
}
tokio::time::sleep(config.cooldown).await;
}
}
}
Ok(Err(_)) => break,
Err(_) => continue,
}
}
info!("Prefetch engine stopped");
});
PrefetchHandle { handle, running }
}
pub fn is_running(&self) -> bool {
self.running.load(Ordering::SeqCst)
}
pub fn in_flight_count(&self) -> usize {
self.in_flight.lock().len()
}
pub fn update_config(&mut self, config: PrefetchConfig) {
self.config = config;
}
}
async fn prefetch_file(
file_id: FileId,
fetcher: &Arc<ContentFetcher>,
in_flight: &Arc<ParkingMutex<HashSet<FileId>>>,
semaphore: &Arc<Semaphore>,
) {
{
let mut guard = in_flight.lock();
if guard.contains(&file_id) {
debug!("Skipping prefetch for {:?} - already in flight", file_id);
return;
}
guard.insert(file_id);
}
let permit = match semaphore.clone().try_acquire_owned() {
Ok(p) => p,
Err(_) => {
debug!("Skipping prefetch for {:?} - concurrency limit", file_id);
in_flight.lock().remove(&file_id);
return;
}
};
let fetcher = fetcher.clone();
let in_flight = in_flight.clone();
tokio::spawn(async move {
debug!("Prefetching file {:?}", file_id);
match fetcher.ensure_cached(file_id).await {
Ok(manifest) => {
info!(
"Prefetched {:?}: {} chunks, {} bytes",
file_id,
manifest.chunks.len(),
manifest.total_size
);
}
Err(e) => {
debug!("Prefetch failed for {:?}: {}", file_id, e);
}
}
in_flight.lock().remove(&file_id);
drop(permit);
});
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_prefetch_config_defaults() {
let config = PrefetchConfig::default();
assert_eq!(config.lookahead, 3);
assert_eq!(config.max_concurrent, 2);
assert!(config.enabled);
}
}
crates/musicfs-cache/src/schema.sql
-111
View File
@@ -1,111 +0,0 @@
PRAGMA journal_mode = WAL;
PRAGMA foreign_keys = ON;
PRAGMA synchronous = NORMAL;
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY,
origin_id TEXT NOT NULL,
real_path TEXT NOT NULL,
virtual_path TEXT NOT NULL,
title TEXT,
artist TEXT,
album TEXT,
album_artist TEXT,
genre TEXT,
year INTEGER,
track INTEGER,
disc INTEGER,
duration_ms INTEGER,
bitrate INTEGER,
sample_rate INTEGER,
format TEXT,
track_total INTEGER,
disc_total INTEGER,
date TEXT,
composer TEXT,
comment TEXT,
lyrics TEXT,
copyright TEXT,
compilation INTEGER,
artist_sort TEXT,
album_artist_sort TEXT,
album_sort TEXT,
title_sort TEXT,
mb_recording_id TEXT,
mb_album_id TEXT,
mb_artist_id TEXT,
mb_album_artist_id TEXT,
mb_release_group_id TEXT,
replaygain_track_gain REAL,
replaygain_track_peak REAL,
replaygain_album_gain REAL,
replaygain_album_peak REAL,
channels INTEGER,
bits_per_sample INTEGER,
encoder TEXT,
custom_tags TEXT,
format_layout BLOB,
label TEXT,
album_type TEXT,
cover_url TEXT,
genres_json TEXT,
enrichment_source TEXT,
enriched_at INTEGER,
enrichment_attempts INTEGER NOT NULL DEFAULT 0,
last_enrichment_error TEXT,
origin_mtime INTEGER NOT NULL,
origin_size INTEGER NOT NULL,
content_hash TEXT,
chunk_manifest BLOB,
last_sync INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
trashed INTEGER NOT NULL DEFAULT 0,
original_path TEXT,
trashed_at INTEGER,
UNIQUE(origin_id, real_path)
);
CREATE TABLE IF NOT EXISTS artwork (
id INTEGER PRIMARY KEY,
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
art_type TEXT NOT NULL,
chunk_hash TEXT NOT NULL,
width INTEGER,
height INTEGER,
mime_type TEXT,
UNIQUE(file_id, art_type)
);
CREATE TABLE IF NOT EXISTS collections (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
query_json TEXT NOT NULL,
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
);
CREATE INDEX IF NOT EXISTS idx_files_virtual ON files(virtual_path);
CREATE INDEX IF NOT EXISTS idx_files_artist_album ON files(artist, album);
CREATE INDEX IF NOT EXISTS idx_files_content_hash ON files(content_hash);
CREATE INDEX IF NOT EXISTS idx_files_real ON files(origin_id, real_path);
CREATE INDEX IF NOT EXISTS idx_files_origin ON files(origin_id);
CREATE INDEX IF NOT EXISTS idx_files_last_sync ON files(last_sync);
CREATE INDEX IF NOT EXISTS idx_files_mb_album ON files(mb_album_id);
CREATE INDEX IF NOT EXISTS idx_files_mb_artist ON files(mb_artist_id);
CREATE INDEX IF NOT EXISTS idx_files_genre ON files(genre);
CREATE INDEX IF NOT EXISTS idx_files_year ON files(year);
CREATE INDEX IF NOT EXISTS idx_files_composer ON files(composer);
CREATE INDEX IF NOT EXISTS idx_artwork_file ON artwork(file_id);
CREATE TABLE IF NOT EXISTS directories (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL UNIQUE,
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
);
CREATE INDEX IF NOT EXISTS idx_directories_path ON directories(path);
CREATE INDEX IF NOT EXISTS idx_files_trashed ON files(trashed) WHERE trashed = 1;
crates/musicfs-cache/src/tree.rs
-1224
View File
File diff suppressed because it is too large Load Diff
crates/musicfs-cas/Cargo.toml
-29
View File
@@ -1,29 +0,0 @@
[package]
name = "musicfs-cas"
version.workspace = true
edition.workspace = true
[features]
default = []
failpoints = ["fail/failpoints"]
[dependencies]
fail = { workspace = true, optional = true }
musicfs-core = { path = "../musicfs-core" }
musicfs-origins = { path = "../musicfs-origins" }
musicfs-sync = { path = "../musicfs-sync" }
tokio.workspace = true
tracing.workspace = true
serde.workspace = true
sled.workspace = true
xxhash-rust.workspace = true
bytes.workspace = true
rmp-serde.workspace = true
hex.workspace = true
dirs.workspace = true
thiserror.workspace = true
parking_lot.workspace = true
[dev-dependencies]
tempfile.workspace = true
musicfs-cache = { path = "../musicfs-cache" }
crates/musicfs-cas/src/chunks.rs
-45
View File
@@ -1,45 +0,0 @@
use musicfs_core::ChunkHash;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChunkLocation {
pub path: PathBuf,
pub size: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChunkRef {
pub hash: ChunkHash,
pub offset: u64,
pub size: u32,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_chunk_hash_from_bytes() {
let data = b"hello world";
let hash = ChunkHash::from_bytes(data);
assert_eq!(hash.as_hex().len(), 16);
}
#[test]
fn test_chunk_hash_deterministic() {
let data = b"test data";
let hash1 = ChunkHash::from_bytes(data);
let hash2 = ChunkHash::from_bytes(data);
assert_eq!(hash1, hash2);
}
#[test]
fn test_chunk_hash_hex_roundtrip() {
let data = b"roundtrip test";
let hash = ChunkHash::from_bytes(data);
let hex = hash.as_hex();
let restored = ChunkHash::from_hex(&hex).unwrap();
assert_eq!(hash, restored);
}
}
crates/musicfs-cas/src/fetcher.rs
-284
View File
@@ -1,284 +0,0 @@
use crate::{CasStore, ChunkManifest, ChunkRef};
use musicfs_core::{Event, EventBus, FileId, FileMeta, OriginId};
use musicfs_origins::Origin;
use musicfs_sync::CdcChunker;
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::Arc;
use tracing::{debug, info, warn};
pub struct ContentFetcher {
store: Arc<CasStore>,
origins: RwLock<HashMap<OriginId, Arc<dyn Origin>>>,
file_meta: RwLock<HashMap<FileId, FileMeta>>,
event_bus: Option<Arc<EventBus>>,
chunker: CdcChunker,
}
impl ContentFetcher {
pub fn new(store: Arc<CasStore>) -> Self {
Self {
store,
origins: RwLock::new(HashMap::new()),
file_meta: RwLock::new(HashMap::new()),
event_bus: None,
chunker: CdcChunker::default(),
}
}
pub fn with_event_bus(store: Arc<CasStore>, event_bus: Arc<EventBus>) -> Self {
Self {
store,
origins: RwLock::new(HashMap::new()),
file_meta: RwLock::new(HashMap::new()),
event_bus: Some(event_bus),
chunker: CdcChunker::default(),
}
}
pub fn register_origin(&self, origin: Arc<dyn Origin>) {
let id = origin.id().clone();
self.origins.write().insert(id, origin);
}
pub fn register_file(&self, meta: FileMeta) {
self.file_meta.write().insert(meta.id, meta);
}
pub fn register_files(&self, files: impl IntoIterator<Item = FileMeta>) {
let mut map = self.file_meta.write();
for meta in files {
map.insert(meta.id, meta);
}
}
pub async fn fetch_file(&self, file_id: FileId) -> Result<ChunkManifest, FetchError> {
let meta = {
let files = self.file_meta.read();
files
.get(&file_id)
.cloned()
.ok_or(FetchError::FileNotFound(file_id))?
};
let origin = {
let origins = self.origins.read();
origins
.get(&meta.real_path.origin_id)
.cloned()
.ok_or_else(|| FetchError::OriginNotFound(meta.real_path.origin_id.clone()))?
};
info!("Fetching file {:?} from origin {}", file_id, origin.id());
let data = origin
.read_full(&meta.real_path.path)
.await
.map_err(|e| FetchError::OriginRead(e.to_string()))?;
let mtime = meta
.mtime
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0);
let chunks = self.chunker.chunk_refs(&data);
info!("Chunked {:?} into {} chunks", file_id, chunks.len());
let mut chunk_refs = Vec::with_capacity(chunks.len());
for chunk in chunks {
if !self.store.exists(&chunk.hash) {
if let Err(e) = self.store.put(chunk.data).await {
warn!(hash = %chunk.hash, error = %e, "CAS write failed, continuing in passthrough mode");
}
}
chunk_refs.push(ChunkRef {
hash: chunk.hash,
offset: chunk.offset,
size: chunk.length,
});
}
let manifest = ChunkManifest {
file_id,
total_size: meta.size,
mtime,
chunks: chunk_refs,
};
debug!(
"Created manifest for {:?}: {} bytes, {} chunks",
file_id,
meta.size,
manifest.chunks.len()
);
Ok(manifest)
}
pub async fn ensure_cached(&self, file_id: FileId) -> Result<ChunkManifest, FetchError> {
self.fetch_file(file_id).await
}
pub fn get_file_meta(&self, file_id: FileId) -> Option<FileMeta> {
self.file_meta.read().get(&file_id).cloned()
}
pub fn emit_access_event(&self, meta: &FileMeta, offset: u64, size: u32) {
if let Some(bus) = &self.event_bus {
bus.publish(Event::FileAccessed {
file_id: meta.id,
path: meta.virtual_path.clone(),
origin_id: meta.real_path.origin_id.clone(),
offset,
size,
});
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum FetchError {
#[error("File not found: {0:?}")]
FileNotFound(FileId),
#[error("Origin not found: {0}")]
OriginNotFound(OriginId),
#[error("Origin read error: {0}")]
OriginRead(String),
#[error("Store error: {0}")]
Store(#[from] crate::CasError),
}
#[cfg(test)]
mod tests {
use super::*;
use crate::CasConfig;
use musicfs_core::{RealPath, VirtualPath};
use musicfs_origins::LocalOrigin;
use std::path::PathBuf;
use std::time::SystemTime;
use tempfile::TempDir;
#[tokio::test]
async fn test_fetch_file() {
let cas_dir = TempDir::new().unwrap();
let origin_dir = TempDir::new().unwrap();
std::fs::write(origin_dir.path().join("test.flac"), b"fake audio data").unwrap();
let config = CasConfig {
chunks_dir: cas_dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let fetcher = ContentFetcher::new(store.clone());
let origin = Arc::new(LocalOrigin::new("local", origin_dir.path()));
fetcher.register_origin(origin);
let meta = FileMeta {
id: FileId(1),
virtual_path: VirtualPath::new("/Artist/Album/test.flac"),
real_path: RealPath {
origin_id: OriginId::from("local"),
path: PathBuf::from("/test.flac"),
},
size: 15,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
};
fetcher.register_file(meta);
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
assert_eq!(manifest.total_size, 15);
assert_eq!(manifest.chunks.len(), 1);
let data = store.get(&manifest.chunks[0].hash).await.unwrap();
assert_eq!(&data[..], b"fake audio data");
}
#[tokio::test]
async fn test_fetch_file_not_found() {
let cas_dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: cas_dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let fetcher = ContentFetcher::new(store);
let result = fetcher.fetch_file(FileId(999)).await;
assert!(matches!(result, Err(FetchError::FileNotFound(_))));
}
#[tokio::test]
async fn test_fetch_emits_event() {
let cas_dir = TempDir::new().unwrap();
let origin_dir = TempDir::new().unwrap();
std::fs::write(origin_dir.path().join("test.flac"), b"audio").unwrap();
let config = CasConfig {
chunks_dir: cas_dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let event_bus = Arc::new(EventBus::default());
let mut rx = event_bus.subscribe();
let fetcher = ContentFetcher::with_event_bus(store, event_bus);
let origin = Arc::new(LocalOrigin::new("local", origin_dir.path()));
fetcher.register_origin(origin);
let meta = FileMeta {
id: FileId(1),
virtual_path: VirtualPath::new("/Artist/test.flac"),
real_path: RealPath {
origin_id: OriginId::from("local"),
path: PathBuf::from("/test.flac"),
},
size: 5,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
};
fetcher.register_file(meta.clone());
fetcher.emit_access_event(&meta, 0, 5);
let event = rx.try_recv().unwrap();
assert!(matches!(event, Event::FileAccessed { .. }));
}
#[tokio::test]
async fn test_fetch_origin_not_found() {
let cas_dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: cas_dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let fetcher = ContentFetcher::new(store);
let meta = FileMeta {
id: FileId(1),
virtual_path: VirtualPath::new("/test.flac"),
real_path: RealPath {
origin_id: OriginId::from("nonexistent"),
path: PathBuf::from("/test.flac"),
},
size: 100,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
};
fetcher.register_file(meta);
let result = fetcher.fetch_file(FileId(1)).await;
assert!(matches!(result, Err(FetchError::OriginNotFound(_))));
}
}
crates/musicfs-cas/src/lib.rs
-9
View File
@@ -1,9 +0,0 @@
mod chunks;
mod fetcher;
mod reader;
mod store;
pub use chunks::{ChunkLocation, ChunkRef};
pub use fetcher::{ContentFetcher, FetchError};
pub use reader::{ChunkManifest, FileReader, ReaderError};
pub use store::{CasConfig, CasError, CasStore, DedupStats};
crates/musicfs-cas/src/reader.rs
-332
View File
@@ -1,332 +0,0 @@
use crate::chunks::ChunkRef;
use crate::fetcher::{ContentFetcher, FetchError};
use crate::store::{CasError, CasStore};
use bytes::{Bytes, BytesMut};
use musicfs_core::FileId;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tracing::{debug, trace, warn};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChunkManifest {
pub file_id: FileId,
pub total_size: u64,
pub mtime: i64,
pub chunks: Vec<ChunkRef>,
}
impl ChunkManifest {
pub fn chunks_to_bytes(&self) -> Vec<u8> {
rmp_serde::to_vec(&self.chunks).unwrap_or_default()
}
pub fn chunks_from_bytes(data: &[u8]) -> Option<Vec<ChunkRef>> {
rmp_serde::from_slice(data).ok()
}
pub fn from_db(
file_id: FileId,
total_size: u64,
mtime: i64,
chunk_blob: &[u8],
) -> Option<Self> {
let chunks = Self::chunks_from_bytes(chunk_blob)?;
Some(Self {
file_id,
total_size,
mtime,
chunks,
})
}
}
pub struct FileReader {
store: Arc<CasStore>,
fetcher: Option<Arc<ContentFetcher>>,
manifests: RwLock<HashMap<FileId, ChunkManifest>>,
}
impl FileReader {
pub fn new(store: Arc<CasStore>) -> Self {
Self {
store,
fetcher: None,
manifests: RwLock::new(HashMap::new()),
}
}
pub fn with_fetcher(store: Arc<CasStore>, fetcher: Arc<ContentFetcher>) -> Self {
Self {
store,
fetcher: Some(fetcher),
manifests: RwLock::new(HashMap::new()),
}
}
pub fn register_manifest(&self, manifest: ChunkManifest) {
let mut manifests = self.manifests.write();
manifests.insert(manifest.file_id, manifest);
}
async fn get_or_fetch_manifest(&self, file_id: FileId) -> Result<ChunkManifest, ReaderError> {
{
let manifests = self.manifests.read();
if let Some(m) = manifests.get(&file_id) {
trace!(file_id = ?file_id, "manifest cache hit");
return Ok(m.clone());
}
}
trace!(file_id = ?file_id, "manifest cache miss");
let Some(fetcher) = &self.fetcher else {
return Err(ReaderError::ManifestNotFound(file_id));
};
let manifest = fetcher.ensure_cached(file_id).await?;
self.manifests.write().insert(file_id, manifest.clone());
Ok(manifest)
}
pub async fn read(
&self,
file_id: FileId,
offset: u64,
size: u32,
) -> Result<Bytes, ReaderError> {
let manifest = self.get_or_fetch_manifest(file_id).await?;
if let Some(fetcher) = &self.fetcher {
if let Some(meta) = fetcher.get_file_meta(file_id) {
fetcher.emit_access_event(&meta, offset, size);
}
}
if offset >= manifest.total_size {
return Ok(Bytes::new());
}
let end = std::cmp::min(offset + size as u64, manifest.total_size);
let mut result = BytesMut::with_capacity((end - offset) as usize);
let mut chunks_read = 0u32;
for chunk_ref in &manifest.chunks {
let chunk_start = chunk_ref.offset;
let chunk_end = chunk_ref.offset + chunk_ref.size as u64;
if chunk_end <= offset || chunk_start >= end {
continue;
}
let chunk_data = match self.store.get(&chunk_ref.hash).await {
Ok(data) => data,
Err(CasError::IntegrityError { .. }) => {
warn!(hash = %chunk_ref.hash, "Chunk corrupt, deleting and re-fetching");
let _ = self.store.delete(&chunk_ref.hash).await;
if let Some(fetcher) = &self.fetcher {
let new_manifest = fetcher.fetch_file(file_id).await?;
self.manifests.write().insert(file_id, new_manifest);
self.store.get(&chunk_ref.hash).await?
} else {
return Err(ReaderError::Cas(CasError::NotFound(
chunk_ref.hash.as_hex(),
)));
}
}
Err(CasError::NotFound(_)) => {
warn!(hash = %chunk_ref.hash, "Chunk missing, attempting re-fetch");
if let Some(fetcher) = &self.fetcher {
let new_manifest = fetcher.fetch_file(file_id).await?;
self.manifests.write().insert(file_id, new_manifest);
self.store.get(&chunk_ref.hash).await?
} else {
return Err(ReaderError::Cas(CasError::NotFound(
chunk_ref.hash.as_hex(),
)));
}
}
Err(e) => return Err(ReaderError::Cas(e)),
};
let read_start = if offset > chunk_start {
(offset - chunk_start) as usize
} else {
0
};
let read_end = if end < chunk_end {
(end - chunk_start) as usize
} else {
chunk_ref.size as usize
};
result.extend_from_slice(&chunk_data[read_start..read_end]);
chunks_read += 1;
}
let bytes_read = result.len() as u64;
debug!(file_id = ?file_id, offset, size, chunks_read, bytes_read, "read completed");
Ok(result.freeze())
}
}
#[derive(Debug, thiserror::Error)]
pub enum ReaderError {
#[error("Manifest not found for file {0:?}")]
ManifestNotFound(FileId),
#[error("Fetch error: {0}")]
Fetch(#[from] FetchError),
#[error("CAS error: {0}")]
Cas(#[from] crate::store::CasError),
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::CasConfig;
use musicfs_core::ChunkHash;
use tempfile::TempDir;
#[tokio::test]
async fn test_file_reader_simple() {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let data = b"Hello, World!";
let hash = store.put(data).await.unwrap();
let reader = FileReader::new(store);
reader.register_manifest(ChunkManifest {
file_id: FileId(1),
total_size: data.len() as u64,
mtime: 0,
chunks: vec![ChunkRef {
hash,
offset: 0,
size: data.len() as u32,
}],
});
let result = reader.read(FileId(1), 0, data.len() as u32).await.unwrap();
assert_eq!(&result[..], data);
}
#[tokio::test]
async fn test_file_reader_partial() {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let data = b"ABCDEFGHIJ";
let hash = store.put(data).await.unwrap();
let reader = FileReader::new(store);
reader.register_manifest(ChunkManifest {
file_id: FileId(1),
total_size: data.len() as u64,
mtime: 0,
chunks: vec![ChunkRef {
hash,
offset: 0,
size: data.len() as u32,
}],
});
let result = reader.read(FileId(1), 3, 4).await.unwrap();
assert_eq!(&result[..], b"DEFG");
}
#[tokio::test]
async fn test_file_reader_multi_chunk() {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let chunk1 = b"AAAA";
let chunk2 = b"BBBB";
let hash1 = store.put(chunk1).await.unwrap();
let hash2 = store.put(chunk2).await.unwrap();
let reader = FileReader::new(store);
reader.register_manifest(ChunkManifest {
file_id: FileId(1),
total_size: 8,
mtime: 0,
chunks: vec![
ChunkRef {
hash: hash1,
offset: 0,
size: 4,
},
ChunkRef {
hash: hash2,
offset: 4,
size: 4,
},
],
});
let result = reader.read(FileId(1), 2, 4).await.unwrap();
assert_eq!(&result[..], b"AABB");
}
#[tokio::test]
async fn test_file_reader_eof() {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let data = b"short";
let hash = store.put(data).await.unwrap();
let reader = FileReader::new(store);
reader.register_manifest(ChunkManifest {
file_id: FileId(1),
total_size: data.len() as u64,
mtime: 0,
chunks: vec![ChunkRef {
hash,
offset: 0,
size: data.len() as u32,
}],
});
let result = reader.read(FileId(1), 100, 10).await.unwrap();
assert!(result.is_empty());
}
#[test]
fn test_chunk_manifest_serialization() {
let manifest = ChunkManifest {
file_id: FileId(42),
total_size: 1024,
mtime: 0,
chunks: vec![ChunkRef {
hash: ChunkHash::from_bytes(b"test"),
offset: 0,
size: 1024,
}],
};
let bytes = manifest.chunks_to_bytes();
let restored = ChunkManifest::chunks_from_bytes(&bytes).unwrap();
assert_eq!(restored.len(), 1);
assert_eq!(restored[0].size, 1024);
}
}
crates/musicfs-cas/src/store.rs
-396
View File
@@ -1,396 +0,0 @@
use crate::chunks::ChunkLocation;
use bytes::Bytes;
use musicfs_core::ChunkHash;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::fs;
use tracing::{debug, info, trace, warn};
#[cfg(feature = "failpoints")]
use fail::fail_point;
const DEFAULT_MAX_SIZE_10GB: u64 = 10 * 1024 * 1024 * 1024;
const DEFAULT_SHARD_LEVELS_256_SUBDIRS: u8 = 2;
#[derive(Debug, Clone)]
pub struct CasConfig {
pub chunks_dir: PathBuf,
pub max_size: u64,
pub shard_levels: u8,
}
impl Default for CasConfig {
fn default() -> Self {
let cache_dir = dirs::cache_dir()
.unwrap_or_else(|| PathBuf::from(".cache"))
.join("musicfs")
.join("chunks");
Self {
chunks_dir: cache_dir,
max_size: DEFAULT_MAX_SIZE_10GB,
shard_levels: DEFAULT_SHARD_LEVELS_256_SUBDIRS,
}
}
}
pub struct CasStore {
config: CasConfig,
index: sled::Db,
current_size: AtomicU64,
}
impl CasStore {
pub async fn open(config: CasConfig) -> Result<Self, CasError> {
fs::create_dir_all(&config.chunks_dir).await?;
let index_path = config.chunks_dir.join("index.sled");
let index = match sled::open(&index_path) {
Ok(db) => db,
Err(e) => {
warn!(error = %e, path = ?index_path, "sled index corrupted, attempting recovery");
match sled::Config::new().path(&index_path).open() {
Ok(db) => {
info!("sled index repaired successfully");
db
}
Err(repair_err) => {
warn!(error = %repair_err, "sled repair failed, recreating index");
if index_path.exists() {
std::fs::remove_dir_all(&index_path).map_err(CasError::Io)?;
}
sled::open(&index_path)?
}
}
}
};
let current_size = Self::calculate_size(&config.chunks_dir).await;
Ok(Self {
config,
index,
current_size: AtomicU64::new(current_size),
})
}
async fn calculate_size(dir: &Path) -> u64 {
Self::calculate_size_recursive(dir).await
}
fn calculate_size_recursive(
dir: &Path,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = u64> + Send + '_>> {
Box::pin(async move {
let mut size = 0u64;
if let Ok(mut entries) = fs::read_dir(dir).await {
while let Ok(Some(entry)) = entries.next_entry().await {
if let Ok(meta) = entry.metadata().await {
if meta.is_file() {
size += meta.len();
} else if meta.is_dir() {
// Skip sled index directory
let name = entry.file_name();
if name != "index.sled" {
size += Self::calculate_size_recursive(&entry.path()).await;
}
}
}
}
}
size
})
}
pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
let hash = ChunkHash::from_bytes(data);
let path = self.chunk_path(&hash);
if path.exists() {
trace!(hash = %hash, size_bytes = data.len(), "dedup hit");
return Ok(hash);
}
if self.config.max_size > 0 {
let new_size = self.current_size.load(Ordering::SeqCst) + data.len() as u64;
if new_size > self.config.max_size {
warn!(
current_size = self.current_size.load(Ordering::SeqCst),
chunk_size = data.len(),
max_size = self.config.max_size,
"CAS store full, rejecting write"
);
return Err(CasError::StoreFull {
current: self.current_size.load(Ordering::SeqCst),
max: self.config.max_size,
});
}
}
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
#[cfg(feature = "failpoints")]
fail_point!("cas-put-before-write", |_| {
Err(CasError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"Failpoint: cas-put-before-write",
)))
});
fs::write(&path, data).await?;
#[cfg(feature = "failpoints")]
fail_point!("cas-put-after-write-before-index", |_| {
Err(CasError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"Failpoint: cas-put-after-write-before-index",
)))
});
let location = ChunkLocation {
path: path.clone(),
size: data.len() as u32,
};
self.index.insert(
hash.0.as_slice(),
rmp_serde::to_vec(&location).map_err(|e| CasError::Serialization(e.to_string()))?,
)?;
self.current_size
.fetch_add(data.len() as u64, Ordering::SeqCst);
debug!(hash = %hash, size_bytes = data.len(), "chunk stored");
Ok(hash)
}
pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
let path = self.chunk_path(hash);
if !path.exists() {
return Err(CasError::NotFound(hash.as_hex()));
}
let data = fs::read(&path).await?;
if self.config.max_size > 0 {
self.verify_integrity(hash, &data)?;
}
debug!(hash = %hash, size_bytes = data.len(), "chunk retrieved");
Ok(Bytes::from(data))
}
pub fn exists(&self, hash: &ChunkHash) -> bool {
self.chunk_path(hash).exists()
}
fn verify_integrity(&self, expected: &ChunkHash, data: &[u8]) -> Result<(), CasError> {
let actual = ChunkHash::from_bytes(data);
if actual != *expected {
warn!(
"Chunk integrity failure: expected {}, got {}",
expected, actual
);
return Err(CasError::IntegrityError {
expected: expected.as_hex(),
actual: actual.as_hex(),
});
}
Ok(())
}
fn chunk_path(&self, hash: &ChunkHash) -> PathBuf {
let hex = hash.as_hex();
let mut path = self.config.chunks_dir.clone();
for i in 0..self.config.shard_levels as usize {
let start = i * 2;
let end = start + 2;
if end <= hex.len() {
path = path.join(&hex[start..end]);
}
}
path.join(&hex)
}
pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
let path = self.chunk_path(hash);
if path.exists() {
let meta = fs::metadata(&path).await?;
fs::remove_file(&path).await?;
self.index.remove(hash.0.as_slice())?;
self.current_size.fetch_sub(meta.len(), Ordering::SeqCst);
debug!(hash = %hash, size_bytes = meta.len(), "chunk deleted");
}
Ok(())
}
pub fn current_size(&self) -> u64 {
self.current_size.load(Ordering::SeqCst)
}
pub fn max_size(&self) -> u64 {
self.config.max_size
}
pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
self.index.iter().filter_map(|r| {
r.ok().and_then(|(k, _)| {
if k.len() == 8 {
let mut arr = [0u8; 8];
arr.copy_from_slice(&k);
Some(ChunkHash(arr))
} else {
None
}
})
})
}
pub fn dedup_stats(&self) -> DedupStats {
let chunks_stored = self.index.len() as u64;
let size_bytes = self.current_size();
DedupStats {
chunks_stored,
chunks_unique: chunks_stored,
size_bytes,
size_limit_bytes: self.config.max_size,
}
}
}
#[derive(Debug, Clone)]
pub struct DedupStats {
pub chunks_stored: u64,
pub chunks_unique: u64,
pub size_bytes: u64,
pub size_limit_bytes: u64,
}
impl DedupStats {
pub fn dedup_ratio(&self) -> f64 {
if self.chunks_stored == 0 {
0.0
} else {
1.0 - (self.chunks_unique as f64 / self.chunks_stored as f64)
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum CasError {
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("Sled error: {0}")]
Sled(#[from] sled::Error),
#[error("Chunk not found: {0}")]
NotFound(String),
#[error("Integrity error: expected {expected}, got {actual}")]
IntegrityError { expected: String, actual: String },
#[error("Serialization error: {0}")]
Serialization(String),
#[error("Store full: {current} / {max} bytes")]
StoreFull { current: u64, max: u64 },
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
async fn test_store() -> (CasStore, TempDir) {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 1024 * 1024,
shard_levels: 2,
};
let store = CasStore::open(config).await.unwrap();
(store, dir)
}
#[tokio::test]
async fn test_cas_put_get() {
let (store, _dir) = test_store().await;
let data = b"test chunk data";
let hash = store.put(data).await.unwrap();
let retrieved = store.get(&hash).await.unwrap();
assert_eq!(&retrieved[..], data);
}
#[tokio::test]
async fn test_cas_dedup() {
let (store, _dir) = test_store().await;
let data = b"duplicate data";
let hash1 = store.put(data).await.unwrap();
let hash2 = store.put(data).await.unwrap();
assert_eq!(hash1, hash2);
}
#[tokio::test]
async fn test_cas_exists() {
let (store, _dir) = test_store().await;
let data = b"existence test";
let hash = store.put(data).await.unwrap();
assert!(store.exists(&hash));
let fake_hash = ChunkHash::from_bytes(b"nonexistent");
assert!(!store.exists(&fake_hash));
}
#[tokio::test]
async fn test_cas_delete() {
let (store, _dir) = test_store().await;
let data = b"delete me";
let hash = store.put(data).await.unwrap();
assert!(store.exists(&hash));
store.delete(&hash).await.unwrap();
assert!(!store.exists(&hash));
}
#[tokio::test]
async fn test_cas_integrity() {
let (store, _dir) = test_store().await;
let data = b"integrity test";
let hash = store.put(data).await.unwrap();
let retrieved = store.get(&hash).await.unwrap();
assert_eq!(&retrieved[..], data);
}
#[tokio::test]
async fn test_cas_dedup_stats() {
let (store, _dir) = test_store().await;
store.put(b"chunk1").await.unwrap();
store.put(b"chunk2").await.unwrap();
store.put(b"chunk1").await.unwrap();
let stats = store.dedup_stats();
assert_eq!(stats.chunks_stored, 2);
assert_eq!(stats.chunks_unique, 2);
}
}
crates/musicfs-cas/tests/integration.rs
-203
View File
@@ -1,203 +0,0 @@
use musicfs_cache::TreeBuilder;
use musicfs_cas::{CasConfig, CasStore, ChunkManifest, ChunkRef, ContentFetcher, FileReader};
use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath};
use musicfs_origins::LocalOrigin;
use std::path::PathBuf;
use std::sync::{Arc, RwLock};
use std::time::SystemTime;
use tempfile::TempDir;
fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(vpath),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from("/test"),
},
size,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
}
}
#[tokio::test]
async fn test_cas_and_tree_integration() {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let file_data = b"This is test audio file content for testing.";
let chunk_hash = store.put(file_data).await.unwrap();
let mut builder = TreeBuilder::new();
builder.add_file(&make_file_meta(
1,
"/Artist/Album/Track.flac",
file_data.len() as u64,
));
let _tree = Arc::new(RwLock::new(builder.build()));
let reader = Arc::new(FileReader::new(store.clone()));
reader.register_manifest(ChunkManifest {
file_id: FileId(1),
total_size: file_data.len() as u64,
mtime: 0,
chunks: vec![ChunkRef {
hash: chunk_hash,
offset: 0,
size: file_data.len() as u32,
}],
});
let result = reader
.read(FileId(1), 0, file_data.len() as u32)
.await
.unwrap();
assert_eq!(&result[..], file_data);
}
#[tokio::test]
async fn test_cache_persistence() {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let data = b"persistent data";
let hash = {
let store = CasStore::open(config.clone()).await.unwrap();
store.put(data).await.unwrap()
};
let store = CasStore::open(config).await.unwrap();
let retrieved = store.get(&hash).await.unwrap();
assert_eq!(&retrieved[..], data);
}
#[tokio::test]
async fn test_deduplication() {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = CasStore::open(config).await.unwrap();
let data = b"duplicate this content";
let hash1 = store.put(data).await.unwrap();
let size_after_first = store.current_size();
let hash2 = store.put(data).await.unwrap();
let size_after_second = store.current_size();
assert_eq!(hash1, hash2);
assert_eq!(size_after_first, size_after_second);
}
#[tokio::test]
async fn test_fetcher_cache_miss_flow() {
let origin_dir = TempDir::new().unwrap();
let cas_dir = TempDir::new().unwrap();
let test_content = b"This is audio content that will be fetched on cache miss";
let test_file_path = origin_dir.path().join("test.flac");
std::fs::write(&test_file_path, test_content).unwrap();
let config = CasConfig {
chunks_dir: cas_dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let origin_id = OriginId::from("test-origin");
let origin = Arc::new(LocalOrigin::new(
origin_id.clone(),
origin_dir.path().to_path_buf(),
));
let fetcher = ContentFetcher::new(store.clone());
fetcher.register_origin(origin);
let file_id = FileId(42);
let file_meta = FileMeta {
id: file_id,
virtual_path: VirtualPath::new("/Artist/Album/test.flac"),
real_path: RealPath {
origin_id,
path: PathBuf::from("/test.flac"),
},
size: test_content.len() as u64,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
};
fetcher.register_file(file_meta);
let manifest = fetcher.fetch_file(file_id).await.unwrap();
assert_eq!(manifest.file_id, file_id);
assert_eq!(manifest.total_size, test_content.len() as u64);
assert_eq!(manifest.chunks.len(), 1);
let chunk_data = store.get(&manifest.chunks[0].hash).await.unwrap();
assert_eq!(&chunk_data[..], test_content);
}
#[tokio::test]
async fn test_reader_with_fetcher_integration() {
let origin_dir = TempDir::new().unwrap();
let cas_dir = TempDir::new().unwrap();
let test_content = b"Audio file content for reader integration test";
let test_file_path = origin_dir.path().join("song.flac");
std::fs::write(&test_file_path, test_content).unwrap();
let config = CasConfig {
chunks_dir: cas_dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(CasStore::open(config).await.unwrap());
let origin_id = OriginId::from("local");
let origin = Arc::new(LocalOrigin::new(
origin_id.clone(),
origin_dir.path().to_path_buf(),
));
let fetcher = ContentFetcher::new(store.clone());
fetcher.register_origin(origin);
let file_id = FileId(100);
let file_meta = FileMeta {
id: file_id,
virtual_path: VirtualPath::new("/Test/song.flac"),
real_path: RealPath {
origin_id,
path: PathBuf::from("/song.flac"),
},
size: test_content.len() as u64,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
};
fetcher.register_file(file_meta);
let reader = FileReader::with_fetcher(store, Arc::new(fetcher));
let result = reader
.read(file_id, 0, test_content.len() as u32)
.await
.unwrap();
assert_eq!(&result[..], test_content);
let result2 = reader.read(file_id, 0, 10).await.unwrap();
assert_eq!(&result2[..], &test_content[..10]);
}
crates/musicfs-cli/Cargo.toml
-37
View File
@@ -1,37 +0,0 @@
[package]
name = "musicfs-cli"
version.workspace = true
edition.workspace = true
[[bin]]
name = "musicfs"
path = "src/main.rs"
[dependencies]
musicfs-core.path = "../musicfs-core"
musicfs-origins.path = "../musicfs-origins"
musicfs-cache.path = "../musicfs-cache"
musicfs-cas.path = "../musicfs-cas"
musicfs-fuse.path = "../musicfs-fuse"
musicfs-metadata.path = "../musicfs-metadata"
musicfs-grpc.path = "../musicfs-grpc"
clap.workspace = true
tokio.workspace = true
tokio-util.workspace = true
tokio-stream.workspace = true
tonic.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
tracing-appender.workspace = true
anyhow.workspace = true
dirs.workspace = true
toml.workspace = true
parking_lot.workspace = true
libc.workspace = true
serde.workspace = true
serde_json.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
tracing-journald.workspace = true
sd-notify.workspace = true
crates/musicfs-cli/src/lib.rs
-1
View File
@@ -1 +0,0 @@
#![allow(dead_code)]
crates/musicfs-cli/src/main.rs
-1088
View File
File diff suppressed because it is too large Load Diff
crates/musicfs-cli/src/metadata.rs
-638
View File
@@ -1,638 +0,0 @@
//! CLI subcommands for metadata overlay management.
use anyhow::{Context, Result};
use clap::Subcommand;
use musicfs_grpc::proto::musicfs::v1::{
metadata_service_client::MetadataServiceClient, ClearOverlayRequest, GetMetadataRequest,
ImportMetadataRequest, UpdateMetadataRequest,
};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use tokio_stream::StreamExt;
use tonic::transport::Channel;
use tracing::{debug, info};
/// Metadata overlay management subcommands.
#[derive(Subcommand)]
pub enum MetadataCommand {
/// Get metadata for a file (prints as JSON)
Get {
/// Virtual path of the file
path: String,
/// Print only a specific field
#[arg(long)]
field: Option<String>,
},
/// Set metadata fields for a file
Set {
/// Virtual path of the file
path: String,
/// Track title
#[arg(long)]
title: Option<String>,
/// Artist name
#[arg(long)]
artist: Option<String>,
/// Album name
#[arg(long)]
album: Option<String>,
/// Album artist
#[arg(long)]
album_artist: Option<String>,
/// Track number
#[arg(long)]
track: Option<u32>,
/// Disc number
#[arg(long)]
disc: Option<u32>,
/// Genre
#[arg(long)]
genre: Option<String>,
/// Date (YYYY-MM-DD or YYYY)
#[arg(long)]
date: Option<String>,
/// Composer
#[arg(long)]
composer: Option<String>,
/// Comment
#[arg(long)]
comment: Option<String>,
/// Set metadata from JSON string
#[arg(long, conflicts_with_all = ["title", "artist", "album", "album_artist", "track", "disc", "genre", "date", "composer", "comment"])]
json: Option<String>,
},
/// Clear metadata overlay (revert to original)
Clear {
/// Virtual path of the file
path: String,
},
/// Show difference between current and original metadata
Diff {
/// Virtual path of the file
path: String,
},
/// Import metadata from CSV or JSON file
Import {
/// Import file path
file: PathBuf,
/// File format (csv or json, auto-detected if not specified)
#[arg(long)]
format: Option<String>,
},
/// Export metadata to file
Export {
/// Output file path
#[arg(long, short)]
output: PathBuf,
/// Filter by search query
#[arg(long)]
query: Option<String>,
/// Output format (csv or json, auto-detected from extension)
#[arg(long)]
format: Option<String>,
},
}
/// Metadata fields for JSON serialization.
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct MetadataFields {
#[serde(skip_serializing_if = "Option::is_none")]
pub file_id: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub artist: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub album: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub album_artist: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub year: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub track: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub disc: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub genre: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub bitrate: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub track_total: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub disc_total: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub date: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub composer: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub lyrics: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub copyright: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub compilation: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub artist_sort: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub album_artist_sort: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub album_sort: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub title_sort: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mb_recording_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mb_album_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mb_artist_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mb_album_artist_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mb_release_group_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub replaygain_track_gain: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub replaygain_track_peak: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub replaygain_album_gain: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub replaygain_album_peak: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub channels: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub bits_per_sample: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub encoder: Option<String>,
#[serde(skip_serializing_if = "HashMap::is_empty", default)]
pub custom_tags: HashMap<String, String>,
}
/// Execute a metadata subcommand.
pub async fn run_metadata(command: MetadataCommand, endpoint: &str) -> Result<()> {
match command {
MetadataCommand::Get { path, field } => run_get(endpoint, &path, field.as_deref()).await,
MetadataCommand::Set {
path,
title,
artist,
album,
album_artist,
track,
disc,
genre,
date,
composer,
comment,
json,
} => {
run_set(
endpoint,
&path,
title,
artist,
album,
album_artist,
track,
disc,
genre,
date,
composer,
comment,
json,
)
.await
}
MetadataCommand::Clear { path } => run_clear(endpoint, &path).await,
MetadataCommand::Diff { path } => run_diff(endpoint, &path).await,
MetadataCommand::Import { file, format } => run_import(endpoint, &file, format).await,
MetadataCommand::Export {
output,
query,
format,
} => run_export(endpoint, &output, query, format).await,
}
}
async fn connect(endpoint: &str) -> Result<MetadataServiceClient<Channel>> {
MetadataServiceClient::connect(endpoint.to_string())
.await
.context("Failed to connect to gRPC server")
}
async fn run_get(endpoint: &str, path: &str, field: Option<&str>) -> Result<()> {
let mut client = connect(endpoint).await?;
let response = client
.get_metadata(GetMetadataRequest {
virtual_path: path.to_string(),
})
.await
.context("GetMetadata RPC failed")?;
let meta = response.into_inner();
let fields = MetadataFields {
file_id: Some(meta.file_id),
title: meta.title,
artist: meta.artist,
album: meta.album,
album_artist: meta.album_artist,
year: meta.year,
track: meta.track,
disc: meta.disc,
genre: meta.genre,
format: meta.format,
duration_ms: meta.duration_ms,
bitrate: meta.bitrate,
track_total: meta.track_total,
disc_total: meta.disc_total,
date: meta.date,
composer: meta.composer,
comment: meta.comment,
lyrics: meta.lyrics,
copyright: meta.copyright,
compilation: meta.compilation,
artist_sort: meta.artist_sort,
album_artist_sort: meta.album_artist_sort,
album_sort: meta.album_sort,
title_sort: meta.title_sort,
mb_recording_id: meta.mb_recording_id,
mb_album_id: meta.mb_album_id,
mb_artist_id: meta.mb_artist_id,
mb_album_artist_id: meta.mb_album_artist_id,
mb_release_group_id: meta.mb_release_group_id,
replaygain_track_gain: meta.replaygain_track_gain,
replaygain_track_peak: meta.replaygain_track_peak,
replaygain_album_gain: meta.replaygain_album_gain,
replaygain_album_peak: meta.replaygain_album_peak,
channels: meta.channels,
bits_per_sample: meta.bits_per_sample,
encoder: meta.encoder,
custom_tags: meta.custom_tags,
};
if let Some(field_name) = field {
let value = get_field_value(&fields, field_name)?;
println!("{}", value);
} else {
let json = serde_json::to_string_pretty(&fields)?;
println!("{}", json);
}
Ok(())
}
fn get_field_value(fields: &MetadataFields, field_name: &str) -> Result<String> {
let value = match field_name {
"file_id" => fields.file_id.map(|v| v.to_string()),
"title" => fields.title.clone(),
"artist" => fields.artist.clone(),
"album" => fields.album.clone(),
"album_artist" => fields.album_artist.clone(),
"year" => fields.year.map(|v| v.to_string()),
"track" => fields.track.map(|v| v.to_string()),
"disc" => fields.disc.map(|v| v.to_string()),
"genre" => fields.genre.clone(),
"format" => fields.format.clone(),
"duration_ms" => fields.duration_ms.map(|v| v.to_string()),
"bitrate" => fields.bitrate.map(|v| v.to_string()),
"track_total" => fields.track_total.map(|v| v.to_string()),
"disc_total" => fields.disc_total.map(|v| v.to_string()),
"date" => fields.date.clone(),
"composer" => fields.composer.clone(),
"comment" => fields.comment.clone(),
"lyrics" => fields.lyrics.clone(),
"copyright" => fields.copyright.clone(),
"compilation" => fields.compilation.map(|v| v.to_string()),
"artist_sort" => fields.artist_sort.clone(),
"album_artist_sort" => fields.album_artist_sort.clone(),
"album_sort" => fields.album_sort.clone(),
"title_sort" => fields.title_sort.clone(),
"mb_recording_id" => fields.mb_recording_id.clone(),
"mb_album_id" => fields.mb_album_id.clone(),
"mb_artist_id" => fields.mb_artist_id.clone(),
"mb_album_artist_id" => fields.mb_album_artist_id.clone(),
"mb_release_group_id" => fields.mb_release_group_id.clone(),
"replaygain_track_gain" => fields.replaygain_track_gain.map(|v| v.to_string()),
"replaygain_track_peak" => fields.replaygain_track_peak.map(|v| v.to_string()),
"replaygain_album_gain" => fields.replaygain_album_gain.map(|v| v.to_string()),
"replaygain_album_peak" => fields.replaygain_album_peak.map(|v| v.to_string()),
"channels" => fields.channels.map(|v| v.to_string()),
"bits_per_sample" => fields.bits_per_sample.map(|v| v.to_string()),
"encoder" => fields.encoder.clone(),
_ => return Err(anyhow::anyhow!("Unknown field: {}", field_name)),
};
Ok(value.unwrap_or_else(|| "null".to_string()))
}
#[allow(clippy::too_many_arguments)]
async fn run_set(
endpoint: &str,
path: &str,
title: Option<String>,
artist: Option<String>,
album: Option<String>,
album_artist: Option<String>,
track: Option<u32>,
disc: Option<u32>,
genre: Option<String>,
date: Option<String>,
composer: Option<String>,
comment: Option<String>,
json: Option<String>,
) -> Result<()> {
let mut client = connect(endpoint).await?;
let get_response = client
.get_metadata(GetMetadataRequest {
virtual_path: path.to_string(),
})
.await
.context("Failed to get file metadata")?;
let file_id = get_response.into_inner().file_id;
let request = if let Some(json_str) = json {
let fields: MetadataFields =
serde_json::from_str(&json_str).context("Failed to parse JSON metadata")?;
UpdateMetadataRequest {
file_id,
title: fields.title,
artist: fields.artist,
album: fields.album,
album_artist: fields.album_artist,
track_number: fields.track,
disc_number: fields.disc,
genre: fields.genre,
date: fields.date,
composer: fields.composer,
comment: fields.comment,
lyrics: fields.lyrics,
copyright: fields.copyright,
compilation: fields.compilation,
artist_sort: fields.artist_sort,
album_artist_sort: fields.album_artist_sort,
album_sort: fields.album_sort,
title_sort: fields.title_sort,
mb_recording_id: fields.mb_recording_id,
mb_album_id: fields.mb_album_id,
mb_artist_id: fields.mb_artist_id,
replaygain_track_gain: fields.replaygain_track_gain,
replaygain_track_peak: fields.replaygain_track_peak,
replaygain_album_gain: fields.replaygain_album_gain,
replaygain_album_peak: fields.replaygain_album_peak,
label: None,
album_type: None,
cover_url: None,
custom_tags: fields.custom_tags,
}
} else {
UpdateMetadataRequest {
file_id,
title,
artist,
album,
album_artist,
track_number: track,
disc_number: disc,
genre,
date,
composer,
comment,
lyrics: None,
copyright: None,
compilation: None,
artist_sort: None,
album_artist_sort: None,
album_sort: None,
title_sort: None,
mb_recording_id: None,
mb_album_id: None,
mb_artist_id: None,
replaygain_track_gain: None,
replaygain_track_peak: None,
replaygain_album_gain: None,
replaygain_album_peak: None,
label: None,
album_type: None,
cover_url: None,
custom_tags: HashMap::new(),
}
};
let response = client
.update_metadata(request)
.await
.context("UpdateMetadata RPC failed")?;
let result = response.into_inner();
if result.success {
info!(file_id = result.file_id, "Metadata updated successfully");
println!("Metadata updated for file_id={}", result.file_id);
} else {
let msg = result
.error_message
.unwrap_or_else(|| "Unknown error".to_string());
anyhow::bail!("Failed to update metadata: {}", msg);
}
Ok(())
}
async fn run_clear(endpoint: &str, path: &str) -> Result<()> {
let mut client = connect(endpoint).await?;
let get_response = client
.get_metadata(GetMetadataRequest {
virtual_path: path.to_string(),
})
.await
.context("Failed to get file metadata")?;
let file_id = get_response.into_inner().file_id;
let response = client
.clear_overlay(ClearOverlayRequest { file_id })
.await
.context("ClearOverlay RPC failed")?;
let result = response.into_inner();
if result.success {
info!(file_id = result.file_id, "Overlay cleared successfully");
println!("Metadata overlay cleared for file_id={}", result.file_id);
} else {
let msg = result
.error_message
.unwrap_or_else(|| "Unknown error".to_string());
anyhow::bail!("Failed to clear overlay: {}", msg);
}
Ok(())
}
async fn run_diff(endpoint: &str, path: &str) -> Result<()> {
let mut client = connect(endpoint).await?;
let response = client
.get_metadata(GetMetadataRequest {
virtual_path: path.to_string(),
})
.await
.context("GetMetadata RPC failed")?;
let meta = response.into_inner();
debug!(file_id = meta.file_id, "Retrieved metadata for diff");
println!("Current metadata for: {}", path);
println!("---");
let fields = MetadataFields {
file_id: Some(meta.file_id),
title: meta.title,
artist: meta.artist,
album: meta.album,
album_artist: meta.album_artist,
year: meta.year,
track: meta.track,
disc: meta.disc,
genre: meta.genre,
format: meta.format,
duration_ms: meta.duration_ms,
bitrate: meta.bitrate,
track_total: meta.track_total,
disc_total: meta.disc_total,
date: meta.date,
composer: meta.composer,
comment: meta.comment,
lyrics: meta.lyrics,
copyright: meta.copyright,
compilation: meta.compilation,
artist_sort: meta.artist_sort,
album_artist_sort: meta.album_artist_sort,
album_sort: meta.album_sort,
title_sort: meta.title_sort,
mb_recording_id: meta.mb_recording_id,
mb_album_id: meta.mb_album_id,
mb_artist_id: meta.mb_artist_id,
mb_album_artist_id: meta.mb_album_artist_id,
mb_release_group_id: meta.mb_release_group_id,
replaygain_track_gain: meta.replaygain_track_gain,
replaygain_track_peak: meta.replaygain_track_peak,
replaygain_album_gain: meta.replaygain_album_gain,
replaygain_album_peak: meta.replaygain_album_peak,
channels: meta.channels,
bits_per_sample: meta.bits_per_sample,
encoder: meta.encoder,
custom_tags: meta.custom_tags,
};
let json = serde_json::to_string_pretty(&fields)?;
println!("{}", json);
println!("---");
println!("Note: Original metadata comparison requires re-parsing the source file.");
println!("Use 'musicfs metadata clear <path>' to revert to original metadata.");
Ok(())
}
async fn run_import(endpoint: &str, file: &PathBuf, format: Option<String>) -> Result<()> {
let mut client = connect(endpoint).await?;
let file_format = format.or_else(|| {
file.extension()
.and_then(|e| e.to_str())
.map(|s| s.to_lowercase())
});
let source_path = file
.canonicalize()
.unwrap_or_else(|_| file.clone())
.to_string_lossy()
.to_string();
info!(source_path = %source_path, format = ?file_format, "Starting metadata import");
let response = client
.import_metadata(ImportMetadataRequest {
source_path,
format: file_format,
})
.await
.context("ImportMetadata RPC failed")?;
let mut stream = response.into_inner();
let mut last_imported = 0u32;
let mut last_total = 0u32;
let mut errors = Vec::new();
while let Some(progress) = stream.next().await {
let progress = progress.context("Stream error")?;
last_imported = progress.imported;
last_total = progress.total;
if let Some(ref err) = progress.error_message {
let file = progress.current_file.as_deref().unwrap_or("unknown");
errors.push(format!("{}: {}", file, err));
}
if let Some(ref current) = progress.current_file {
print!(
"\rImporting: {}/{} - {}",
progress.imported, progress.total, current
);
std::io::Write::flush(&mut std::io::stdout())?;
}
}
println!();
println!(
"Import complete: {}/{} files imported",
last_imported, last_total
);
if !errors.is_empty() {
println!("\nErrors ({}):", errors.len());
for err in errors.iter().take(10) {
println!(" - {}", err);
}
if errors.len() > 10 {
println!(" ... and {} more", errors.len() - 10);
}
}
Ok(())
}
async fn run_export(
_endpoint: &str,
output: &PathBuf,
query: Option<String>,
format: Option<String>,
) -> Result<()> {
let output_format = format.or_else(|| {
output
.extension()
.and_then(|e| e.to_str())
.map(|s| s.to_lowercase())
});
println!("Export metadata to: {}", output.display());
if let Some(ref q) = query {
println!("Filter query: {}", q);
}
println!("Format: {}", output_format.as_deref().unwrap_or("json"));
println!();
println!("Note: Export requires file listing capability.");
println!("This feature requires integration with the Search service.");
println!(
"Use 'musicfs search <query>' to find files, then 'musicfs metadata get <path>' for each."
);
Ok(())
}
crates/musicfs-core/Cargo.toml
-18
View File
@@ -1,18 +0,0 @@
[package]
name = "musicfs-core"
version.workspace = true
edition.workspace = true
[dependencies]
thiserror.workspace = true
serde.workspace = true
serde_json.workspace = true
toml.workspace = true
tokio = { workspace = true, features = ["sync"] }
tracing.workspace = true
xxhash-rust.workspace = true
hex.workspace = true
parking_lot.workspace = true
[dev-dependencies]
tempfile.workspace = true
crates/musicfs-core/src/config.rs
-239
View File
@@ -1,239 +0,0 @@
use crate::OriginId;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Config {
pub mount_point: PathBuf,
pub cache_dir: PathBuf,
pub origins: Vec<OriginConfig>,
#[serde(default)]
pub cache: CacheConfig,
#[serde(default)]
pub health: HealthConfig,
#[serde(default)]
pub logging: LoggingConfig,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OriginConfig {
pub id: String,
pub origin_type: OriginType,
pub priority: u8,
#[serde(default = "default_enabled")]
pub enabled: bool,
#[serde(flatten)]
pub settings: HashMap<String, toml::Value>,
}
fn default_enabled() -> bool {
true
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "lowercase")]
pub enum OriginType {
Local,
Nfs,
Smb,
S3,
Sftp,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CacheConfig {
#[serde(default = "default_metadata_cache_mb")]
pub metadata_cache_mb: u64,
#[serde(default = "default_content_cache_gb")]
pub content_cache_gb: u64,
}
impl Default for CacheConfig {
fn default() -> Self {
Self {
metadata_cache_mb: default_metadata_cache_mb(),
content_cache_gb: default_content_cache_gb(),
}
}
}
fn default_metadata_cache_mb() -> u64 {
100
}
fn default_content_cache_gb() -> u64 {
10
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthConfig {
#[serde(default = "default_check_interval_secs")]
pub check_interval_secs: u64,
#[serde(default = "default_timeout_ms")]
pub timeout_ms: u64,
#[serde(default = "default_unhealthy_threshold")]
pub unhealthy_threshold: u32,
#[serde(default)]
pub per_origin_thresholds: HashMap<OriginType, u32>,
}
impl Default for HealthConfig {
fn default() -> Self {
let mut per_origin = HashMap::new();
per_origin.insert(OriginType::Local, 1);
per_origin.insert(OriginType::Nfs, 3);
per_origin.insert(OriginType::Smb, 3);
per_origin.insert(OriginType::S3, 3);
per_origin.insert(OriginType::Sftp, 3);
Self {
check_interval_secs: default_check_interval_secs(),
timeout_ms: default_timeout_ms(),
unhealthy_threshold: default_unhealthy_threshold(),
per_origin_thresholds: per_origin,
}
}
}
impl HealthConfig {
pub fn threshold_for(&self, origin_type: OriginType) -> u32 {
self.per_origin_thresholds
.get(&origin_type)
.copied()
.unwrap_or(self.unhealthy_threshold)
}
}
fn default_check_interval_secs() -> u64 {
30
}
fn default_timeout_ms() -> u64 {
5000
}
fn default_unhealthy_threshold() -> u32 {
3
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoggingConfig {
#[serde(default = "default_log_dir")]
pub log_dir: PathBuf,
#[serde(default)]
pub json_output: bool,
#[serde(default = "default_true")]
pub journald: bool,
#[serde(default = "default_log_level")]
pub level: String,
#[serde(default = "default_sample_rate")]
pub trace_sample_rate: f32,
}
impl Default for LoggingConfig {
fn default() -> Self {
Self {
log_dir: default_log_dir(),
json_output: false,
journald: true,
level: default_log_level(),
trace_sample_rate: default_sample_rate(),
}
}
}
fn default_log_dir() -> PathBuf {
PathBuf::from("/var/log/musicfs")
}
fn default_log_level() -> String {
"musicfs=info,warn".to_string()
}
fn default_true() -> bool {
true
}
fn default_sample_rate() -> f32 {
1.0
}
impl Config {
pub fn from_file(path: &std::path::Path) -> Result<Self, ConfigError> {
let content =
std::fs::read_to_string(path).map_err(|e| ConfigError::Read(e.to_string()))?;
toml::from_str(&content).map_err(|e| ConfigError::Parse(e.to_string()))
}
pub fn origin_id(&self, id: &str) -> Option<OriginId> {
self.origins
.iter()
.find(|o| o.id == id)
.map(|_| OriginId::from(id))
}
}
#[derive(Debug, thiserror::Error)]
pub enum ConfigError {
#[error("Failed to read config: {0}")]
Read(String),
#[error("Failed to parse config: {0}")]
Parse(String),
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_config() {
let toml = r#"
mount_point = "/mnt/music"
cache_dir = "/home/user/.cache/musicfs"
[[origins]]
id = "local"
origin_type = "local"
priority = 1
path = "/mnt/nas/music"
[[origins]]
id = "backup"
origin_type = "s3"
priority = 2
bucket = "music-backup"
region = "us-east-1"
"#;
let config: Config = toml::from_str(toml).unwrap();
assert_eq!(config.origins.len(), 2);
assert_eq!(config.origins[0].priority, 1);
assert_eq!(config.origins[1].origin_type, OriginType::S3);
}
#[test]
fn test_default_health_thresholds() {
let health = HealthConfig::default();
assert_eq!(health.threshold_for(OriginType::Local), 1);
assert_eq!(health.threshold_for(OriginType::Sftp), 3);
}
#[test]
fn test_cache_defaults() {
let cache = CacheConfig::default();
assert_eq!(cache.metadata_cache_mb, 100);
assert_eq!(cache.content_cache_gb, 10);
}
}
crates/musicfs-core/src/credentials.rs
-284
View File
@@ -1,284 +0,0 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use thiserror::Error;
use tracing::{debug, info, trace, warn};
#[derive(Clone)]
pub struct CredentialStore {
cache: HashMap<String, Credential>,
}
impl std::fmt::Debug for CredentialStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CredentialStore")
.field("cache_keys", &self.cache.keys().collect::<Vec<_>>())
.finish()
}
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum Credential {
Basic {
username: String,
#[serde(skip_serializing)]
password: String,
},
AwsKey {
access_key_id: String,
#[serde(skip_serializing)]
secret_access_key: String,
session_token: Option<String>,
region: String,
},
SshKey {
username: String,
private_key_path: PathBuf,
#[serde(skip_serializing)]
passphrase: Option<String>,
},
EnvVar {
var_name: String,
},
}
impl std::fmt::Debug for Credential {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Basic { username, .. } => f
.debug_struct("Basic")
.field("username", username)
.field("password", &"[REDACTED]")
.finish(),
Self::AwsKey {
access_key_id,
session_token,
region,
..
} => {
let key_preview = if access_key_id.len() > 4 {
format!("{}...", &access_key_id[..4])
} else {
"****".to_string()
};
let token_display = if session_token.is_some() {
"[REDACTED]"
} else {
"None"
};
f.debug_struct("AwsKey")
.field("access_key_id", &key_preview)
.field("secret_access_key", &"[REDACTED]")
.field("session_token", &token_display)
.field("region", region)
.finish()
}
Self::SshKey {
username,
private_key_path,
..
} => f
.debug_struct("SshKey")
.field("username", username)
.field("private_key_path", private_key_path)
.field("passphrase", &"[REDACTED]")
.finish(),
Self::EnvVar { var_name } => f
.debug_struct("EnvVar")
.field("var_name", var_name)
.finish(),
}
}
}
impl CredentialStore {
pub fn new() -> Self {
Self {
cache: HashMap::new(),
}
}
pub fn load(
&mut self,
origin_id: &str,
config: &CredentialConfig,
) -> Result<Credential, CredentialError> {
debug!(origin_id = %origin_id, "Loading credentials");
if let Some(cred) = self.cache.get(origin_id) {
trace!(origin_id = %origin_id, "Credential cache hit");
return Ok(cred.clone());
}
let cred = match config {
CredentialConfig::Environment { prefix } => {
trace!(origin_id = %origin_id, prefix = %prefix, "Loading from environment");
self.load_from_env(prefix)?
}
CredentialConfig::File { path } => {
trace!(origin_id = %origin_id, path = ?path, "Loading from file");
self.load_from_file(path)?
}
CredentialConfig::Inline(cred) => {
trace!(origin_id = %origin_id, "Using inline credential");
cred.clone()
}
};
let cred_type = match &cred {
Credential::Basic { .. } => "Basic",
Credential::AwsKey { .. } => "AwsKey",
Credential::SshKey { .. } => "SshKey",
Credential::EnvVar { .. } => "EnvVar",
};
info!(origin_id = %origin_id, cred_type = %cred_type, "Credential loaded");
self.cache.insert(origin_id.to_string(), cred.clone());
Ok(cred)
}
fn load_from_env(&self, prefix: &str) -> Result<Credential, CredentialError> {
if let (Ok(key), Ok(secret)) = (
std::env::var(format!("{}_ACCESS_KEY_ID", prefix)),
std::env::var(format!("{}_SECRET_ACCESS_KEY", prefix)),
) {
return Ok(Credential::AwsKey {
access_key_id: key,
secret_access_key: secret,
session_token: std::env::var(format!("{}_SESSION_TOKEN", prefix)).ok(),
region: std::env::var(format!("{}_REGION", prefix))
.unwrap_or_else(|_| "us-east-1".to_string()),
});
}
if let (Ok(user), Ok(pass)) = (
std::env::var(format!("{}_USERNAME", prefix)),
std::env::var(format!("{}_PASSWORD", prefix)),
) {
return Ok(Credential::Basic {
username: user,
password: pass,
});
}
warn!(prefix = %prefix, "No credentials found in environment");
Err(CredentialError::NotFound(format!(
"No credentials found with prefix {}",
prefix
)))
}
fn load_from_file(&self, path: &PathBuf) -> Result<Credential, CredentialError> {
let content =
std::fs::read_to_string(path).map_err(|e| CredentialError::FileRead(e.to_string()))?;
if path.extension().map(|e| e == "json").unwrap_or(false) {
serde_json::from_str(&content).map_err(|e| CredentialError::Parse(e.to_string()))
} else {
toml::from_str(&content).map_err(|e| CredentialError::Parse(e.to_string()))
}
}
pub fn clear(&mut self) {
self.cache.clear();
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "source")]
pub enum CredentialConfig {
Environment { prefix: String },
File { path: PathBuf },
Inline(Credential),
}
#[derive(Debug, Error)]
pub enum CredentialError {
#[error("Credential not found: {0}")]
NotFound(String),
#[error("Failed to read credential file: {0}")]
FileRead(String),
#[error("Failed to parse credential: {0}")]
Parse(String),
}
impl Default for CredentialStore {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_credential_debug_redacted() {
let cred = Credential::Basic {
username: "user".to_string(),
password: "secret123".to_string(),
};
let debug_output = format!("{:?}", cred);
assert!(debug_output.contains("user"));
assert!(!debug_output.contains("secret123"));
assert!(debug_output.contains("[REDACTED]"));
}
#[test]
fn test_aws_credential_debug_redacted() {
let cred = Credential::AwsKey {
access_key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
session_token: None,
region: "us-east-1".to_string(),
};
let debug_output = format!("{:?}", cred);
assert!(debug_output.contains("AKIA..."));
assert!(!debug_output.contains("wJalrXUtnFEMI"));
assert!(debug_output.contains("[REDACTED]"));
}
#[test]
fn test_credential_store_debug() {
let mut store = CredentialStore::new();
store.cache.insert(
"test".to_string(),
Credential::Basic {
username: "user".to_string(),
password: "secret".to_string(),
},
);
let debug_output = format!("{:?}", store);
assert!(debug_output.contains("test"));
assert!(!debug_output.contains("secret"));
}
#[test]
fn test_load_from_env() {
std::env::set_var("TEST_ORIGIN_USERNAME", "testuser");
std::env::set_var("TEST_ORIGIN_PASSWORD", "testpass");
let store = CredentialStore::new();
let cred = store.load_from_env("TEST_ORIGIN").unwrap();
match cred {
Credential::Basic { username, password } => {
assert_eq!(username, "testuser");
assert_eq!(password, "testpass");
}
_ => panic!("Expected Basic credential"),
}
std::env::remove_var("TEST_ORIGIN_USERNAME");
std::env::remove_var("TEST_ORIGIN_PASSWORD");
}
}
crates/musicfs-core/src/error.rs
-70
View File
@@ -1,70 +0,0 @@
use thiserror::Error;
#[derive(Error, Debug)]
pub enum Error {
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("Origin not found: {0}")]
OriginNotFound(String),
#[error("File not found: {0}")]
FileNotFound(String),
#[error("Path resolution failed: {0}")]
PathResolution(String),
#[error("Cache error: {0}")]
Cache(String),
#[error("Metadata extraction error: {0}")]
Metadata(String),
#[error("Database error: {0}")]
Database(String),
#[error("Database corrupted: {0}")]
DatabaseCorrupted(String),
#[error("NFS stale file handle")]
NfsStaleHandle,
#[error("Operation not permitted (read-only filesystem)")]
ReadOnly,
#[error("No origin available to serve request")]
NoOriginAvailable,
#[error("Maximum retries exceeded")]
MaxRetriesExceeded,
#[error("Origin error: {0}")]
Origin(String),
#[error("S3 error: {0}")]
S3(String),
#[error("SFTP error: {0}")]
Sftp(String),
#[error("Operation timed out: {0}")]
Timeout(String),
#[error("Credential error: {0}")]
Credential(String),
}
pub type Result<T> = std::result::Result<T, Error>;
impl Error {
pub fn is_not_found(&self) -> bool {
matches!(self, Error::FileNotFound(_))
}
pub fn downcast_io(&self) -> Option<&std::io::Error> {
match self {
Error::Io(e) => Some(e),
_ => None,
}
}
}
crates/musicfs-core/src/events.rs
-113
View File
@@ -1,113 +0,0 @@
use crate::types::{FileId, OriginId, VirtualPath};
use tokio::sync::broadcast;
use tracing::{debug, trace};
pub struct EventBus {
sender: broadcast::Sender<Event>,
}
impl EventBus {
pub fn new(capacity: usize) -> Self {
let (sender, _) = broadcast::channel(capacity);
Self { sender }
}
pub fn publish(&self, event: Event) {
trace!(event = ?event, "Publishing event");
let receiver_count = self.sender.receiver_count();
if self.sender.send(event).is_err() && receiver_count > 0 {
debug!(
receiver_count = receiver_count,
"Event dropped, no active receivers"
);
}
}
pub fn subscribe(&self) -> broadcast::Receiver<Event> {
self.sender.subscribe()
}
}
impl Default for EventBus {
fn default() -> Self {
Self::new(1024)
}
}
#[derive(Clone, Debug)]
pub enum Event {
FileAdded {
path: VirtualPath,
origin_id: OriginId,
},
FileRemoved {
path: VirtualPath,
file_id: Option<FileId>,
},
FileModified {
path: VirtualPath,
},
FileAccessed {
file_id: FileId,
path: VirtualPath,
origin_id: OriginId,
offset: u64,
size: u32,
},
OriginConnected {
origin_id: OriginId,
},
OriginDisconnected {
origin_id: OriginId,
},
SyncStarted {
origin_id: OriginId,
},
SyncCompleted {
origin_id: OriginId,
files_changed: u64,
},
CacheEviction {
bytes_freed: u64,
},
AllOriginsUnhealthy {
candidate_count: usize,
},
OriginHealthChanged {
origin_id: OriginId,
healthy: bool,
},
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_event_bus() {
let bus = EventBus::new(16);
let mut rx = bus.subscribe();
bus.publish(Event::SyncStarted {
origin_id: OriginId::from("test"),
});
let event = rx.recv().await.unwrap();
assert!(matches!(event, Event::SyncStarted { .. }));
}
#[tokio::test]
async fn test_event_bus_multiple_subscribers() {
let bus = EventBus::new(16);
let mut rx1 = bus.subscribe();
let mut rx2 = bus.subscribe();
bus.publish(Event::CacheEviction { bytes_freed: 1024 });
let e1 = rx1.recv().await.unwrap();
let e2 = rx2.recv().await.unwrap();
assert!(matches!(e1, Event::CacheEviction { bytes_freed: 1024 }));
assert!(matches!(e2, Event::CacheEviction { bytes_freed: 1024 }));
}
}
crates/musicfs-core/src/lib.rs
-60
View File
@@ -1,60 +0,0 @@
pub mod config;
pub mod credentials;
pub mod error;
pub mod events;
pub mod metrics;
pub mod resolver;
pub mod supervisor;
pub mod types;
pub use config::{
CacheConfig, Config, ConfigError, HealthConfig, LoggingConfig, OriginConfig, OriginType,
};
use std::path::Path;
pub fn sanitize_path(path: &Path) -> String {
if let Ok(home) = std::env::var("HOME") {
path.to_string_lossy().replace(&home, "~")
} else {
path.to_string_lossy().to_string()
}
}
/// Install a custom panic hook that logs panics via tracing before the default behavior.
/// This ensures panics are captured in log files and journald.
pub fn install_panic_hook() {
let default_hook = std::panic::take_hook();
std::panic::set_hook(Box::new(move |info| {
let thread = std::thread::current();
let thread_name = thread.name().unwrap_or("<unnamed>");
let message = if let Some(s) = info.payload().downcast_ref::<&str>() {
(*s).to_string()
} else if let Some(s) = info.payload().downcast_ref::<String>() {
s.clone()
} else {
"unknown panic".to_string()
};
let location = info
.location()
.map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
.unwrap_or_else(|| "unknown location".to_string());
tracing::error!(
thread = thread_name,
location = %location,
"PANIC: {}",
message
);
default_hook(info);
}));
}
pub use credentials::{Credential, CredentialConfig, CredentialError, CredentialStore};
pub use error::{Error, Result};
pub use events::{Event, EventBus};
pub use metrics::{CacheMetrics, FuseOpsMetrics, Metrics, OriginsMetrics};
pub use resolver::{PathResolver, PathTemplate};
pub use types::*;
crates/musicfs-core/src/metrics.rs
-322
View File
@@ -1,322 +0,0 @@
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
#[derive(Default)]
pub struct Metrics {
pub fuse_ops: FuseOpsMetrics,
pub fuse_latency: FuseLatencyMetrics,
pub cache: CacheMetrics,
pub origins: OriginsMetrics,
pub origin_health: OriginHealthMetrics,
start_time: Option<Instant>,
}
impl Metrics {
pub fn new() -> Self {
Self {
start_time: Some(Instant::now()),
..Default::default()
}
}
pub fn uptime_secs(&self) -> u64 {
self.start_time.map(|t| t.elapsed().as_secs()).unwrap_or(0)
}
pub fn to_prometheus(&self) -> String {
let mut output = String::new();
output.push_str(&format!(
"# HELP musicfs_fuse_ops_total Total FUSE operations\n\
# TYPE musicfs_fuse_ops_total counter\n\
musicfs_fuse_ops_total{{op=\"lookup\"}} {}\n\
musicfs_fuse_ops_total{{op=\"getattr\"}} {}\n\
musicfs_fuse_ops_total{{op=\"read\"}} {}\n\
musicfs_fuse_ops_total{{op=\"readdir\"}} {}\n\
musicfs_fuse_ops_total{{op=\"open\"}} {}\n",
self.fuse_ops.lookup.load(Ordering::Relaxed),
self.fuse_ops.getattr.load(Ordering::Relaxed),
self.fuse_ops.read.load(Ordering::Relaxed),
self.fuse_ops.readdir.load(Ordering::Relaxed),
self.fuse_ops.open.load(Ordering::Relaxed),
));
for (op, histogram) in self.fuse_latency.histograms.read().iter() {
let quantiles = histogram.quantiles();
output.push_str(&format!(
"# HELP musicfs_fuse_latency_seconds FUSE operation latency\n\
# TYPE musicfs_fuse_latency_seconds summary\n\
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.5\"}} {:.6}\n\
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.95\"}} {:.6}\n\
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.99\"}} {:.6}\n\
musicfs_fuse_latency_seconds_sum{{op=\"{}\"}} {:.6}\n\
musicfs_fuse_latency_seconds_count{{op=\"{}\"}} {}\n",
op,
quantiles.p50,
op,
quantiles.p95,
op,
quantiles.p99,
op,
histogram.sum_secs(),
op,
histogram.count(),
));
}
output.push_str(&format!(
"# HELP musicfs_cache_hits_total Cache hits\n\
# TYPE musicfs_cache_hits_total counter\n\
musicfs_cache_hits_total {}\n",
self.cache.hits.load(Ordering::Relaxed),
));
output.push_str(&format!(
"# HELP musicfs_cache_misses_total Cache misses\n\
# TYPE musicfs_cache_misses_total counter\n\
musicfs_cache_misses_total {}\n",
self.cache.misses.load(Ordering::Relaxed),
));
output.push_str(&format!(
"# HELP musicfs_cache_size_bytes Current cache size in bytes\n\
# TYPE musicfs_cache_size_bytes gauge\n\
musicfs_cache_size_bytes {}\n",
self.cache.size_bytes.load(Ordering::Relaxed),
));
output.push_str(&format!(
"# HELP musicfs_cache_chunks_total Number of cached chunks\n\
# TYPE musicfs_cache_chunks_total gauge\n\
musicfs_cache_chunks_total {}\n",
self.cache.chunk_count.load(Ordering::Relaxed),
));
output.push_str(
"# HELP musicfs_origin_health Origin health status (1=healthy, 0=unhealthy)\n\
# TYPE musicfs_origin_health gauge\n",
);
for (origin_id, healthy) in self.origin_health.status.read().iter() {
output.push_str(&format!(
"musicfs_origin_health{{origin=\"{}\"}} {}\n",
origin_id,
if *healthy { 1 } else { 0 }
));
}
output.push_str(&format!(
"# HELP musicfs_uptime_seconds Daemon uptime in seconds\n\
# TYPE musicfs_uptime_seconds gauge\n\
musicfs_uptime_seconds {}\n",
self.uptime_secs(),
));
output
}
pub fn hit_ratio(&self) -> f64 {
let hits = self.cache.hits.load(Ordering::Relaxed) as f64;
let misses = self.cache.misses.load(Ordering::Relaxed) as f64;
let total = hits + misses;
if total == 0.0 {
0.0
} else {
hits / total
}
}
}
#[derive(Default)]
pub struct FuseOpsMetrics {
pub lookup: AtomicU64,
pub getattr: AtomicU64,
pub read: AtomicU64,
pub readdir: AtomicU64,
pub open: AtomicU64,
}
impl FuseOpsMetrics {
pub fn record_lookup(&self) {
self.lookup.fetch_add(1, Ordering::Relaxed);
}
pub fn record_getattr(&self) {
self.getattr.fetch_add(1, Ordering::Relaxed);
}
pub fn record_read(&self) {
self.read.fetch_add(1, Ordering::Relaxed);
}
pub fn record_readdir(&self) {
self.readdir.fetch_add(1, Ordering::Relaxed);
}
pub fn record_open(&self) {
self.open.fetch_add(1, Ordering::Relaxed);
}
}
#[derive(Default)]
pub struct CacheMetrics {
pub hits: AtomicU64,
pub misses: AtomicU64,
pub size_bytes: AtomicU64,
pub chunk_count: AtomicU64,
}
impl CacheMetrics {
pub fn record_hit(&self) {
self.hits.fetch_add(1, Ordering::Relaxed);
}
pub fn record_miss(&self) {
self.misses.fetch_add(1, Ordering::Relaxed);
}
pub fn update_size(&self, size: u64) {
self.size_bytes.store(size, Ordering::Relaxed);
}
pub fn update_chunk_count(&self, count: u64) {
self.chunk_count.store(count, Ordering::Relaxed);
}
}
#[derive(Default)]
pub struct OriginsMetrics {
pub healthy_count: AtomicU64,
pub total_count: AtomicU64,
}
impl OriginsMetrics {
pub fn update(&self, healthy: u64, total: u64) {
self.healthy_count.store(healthy, Ordering::Relaxed);
self.total_count.store(total, Ordering::Relaxed);
}
}
#[derive(Default)]
pub struct FuseLatencyMetrics {
pub histograms: RwLock<HashMap<String, LatencyHistogram>>,
}
impl FuseLatencyMetrics {
pub fn record(&self, op: &str, latency_secs: f64) {
let mut histograms = self.histograms.write();
histograms
.entry(op.to_string())
.or_default()
.record(latency_secs);
}
}
#[derive(Default)]
pub struct LatencyHistogram {
samples: Vec<f64>,
sum: f64,
}
impl LatencyHistogram {
pub fn record(&mut self, latency_secs: f64) {
self.samples.push(latency_secs);
self.sum += latency_secs;
if self.samples.len() > 10000 {
self.samples.drain(..5000);
}
}
pub fn quantiles(&self) -> Quantiles {
if self.samples.is_empty() {
return Quantiles::default();
}
let mut sorted = self.samples.clone();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let len = sorted.len();
Quantiles {
p50: sorted[len / 2],
p95: sorted[(len as f64 * 0.95) as usize],
p99: sorted[(len as f64 * 0.99) as usize],
}
}
pub fn sum_secs(&self) -> f64 {
self.sum
}
pub fn count(&self) -> u64 {
self.samples.len() as u64
}
}
#[derive(Default)]
pub struct Quantiles {
pub p50: f64,
pub p95: f64,
pub p99: f64,
}
#[derive(Default)]
pub struct OriginHealthMetrics {
pub status: RwLock<HashMap<String, bool>>,
}
impl OriginHealthMetrics {
pub fn set_health(&self, origin_id: &str, healthy: bool) {
self.status.write().insert(origin_id.to_string(), healthy);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_metrics_new() {
let metrics = Metrics::new();
assert!(metrics.uptime_secs() < 5);
}
#[test]
fn test_fuse_ops_recording() {
let metrics = Metrics::new();
metrics.fuse_ops.record_lookup();
metrics.fuse_ops.record_lookup();
metrics.fuse_ops.record_read();
assert_eq!(metrics.fuse_ops.lookup.load(Ordering::Relaxed), 2);
assert_eq!(metrics.fuse_ops.read.load(Ordering::Relaxed), 1);
}
#[test]
fn test_cache_hit_ratio() {
let metrics = Metrics::new();
metrics.cache.hits.store(8, Ordering::Relaxed);
metrics.cache.misses.store(2, Ordering::Relaxed);
assert!((metrics.hit_ratio() - 0.8).abs() < 0.001);
}
#[test]
fn test_cache_hit_ratio_zero() {
let metrics = Metrics::new();
assert_eq!(metrics.hit_ratio(), 0.0);
}
#[test]
fn test_prometheus_format() {
let metrics = Metrics::new();
metrics.fuse_ops.record_lookup();
metrics.cache.record_hit();
let output = metrics.to_prometheus();
assert!(output.contains("musicfs_fuse_ops_total"));
assert!(output.contains("musicfs_cache_hits_total"));
}
}
crates/musicfs-core/src/resolver.rs
-174
View File
@@ -1,174 +0,0 @@
use crate::{AudioMeta, VirtualPath};
#[derive(Debug, Clone)]
pub struct PathTemplate {
pub pattern: String,
pub fallback_artist: String,
pub fallback_album: String,
pub fallback_title: String,
pub fallback_year: String,
}
impl Default for PathTemplate {
fn default() -> Self {
Self {
pattern: "$artist/$album ($year) [$format_upper]/$track - $title.$format".to_string(),
fallback_artist: "Unknown Artist".to_string(),
fallback_album: "Unknown Album".to_string(),
fallback_title: "Unknown Track".to_string(),
fallback_year: "Unknown".to_string(),
}
}
}
pub struct PathResolver {
template: PathTemplate,
}
impl PathResolver {
pub fn new(template: PathTemplate) -> Self {
Self { template }
}
pub fn resolve(&self, meta: &AudioMeta, extension: &str) -> VirtualPath {
let artist = meta
.artist
.as_deref()
.unwrap_or(&self.template.fallback_artist);
let album = meta
.album
.as_deref()
.unwrap_or(&self.template.fallback_album);
let title = meta
.title
.as_deref()
.unwrap_or(&self.template.fallback_title);
let year = meta
.year
.map(|y| y.to_string())
.unwrap_or_else(|| self.template.fallback_year.clone());
let track = meta.track.unwrap_or(0);
let disc = meta.disc.unwrap_or(1);
let genre = meta.genre.as_deref().unwrap_or("Unknown");
let format = extension.to_lowercase();
let format_upper = extension.to_uppercase();
let artist = sanitize_path_component(artist);
let album = sanitize_path_component(album);
let title = sanitize_path_component(title);
let genre = sanitize_path_component(genre);
let path = self
.template
.pattern
.replace("$artist", &artist)
.replace("$album", &album)
.replace("$title", &title)
.replace("$track", &format!("{:02}", track))
.replace("$disc", &disc.to_string())
.replace("$year", &year)
.replace("$genre", &genre)
.replace("$format_upper", &format_upper)
.replace("$format", &format);
VirtualPath::new(path)
}
}
impl Default for PathResolver {
fn default() -> Self {
Self::new(PathTemplate::default())
}
}
fn sanitize_path_component(s: &str) -> String {
s.chars()
.map(|c| match c {
'/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' | '\0' => '_',
c => c,
})
.collect::<String>()
.trim()
.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::AudioFormat;
#[test]
fn test_resolve_complete_metadata() {
let resolver = PathResolver::default();
let meta = AudioMeta {
artist: Some("Metallica".to_string()),
album: Some("Master of Puppets".to_string()),
title: Some("Battery".to_string()),
track: Some(1),
year: Some(1986),
format: AudioFormat::Flac,
..Default::default()
};
let path = resolver.resolve(&meta, "flac");
assert_eq!(
path.as_str(),
"Metallica/Master of Puppets (1986) [FLAC]/01 - Battery.flac"
);
}
#[test]
fn test_resolve_missing_album() {
let resolver = PathResolver::default();
let meta = AudioMeta {
artist: Some("Artist".to_string()),
title: Some("Track".to_string()),
track: Some(5),
..Default::default()
};
let path = resolver.resolve(&meta, "mp3");
assert_eq!(
path.as_str(),
"Artist/Unknown Album (Unknown) [MP3]/05 - Track.mp3"
);
}
#[test]
fn test_sanitize_special_chars() {
let resolver = PathResolver::default();
let meta = AudioMeta {
artist: Some("AC/DC".to_string()),
album: Some("Who Made Who?".to_string()),
title: Some("Test:Track".to_string()),
track: Some(1),
year: Some(1986),
..Default::default()
};
let path = resolver.resolve(&meta, "flac");
assert!(!path.as_str().contains(':'));
assert!(!path.as_str().contains('?'));
assert!(path.as_str().contains("AC_DC"));
}
#[test]
fn test_custom_template() {
let template = PathTemplate {
pattern: "$genre/$artist - $album/$track $title.$format".to_string(),
..Default::default()
};
let resolver = PathResolver::new(template);
let meta = AudioMeta {
artist: Some("Artist".to_string()),
album: Some("Album".to_string()),
title: Some("Song".to_string()),
genre: Some("Rock".to_string()),
track: Some(3),
..Default::default()
};
let path = resolver.resolve(&meta, "flac");
assert_eq!(path.as_str(), "Rock/Artist - Album/03 Song.flac");
}
}
crates/musicfs-core/src/supervisor.rs
-181
View File
@@ -1,181 +0,0 @@
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::task::JoinHandle;
use tracing::{error, warn};
pub struct TaskSupervisor {
tasks: Arc<RwLock<HashMap<String, TaskEntry>>>,
}
struct TaskEntry {
handle: JoinHandle<()>,
status: TaskStatus,
restart_count: u32,
last_restart: Option<Instant>,
}
#[derive(Debug, Clone)]
pub enum TaskStatus {
Running,
Failed { error: String, at: Instant },
Restarting { attempt: u32 },
Stopped,
}
impl Default for TaskSupervisor {
fn default() -> Self {
Self::new()
}
}
impl TaskSupervisor {
pub fn new() -> Self {
Self {
tasks: Arc::new(RwLock::new(HashMap::new())),
}
}
pub fn spawn_supervised<F>(&self, name: &str, future: F)
where
F: std::future::Future<Output = ()> + Send + 'static,
{
let name_owned = name.to_string();
let handle = tokio::spawn(async move {
future.await;
});
self.tasks.write().insert(
name_owned,
TaskEntry {
handle,
status: TaskStatus::Running,
restart_count: 0,
last_restart: None,
},
);
}
pub fn spawn_critical<F, Fut>(&self, name: &str, factory: F)
where
F: Fn() -> Fut + Send + Sync + 'static,
Fut: std::future::Future<Output = ()> + Send + 'static,
{
let tasks = self.tasks.clone();
let name_owned = name.to_string();
let monitor_handle = tokio::spawn(async move {
let mut restart_count = 0u32;
let max_restarts = 5u32;
let backoff_durations = [
Duration::from_secs(1),
Duration::from_secs(5),
Duration::from_secs(30),
];
loop {
let handle = tokio::spawn(factory());
{
let mut t = tasks.write();
if let Some(entry) = t.get_mut(&name_owned) {
entry.status = TaskStatus::Running;
}
}
match handle.await {
Ok(()) => {
let mut t = tasks.write();
if let Some(entry) = t.get_mut(&name_owned) {
entry.status = TaskStatus::Stopped;
}
break;
}
Err(e) => {
restart_count += 1;
if restart_count > max_restarts {
error!(task = %name_owned, "Task exceeded max restarts ({}), giving up", max_restarts);
let mut t = tasks.write();
if let Some(entry) = t.get_mut(&name_owned) {
entry.status = TaskStatus::Failed {
error: format!("Exceeded max restarts: {}", e),
at: Instant::now(),
};
}
break;
}
let backoff_idx =
(restart_count as usize - 1).min(backoff_durations.len() - 1);
let backoff = backoff_durations[backoff_idx];
warn!(
task = %name_owned,
error = %e,
attempt = restart_count,
backoff_ms = backoff.as_millis() as u64,
"Critical task failed, restarting with backoff"
);
{
let mut t = tasks.write();
if let Some(entry) = t.get_mut(&name_owned) {
entry.status = TaskStatus::Restarting {
attempt: restart_count,
};
entry.restart_count = restart_count;
entry.last_restart = Some(Instant::now());
}
}
tokio::time::sleep(backoff).await;
}
}
}
});
self.tasks.write().insert(
name.to_string(),
TaskEntry {
handle: monitor_handle,
status: TaskStatus::Running,
restart_count: 0,
last_restart: None,
},
);
}
pub fn task_status(&self, name: &str) -> TaskStatus {
let mut tasks = self.tasks.write();
if let Some(entry) = tasks.get_mut(name) {
if entry.handle.is_finished() {
entry.status = TaskStatus::Failed {
error: "Task exited".into(),
at: Instant::now(),
};
}
entry.status.clone()
} else {
TaskStatus::Stopped
}
}
pub fn check_all(&self) -> Vec<(String, TaskStatus)> {
let mut tasks = self.tasks.write();
tasks
.iter_mut()
.map(|(name, entry)| {
if entry.handle.is_finished() {
entry.status = TaskStatus::Failed {
error: "Task exited".into(),
at: Instant::now(),
};
}
(name.clone(), entry.status.clone())
})
.collect()
}
}
crates/musicfs-core/src/types.rs
-223
View File
@@ -1,223 +0,0 @@
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use std::time::SystemTime;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct OriginId(pub String);
impl From<&str> for OriginId {
fn from(s: &str) -> Self {
Self(s.to_string())
}
}
impl std::fmt::Display for OriginId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct FileId(pub i64);
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct VirtualPath(pub PathBuf);
impl VirtualPath {
pub fn new(path: impl Into<PathBuf>) -> Self {
Self(path.into())
}
pub fn as_path(&self) -> &std::path::Path {
&self.0
}
pub fn as_str(&self) -> &str {
self.0.to_str().unwrap_or("")
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RealPath {
pub origin_id: OriginId,
pub path: PathBuf,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct ContentHash(pub [u8; 8]);
impl ContentHash {
pub fn from_bytes(data: &[u8]) -> Self {
use xxhash_rust::xxh64::xxh64;
Self(xxh64(data, 0).to_le_bytes())
}
pub fn to_hex(&self) -> String {
hex::encode(self.0)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct ChunkHash(pub [u8; 8]);
impl ChunkHash {
pub fn from_bytes(data: &[u8]) -> Self {
use xxhash_rust::xxh64::xxh64;
Self(xxh64(data, 0).to_le_bytes())
}
pub fn as_hex(&self) -> String {
hex::encode(self.0)
}
pub fn to_hex(&self) -> String {
self.as_hex()
}
pub fn from_hex(s: &str) -> Option<Self> {
let bytes = hex::decode(s).ok()?;
if bytes.len() != 8 {
return None;
}
let mut arr = [0u8; 8];
arr.copy_from_slice(&bytes);
Some(Self(arr))
}
}
impl std::fmt::Display for ChunkHash {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.as_hex())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
pub enum AudioFormat {
Flac,
Mp3,
Opus,
Vorbis,
Aac,
Alac,
Wav,
#[default]
Unknown,
}
impl AudioFormat {
pub fn from_extension(ext: &str) -> Self {
match ext.to_lowercase().as_str() {
"flac" => Self::Flac,
"mp3" => Self::Mp3,
"opus" => Self::Opus,
"ogg" => Self::Vorbis,
"m4a" | "aac" => Self::Aac,
"wav" => Self::Wav,
_ => Self::Unknown,
}
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AudioMeta {
pub title: Option<String>,
pub artist: Option<String>,
pub album: Option<String>,
pub album_artist: Option<String>,
pub genre: Option<String>,
pub year: Option<u32>,
pub track: Option<u32>,
pub disc: Option<u32>,
pub duration_ms: Option<u64>,
pub bitrate: Option<u32>,
pub sample_rate: Option<u32>,
pub format: AudioFormat,
pub track_total: Option<u32>,
pub disc_total: Option<u32>,
pub date: Option<String>,
pub composer: Option<String>,
pub comment: Option<String>,
pub lyrics: Option<String>,
pub copyright: Option<String>,
pub compilation: Option<bool>,
pub artist_sort: Option<String>,
pub album_artist_sort: Option<String>,
pub album_sort: Option<String>,
pub title_sort: Option<String>,
pub mb_recording_id: Option<String>,
pub mb_album_id: Option<String>,
pub mb_artist_id: Option<String>,
pub mb_album_artist_id: Option<String>,
pub mb_release_group_id: Option<String>,
pub replaygain_track_gain: Option<f32>,
pub replaygain_track_peak: Option<f32>,
pub replaygain_album_gain: Option<f32>,
pub replaygain_album_peak: Option<f32>,
pub channels: Option<u32>,
pub bits_per_sample: Option<u32>,
pub encoder: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileMeta {
pub id: FileId,
pub virtual_path: VirtualPath,
pub real_path: RealPath,
pub size: u64,
pub mtime: SystemTime,
pub content_hash: Option<ContentHash>,
pub audio: Option<AudioMeta>,
}
#[derive(Debug, Clone)]
pub struct DirEntry {
pub name: String,
pub is_dir: bool,
pub size: u64,
pub mtime: SystemTime,
}
#[derive(Debug, Clone)]
pub struct FileStat {
pub size: u64,
pub mtime: SystemTime,
pub is_dir: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum HealthStatus {
Healthy,
Degraded,
Unhealthy,
#[default]
Unknown,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_content_hash() {
let data = b"hello world";
let hash1 = ContentHash::from_bytes(data);
let hash2 = ContentHash::from_bytes(data);
assert_eq!(hash1, hash2);
let hash3 = ContentHash::from_bytes(b"different");
assert_ne!(hash1, hash3);
}
#[test]
fn test_audio_format_from_extension() {
assert_eq!(AudioFormat::from_extension("flac"), AudioFormat::Flac);
assert_eq!(AudioFormat::from_extension("MP3"), AudioFormat::Mp3);
assert_eq!(AudioFormat::from_extension("unknown"), AudioFormat::Unknown);
}
#[test]
fn test_virtual_path() {
let path = VirtualPath::new("/Artist/Album/Track.flac");
assert_eq!(path.as_str(), "/Artist/Album/Track.flac");
}
}
crates/musicfs-fuse/Cargo.toml
-19
View File
@@ -1,19 +0,0 @@
[package]
name = "musicfs-fuse"
version.workspace = true
edition.workspace = true
[dependencies]
musicfs-core = { path = "../musicfs-core" }
musicfs-cache = { path = "../musicfs-cache" }
musicfs-cas = { path = "../musicfs-cas" }
musicfs-search = { path = "../musicfs-search" }
fuser.workspace = true
tokio.workspace = true
tracing.workspace = true
moka.workspace = true
parking_lot.workspace = true
libc = "0.2"
[dev-dependencies]
tempfile.workspace = true
crates/musicfs-fuse/src/filesystem.rs
-954
View File
@@ -1,954 +0,0 @@
use crate::ops::SearchOps;
use fuser::{
FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, ReplyOpen,
Request,
};
use musicfs_cache::{
Database, OverlayError, OverlayReader, RemoveError, RenameError, VirtualNode, VirtualTree,
ROOT_INODE,
};
use musicfs_cas::FileReader;
use musicfs_core::{Result, VirtualPath};
use parking_lot::RwLock;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tokio::runtime::Handle;
use tracing::{debug, info, instrument, trace, warn};
const TTL: Duration = Duration::from_secs(1);
const BLOCK_SIZE: u32 = 512;
const SEARCH_QUERY_INODE_BASE: u64 = 0xFFFF_FFFF_0000_0100;
pub struct MusicFs {
tree: Arc<RwLock<VirtualTree>>,
reader: Option<Arc<FileReader>>,
db: Option<Arc<Database>>,
overlay_reader: Option<Arc<OverlayReader>>,
runtime_handle: Handle,
search_ops: Option<SearchOps>,
query_inodes: RwLock<HashMap<String, u64>>,
inode_queries: RwLock<HashMap<u64, String>>,
next_query_inode: RwLock<u64>,
uid: u32,
gid: u32,
}
impl MusicFs {
pub fn new(tree: Arc<RwLock<VirtualTree>>, runtime_handle: Handle) -> Self {
Self {
tree,
reader: None,
db: None,
overlay_reader: None,
runtime_handle,
search_ops: None,
query_inodes: RwLock::new(HashMap::new()),
inode_queries: RwLock::new(HashMap::new()),
next_query_inode: RwLock::new(SEARCH_QUERY_INODE_BASE),
uid: unsafe { libc::getuid() },
gid: unsafe { libc::getgid() },
}
}
pub fn with_reader(
tree: Arc<RwLock<VirtualTree>>,
reader: Arc<FileReader>,
runtime_handle: Handle,
) -> Self {
Self {
tree,
reader: Some(reader),
db: None,
overlay_reader: None,
runtime_handle,
search_ops: None,
query_inodes: RwLock::new(HashMap::new()),
inode_queries: RwLock::new(HashMap::new()),
next_query_inode: RwLock::new(SEARCH_QUERY_INODE_BASE),
uid: unsafe { libc::getuid() },
gid: unsafe { libc::getgid() },
}
}
pub fn with_db(mut self, db: Arc<Database>) -> Self {
self.db = Some(db);
self
}
pub fn with_overlay(mut self, overlay: Arc<OverlayReader>) -> Self {
self.overlay_reader = Some(overlay);
self
}
pub fn with_search(mut self, search_ops: SearchOps) -> Self {
self.search_ops = Some(search_ops);
self
}
fn resolve_path(&self, parent_inode: u64, name: &OsStr) -> Option<VirtualPath> {
let tree = self.tree.read();
let parent_path = self.inode_to_path_inner(&tree, parent_inode)?;
let name_str = name.to_string_lossy();
let full_path = if parent_path == "/" {
format!("/{}", name_str)
} else {
format!("{}/{}", parent_path, name_str)
};
Some(VirtualPath::new(full_path))
}
fn inode_to_path_inner(&self, tree: &VirtualTree, inode: u64) -> Option<String> {
for (path, &ino) in tree.path_to_inode_iter() {
if ino == inode {
return Some(path.as_str().to_string());
}
}
None
}
fn get_or_create_query_inode(&self, query: &str) -> u64 {
let query_inodes = self.query_inodes.read();
if let Some(&inode) = query_inodes.get(query) {
return inode;
}
drop(query_inodes);
let mut query_inodes = self.query_inodes.write();
let mut inode_queries = self.inode_queries.write();
let mut next_inode = self.next_query_inode.write();
if let Some(&inode) = query_inodes.get(query) {
return inode;
}
let inode = *next_inode;
*next_inode += 1;
query_inodes.insert(query.to_string(), inode);
inode_queries.insert(inode, query.to_string());
inode
}
fn get_query_for_inode(&self, inode: u64) -> Option<String> {
self.inode_queries.read().get(&inode).cloned()
}
pub fn mount(self, mountpoint: &Path) -> Result<()> {
info!("Mounting MusicFS at {:?}", mountpoint);
let options = vec![
fuser::MountOption::FSName("musicfs".to_string()),
fuser::MountOption::AutoUnmount,
fuser::MountOption::AllowOther,
];
fuser::mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
Ok(())
}
pub fn spawn_mount(self, mountpoint: &Path) -> Result<fuser::BackgroundSession> {
info!("Mounting MusicFS at {:?}", mountpoint);
let options = vec![
fuser::MountOption::FSName("musicfs".to_string()),
fuser::MountOption::AutoUnmount,
fuser::MountOption::AllowOther,
];
let session =
fuser::spawn_mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
Ok(session)
}
fn node_to_attr(&self, node: &VirtualNode) -> FileAttr {
match node {
VirtualNode::Directory(dir) => FileAttr {
ino: dir.inode,
size: 0,
blocks: 0,
atime: dir.mtime,
mtime: dir.mtime,
ctime: dir.mtime,
crtime: dir.mtime,
kind: FileType::Directory,
perm: 0o755,
nlink: 2,
uid: self.uid,
gid: self.gid,
rdev: 0,
blksize: BLOCK_SIZE,
flags: 0,
},
VirtualNode::File(file) => FileAttr {
ino: file.inode,
size: file.size,
blocks: (file.size + BLOCK_SIZE as u64 - 1) / BLOCK_SIZE as u64,
atime: file.mtime,
mtime: file.mtime,
ctime: file.mtime,
crtime: file.mtime,
kind: FileType::RegularFile,
perm: 0o644,
nlink: 1,
uid: self.uid,
gid: self.gid,
rdev: 0,
blksize: BLOCK_SIZE,
flags: 0,
},
}
}
}
impl Filesystem for MusicFs {
fn init(
&mut self,
_req: &Request<'_>,
_config: &mut fuser::KernelConfig,
) -> std::result::Result<(), libc::c_int> {
info!("MusicFS initialized");
Ok(())
}
fn destroy(&mut self) {
info!("MusicFS destroyed");
}
#[instrument(level = "debug", skip(self, reply))]
fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) {
let name_str = name.to_string_lossy();
if parent == ROOT_INODE && SearchOps::is_search_dir_name(&name_str) {
trace!(parent, name = %name_str, "search_dir_name matched");
if let Some(ref search_ops) = self.search_ops {
search_ops.lookup_search_dir(reply);
return;
}
}
if parent == SearchOps::search_dir_inode() {
trace!(parent, name = %name_str, "search_dir_inode matched");
if let Some(ref search_ops) = self.search_ops {
let inode = self.get_or_create_query_inode(&name_str);
search_ops.lookup_query_dir(&name_str, inode, reply);
return;
}
}
if let Some(query) = self.get_query_for_inode(parent) {
trace!(parent, name = %name_str, query = %query, "query_inode matched");
if let Some(ref search_ops) = self.search_ops {
let inode = self.get_or_create_query_inode(&format!("{}:{}", query, name_str));
search_ops.lookup_result(inode, reply);
return;
}
}
let tree = self.tree.read();
if let Some(inode) = tree.lookup(parent, name) {
trace!(parent, name = %name_str, ino = inode, "file found in tree");
if let Some(node) = tree.get(inode) {
let attr = self.node_to_attr(node);
reply.entry(&TTL, &attr, 0);
return;
}
}
trace!(parent, name = %name_str, "file not found");
reply.error(libc::ENOENT);
}
#[instrument(level = "debug", skip(self, reply))]
fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) {
if ino == SearchOps::search_dir_inode() {
trace!(ino, "search_dir_inode matched");
if let Some(ref search_ops) = self.search_ops {
search_ops.getattr_search_dir(reply);
return;
}
}
if SearchOps::is_search_inode(ino) {
trace!(ino, "search_inode matched");
if let Some(ref search_ops) = self.search_ops {
search_ops.getattr_result(ino, reply);
return;
}
}
if self.get_query_for_inode(ino).is_some() {
trace!(ino, "query_inode matched");
if let Some(ref search_ops) = self.search_ops {
search_ops.getattr_search_dir(reply);
return;
}
}
let tree = self.tree.read();
if let Some(node) = tree.get(ino) {
trace!(ino, "inode found in tree");
let mut attr = self.node_to_attr(node);
if let VirtualNode::File(file) = node {
if let Some(ref overlay) = self.overlay_reader {
match overlay.estimate_virtual_size(file.file_id) {
Ok(Some(virtual_size)) => {
trace!(ino, file_id = ?file.file_id, virtual_size, "using overlay virtual size");
attr.size = virtual_size;
attr.blocks =
(virtual_size + BLOCK_SIZE as u64 - 1) / BLOCK_SIZE as u64;
}
Ok(None) => {
trace!(ino, file_id = ?file.file_id, "no overlay, using original size");
}
Err(e) => {
warn!(ino, file_id = ?file.file_id, error = %e, "overlay size estimation failed, using original");
}
}
}
}
reply.attr(&TTL, &attr);
} else {
trace!(ino, "inode not found");
reply.error(libc::ENOENT);
}
}
#[instrument(level = "debug", skip(self, reply))]
fn readdir(
&mut self,
_req: &Request,
ino: u64,
_fh: u64,
offset: i64,
mut reply: ReplyDirectory,
) {
if ino == SearchOps::search_dir_inode() {
trace!(ino, offset, "search_dir_inode matched");
if let Some(ref search_ops) = self.search_ops {
search_ops.readdir_search_root(offset, reply);
return;
}
}
if let Some(query) = self.get_query_for_inode(ino) {
trace!(ino, offset, query = %query, "query_inode matched");
if let Some(ref search_ops) = self.search_ops {
search_ops.readdir_query(&query, offset, reply);
return;
}
}
let tree = self.tree.read();
if let Some(children) = tree.readdir(ino) {
trace!(
ino,
offset,
children_count = children.len(),
"directory found"
);
let parent_ino = tree.get_parent(ino).unwrap_or(ROOT_INODE);
let entries: Vec<(u64, FileType, &str)> = vec![
(ino, FileType::Directory, "."),
(parent_ino, FileType::Directory, ".."),
];
let child_entries: Vec<(u64, FileType, String)> = children
.iter()
.map(|(name, child_ino, is_dir)| {
let kind = if *is_dir {
FileType::Directory
} else {
FileType::RegularFile
};
(*child_ino, kind, name.to_string_lossy().to_string())
})
.collect();
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
if reply.add(*inode, (i + 1) as i64, *kind, name) {
reply.ok();
return;
}
}
let base_offset = entries.len();
for (i, (inode, kind, name)) in child_entries.iter().enumerate() {
let entry_offset = base_offset + i;
if entry_offset < offset as usize {
continue;
}
if reply.add(*inode, (entry_offset + 1) as i64, *kind, name) {
break;
}
}
reply.ok();
} else {
trace!(ino, offset, "directory not found");
reply.error(libc::ENOENT);
}
}
#[instrument(level = "debug", skip(self, reply))]
fn open(&mut self, _req: &Request, ino: u64, flags: i32, reply: ReplyOpen) {
let write_flags = libc::O_WRONLY | libc::O_RDWR | libc::O_APPEND | libc::O_TRUNC;
if flags & write_flags != 0 {
trace!(ino, flags, "write flags detected");
reply.error(libc::EROFS);
return;
}
let tree = self.tree.read();
if tree.get(ino).is_some() {
trace!(ino, "inode found");
reply.opened(0, 0);
} else {
trace!(ino, "inode not found");
reply.error(libc::ENOENT);
}
}
#[instrument(level = "debug", skip(self, reply))]
fn read(
&mut self,
_req: &Request,
ino: u64,
_fh: u64,
offset: i64,
size: u32,
_flags: i32,
_lock_owner: Option<u64>,
reply: ReplyData,
) {
let file_id = {
let tree = self.tree.read();
if let Some(VirtualNode::File(file)) = tree.get(ino) {
trace!(ino, "file found in tree");
file.file_id
} else {
trace!(ino, "file not found");
reply.error(libc::ENOENT);
return;
}
};
let handle = self.runtime_handle.clone();
if let Some(ref overlay) = self.overlay_reader {
let overlay = overlay.clone();
let result = std::thread::scope(|_| {
handle.block_on(async {
tokio::time::timeout(
Duration::from_secs(30),
overlay.read(file_id, offset as u64, size),
)
.await
})
});
match result {
Ok(Ok(data)) => {
trace!(
ino,
offset,
size_bytes = size,
bytes_read = data.len(),
"overlay read successful"
);
reply.data(&data);
}
Ok(Err(e)) => {
let errno = match &e {
OverlayError::NotFound(_) => libc::ENOENT,
OverlayError::Database(_) => libc::EIO,
OverlayError::Handler(_) => libc::EIO,
OverlayError::Cas(_) => libc::EIO,
OverlayError::NoHandler(_) => libc::EIO,
};
warn!(ino, offset, size_bytes = size, error = %e, "overlay read failed");
reply.error(errno);
}
Err(_timeout) => {
warn!(
ino,
offset,
size_bytes = size,
"overlay read timed out after 30s"
);
reply.error(libc::EIO);
}
}
} else {
let Some(reader) = &self.reader else {
trace!(ino, "no reader available");
reply.data(&[]);
return;
};
let reader = reader.clone();
let result = std::thread::scope(|_| {
handle.block_on(async {
tokio::time::timeout(
Duration::from_secs(30),
reader.read(file_id, offset as u64, size),
)
.await
})
});
match result {
Ok(Ok(data)) => {
trace!(
ino,
offset,
size_bytes = size,
bytes_read = data.len(),
"read successful"
);
reply.data(&data);
}
Ok(Err(e)) => {
warn!(ino, offset, size_bytes = size, error = %e, "read failed");
reply.error(libc::EIO);
}
Err(_timeout) => {
warn!(ino, offset, size_bytes = size, "read timed out after 30s");
reply.error(libc::EIO);
}
}
}
}
#[instrument(level = "debug", skip(self, reply))]
fn release(
&mut self,
_req: &Request,
ino: u64,
_fh: u64,
_flags: i32,
_lock_owner: Option<u64>,
_flush: bool,
reply: fuser::ReplyEmpty,
) {
trace!(ino, "releasing file handle");
reply.ok();
}
fn readlink(&mut self, _req: &Request, ino: u64, reply: ReplyData) {
debug!("readlink(ino={})", ino);
if SearchOps::is_search_inode(ino) {
if let Some(ref search_ops) = self.search_ops {
search_ops.readlink(ino, reply);
return;
}
}
reply.error(libc::EINVAL);
}
fn write(
&mut self,
_req: &Request,
_ino: u64,
_fh: u64,
_offset: i64,
_data: &[u8],
_write_flags: u32,
_flags: i32,
_lock_owner: Option<u64>,
reply: fuser::ReplyWrite,
) {
reply.error(libc::EROFS);
}
fn mkdir(
&mut self,
_req: &Request,
parent: u64,
name: &OsStr,
_mode: u32,
_umask: u32,
reply: ReplyEntry,
) {
let path = match self.resolve_path(parent, name) {
Some(p) => p,
None => {
reply.error(libc::ENOENT);
return;
}
};
let mut tree = self.tree.write();
match tree.mkdir(&path) {
Ok(inode) => {
if let Some(ref db) = self.db {
if let Err(e) = db.insert_directory(&path) {
warn!(error = %e, "failed to persist directory to database");
}
}
let attr = FileAttr {
ino: inode,
size: 0,
blocks: 0,
atime: SystemTime::now(),
mtime: SystemTime::now(),
ctime: SystemTime::now(),
crtime: SystemTime::now(),
kind: FileType::Directory,
perm: 0o755,
nlink: 2,
uid: self.uid,
gid: self.gid,
rdev: 0,
blksize: BLOCK_SIZE,
flags: 0,
};
debug!(path = %path.as_str(), inode, "mkdir successful");
reply.entry(&TTL, &attr, 0);
}
Err(RenameError::TargetExists) => reply.error(libc::EEXIST),
Err(RenameError::ParentNotFound) => reply.error(libc::ENOENT),
Err(_) => reply.error(libc::EIO),
}
}
fn unlink(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: fuser::ReplyEmpty) {
let path = match self.resolve_path(parent, name) {
Some(p) => p,
None => {
reply.error(libc::ENOENT);
return;
}
};
let (file_id, is_dir) = {
let tree = self.tree.read();
match tree.get_by_path(&path) {
Some(VirtualNode::File(f)) => (Some(f.file_id), false),
Some(VirtualNode::Directory(_)) => (None, true),
None => {
reply.error(libc::ENOENT);
return;
}
}
};
if is_dir {
reply.error(libc::EISDIR);
return;
}
let trash_path = VirtualPath::new(format!("/.trash{}", path.as_str()));
{
let mut tree = self.tree.write();
tree.ensure_trash_dir();
let trash_parent = std::path::Path::new(trash_path.as_str())
.parent()
.map(|p| VirtualPath::new(p.to_string_lossy().into_owned()))
.unwrap_or_else(|| VirtualPath::new("/.trash"));
if let Err(e) = tree.mkdir_p(&trash_parent) {
if !matches!(e, RenameError::TargetExists) {
warn!(error = ?e, "failed to create trash parent directories");
reply.error(libc::EIO);
return;
}
}
if let Err(e) = tree.rename_file(&path, &trash_path) {
match e {
RenameError::SourceNotFound => reply.error(libc::ENOENT),
RenameError::TargetExists => reply.error(libc::EEXIST),
_ => reply.error(libc::EIO),
}
return;
}
}
if let (Some(ref db), Some(id)) = (&self.db, file_id) {
if let Err(e) = db.update_virtual_path(id, &trash_path) {
warn!(error = %e, "failed to update virtual path in database");
}
if let Err(e) = db.mark_trashed(id, &path) {
warn!(error = %e, "failed to mark file as trashed in database");
}
}
debug!(path = %path.as_str(), trash = %trash_path.as_str(), "file moved to trash");
reply.ok();
}
fn rmdir(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: fuser::ReplyEmpty) {
let path = match self.resolve_path(parent, name) {
Some(p) => p,
None => {
reply.error(libc::ENOENT);
return;
}
};
if VirtualTree::is_trash_path(&path) {
reply.error(libc::EPERM);
return;
}
{
let mut tree = self.tree.write();
match tree.remove_directory(&path) {
Ok(()) => {}
Err(RemoveError::NotFound) => {
reply.error(libc::ENOENT);
return;
}
Err(RemoveError::NotEmpty) => {
reply.error(libc::ENOTEMPTY);
return;
}
Err(RemoveError::NotDirectory) => {
reply.error(libc::ENOTDIR);
return;
}
}
}
if let Some(ref db) = self.db {
if let Err(e) = db.delete_directory(&path) {
warn!(error = %e, "failed to delete directory from database");
}
}
debug!(path = %path.as_str(), "directory removed");
reply.ok();
}
fn rename(
&mut self,
_req: &Request,
parent: u64,
name: &OsStr,
newparent: u64,
newname: &OsStr,
_flags: u32,
reply: fuser::ReplyEmpty,
) {
let old_path = match self.resolve_path(parent, name) {
Some(p) => p,
None => {
reply.error(libc::ENOENT);
return;
}
};
let new_path = match self.resolve_path(newparent, newname) {
Some(p) => p,
None => {
reply.error(libc::ENOENT);
return;
}
};
if old_path.as_str() == new_path.as_str() {
reply.ok();
return;
}
let is_dir = {
let tree = self.tree.read();
tree.get_by_path(&old_path)
.map(|n| n.is_dir())
.unwrap_or(false)
};
let result = if is_dir {
let mut tree = self.tree.write();
match tree.rename_directory(&old_path, &new_path) {
Ok(count) => {
if let Some(ref db) = self.db {
let old_prefix = if old_path.as_str().ends_with('/') {
old_path.as_str().to_string()
} else {
format!("{}/", old_path.as_str())
};
let new_prefix = if new_path.as_str().ends_with('/') {
new_path.as_str().to_string()
} else {
format!("{}/", new_path.as_str())
};
if let Err(e) = db.rename_directory(&old_prefix, &new_prefix) {
warn!(error = %e, "failed to persist file path rename to database");
}
if let Err(e) = db.rename_directories(&old_prefix, &new_prefix) {
warn!(error = %e, "failed to persist directory rename to database");
}
}
debug!(old = %old_path.as_str(), new = %new_path.as_str(), count, "directory renamed");
Ok(())
}
Err(e) => Err(e),
}
} else {
let file_id = {
let tree = self.tree.read();
match tree.get_by_path(&old_path) {
Some(VirtualNode::File(f)) => Some(f.file_id),
_ => None,
}
};
let mut tree = self.tree.write();
match tree.rename_file(&old_path, &new_path) {
Ok(()) => {
if let (Some(ref db), Some(id)) = (&self.db, file_id) {
if let Err(e) = db.update_virtual_path(id, &new_path) {
warn!(error = %e, "failed to persist file rename to database");
}
let was_in_trash = VirtualTree::is_trash_path(&old_path);
let now_in_trash = VirtualTree::is_trash_path(&new_path);
if was_in_trash && !now_in_trash {
if let Err(e) = db.unmark_trashed(id) {
warn!(error = %e, "failed to unmark trashed after restore");
}
debug!(path = %new_path.as_str(), "file restored from trash");
}
}
debug!(old = %old_path.as_str(), new = %new_path.as_str(), "file renamed");
Ok(())
}
Err(e) => Err(e),
}
};
match result {
Ok(()) => reply.ok(),
Err(RenameError::SourceNotFound) => reply.error(libc::ENOENT),
Err(RenameError::TargetExists) => reply.error(libc::EEXIST),
Err(RenameError::ParentNotFound) => reply.error(libc::ENOENT),
Err(RenameError::IsDirectory) => reply.error(libc::EISDIR),
Err(RenameError::NotDirectory) => reply.error(libc::ENOTDIR),
}
}
fn create(
&mut self,
_req: &Request,
_parent: u64,
_name: &OsStr,
_mode: u32,
_umask: u32,
_flags: i32,
reply: fuser::ReplyCreate,
) {
reply.error(libc::EROFS);
}
fn setattr(
&mut self,
_req: &Request,
_ino: u64,
_mode: Option<u32>,
_uid: Option<u32>,
_gid: Option<u32>,
_size: Option<u64>,
_atime: Option<fuser::TimeOrNow>,
_mtime: Option<fuser::TimeOrNow>,
_ctime: Option<SystemTime>,
_fh: Option<u64>,
_crtime: Option<SystemTime>,
_chgtime: Option<SystemTime>,
_bkuptime: Option<SystemTime>,
_flags: Option<u32>,
reply: ReplyAttr,
) {
reply.error(libc::EROFS);
}
fn symlink(
&mut self,
_req: &Request,
_parent: u64,
_name: &OsStr,
_link: &Path,
reply: ReplyEntry,
) {
reply.error(libc::EROFS);
}
fn link(
&mut self,
_req: &Request,
_ino: u64,
_newparent: u64,
_newname: &OsStr,
reply: ReplyEntry,
) {
reply.error(libc::EROFS);
}
fn mknod(
&mut self,
_req: &Request,
_parent: u64,
_name: &OsStr,
_mode: u32,
_umask: u32,
_rdev: u32,
reply: ReplyEntry,
) {
reply.error(libc::EROFS);
}
}
#[cfg(test)]
mod tests {
use super::*;
use musicfs_cache::TreeBuilder;
use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath};
use std::path::PathBuf;
fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(vpath),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from("/test"),
},
size,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
}
}
#[test]
fn test_tree_integration() {
let runtime = tokio::runtime::Runtime::new().unwrap();
let handle = runtime.handle().clone();
let mut builder = TreeBuilder::new();
builder.add_file(&make_file_meta(1, "/Artist/Album/Track.flac", 30_000_000));
let tree = Arc::new(RwLock::new(builder.build()));
let _fs = MusicFs::new(tree.clone(), handle);
let tree_read = tree.read();
assert!(tree_read.get(ROOT_INODE).is_some());
assert!(tree_read
.get_by_path(&VirtualPath::new("/Artist"))
.is_some());
}
}
crates/musicfs-fuse/src/lib.rs
-5
View File
@@ -1,5 +0,0 @@
mod filesystem;
pub mod ops;
pub use filesystem::MusicFs;
pub use ops::SearchOps;
crates/musicfs-fuse/src/ops/mod.rs
-5
View File
@@ -1,5 +0,0 @@
mod prefetch;
mod search;
pub use prefetch::PrefetchOps;
pub use search::SearchOps;
crates/musicfs-fuse/src/ops/prefetch.rs
-298
View File
@@ -1,298 +0,0 @@
use fuser::{FileAttr, FileType, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry};
use musicfs_cache::{PatternStore, PrefetchConfig, PrefetchEngine};
use musicfs_cas::ContentFetcher;
use musicfs_core::{EventBus, FileId};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
const PREFETCH_DIR_INODE: u64 = 0xFFFF_FFFF_0000_0002;
const PREFETCH_STATUS_INODE: u64 = 0xFFFF_FFFF_0000_0003;
const PREFETCH_HINTS_BASE: u64 = 0xFFFF_FFFF_2000_0000;
pub struct PrefetchOps {
pattern_store: Arc<PatternStore>,
engine: Option<Arc<PrefetchEngine>>,
uid: u32,
gid: u32,
}
impl PrefetchOps {
pub fn new(pattern_store: Arc<PatternStore>, uid: u32, gid: u32) -> Self {
Self {
pattern_store,
engine: None,
uid,
gid,
}
}
pub fn with_engine(
pattern_store: Arc<PatternStore>,
fetcher: Arc<ContentFetcher>,
config: PrefetchConfig,
uid: u32,
gid: u32,
) -> Self {
let engine = Arc::new(PrefetchEngine::new(config, pattern_store.clone(), fetcher));
Self {
pattern_store,
engine: Some(engine),
uid,
gid,
}
}
pub fn start_engine(&self, event_bus: Arc<EventBus>) -> Option<musicfs_cache::PrefetchHandle> {
self.engine
.as_ref()
.map(|e| e.clone().start(event_bus, self.pattern_store.clone()))
}
pub fn is_prefetch_dir_name(name: &str) -> bool {
name == ".prefetch"
}
pub fn is_prefetch_inode(inode: u64) -> bool {
inode == PREFETCH_DIR_INODE
|| inode == PREFETCH_STATUS_INODE
|| inode >= PREFETCH_HINTS_BASE
}
pub fn prefetch_dir_inode() -> u64 {
PREFETCH_DIR_INODE
}
pub fn lookup_prefetch_dir(&self, reply: ReplyEntry) {
let attr = self.dir_attr(PREFETCH_DIR_INODE);
reply.entry(&Duration::from_secs(60), &attr, 0);
}
pub fn lookup_status(&self, reply: ReplyEntry) {
let status = self.generate_status();
let attr = self.file_attr(PREFETCH_STATUS_INODE, status.len() as u64);
reply.entry(&Duration::from_secs(1), &attr, 0);
}
pub fn lookup_hint(&self, name: &str, reply: ReplyEntry) {
if let Some(inode) = self.hint_name_to_inode(name) {
let attr = self.file_attr(inode, 256);
reply.entry(&Duration::from_secs(1), &attr, 0);
} else {
reply.error(libc::ENOENT);
}
}
pub fn getattr_prefetch_dir(&self, reply: ReplyAttr) {
let attr = self.dir_attr(PREFETCH_DIR_INODE);
reply.attr(&Duration::from_secs(60), &attr);
}
pub fn getattr_status(&self, reply: ReplyAttr) {
let status = self.generate_status();
let attr = self.file_attr(PREFETCH_STATUS_INODE, status.len() as u64);
reply.attr(&Duration::from_secs(1), &attr);
}
pub fn getattr_hint(&self, inode: u64, reply: ReplyAttr) {
let attr = self.file_attr(inode, 256);
reply.attr(&Duration::from_secs(1), &attr);
}
pub fn readdir_prefetch_root(&self, offset: i64, mut reply: ReplyDirectory) {
let entries: Vec<(u64, FileType, &str)> = vec![
(PREFETCH_DIR_INODE, FileType::Directory, "."),
(1, FileType::Directory, ".."),
(PREFETCH_STATUS_INODE, FileType::RegularFile, "status"),
];
let recently_played = self.pattern_store.recently_played(7).unwrap_or_default();
let predictions: Vec<(u64, FileType, String)> = recently_played
.iter()
.take(10)
.enumerate()
.map(|(i, file_id)| {
let inode = PREFETCH_HINTS_BASE + i as u64;
let name = format!("hint_{:04}", file_id.0);
(inode, FileType::RegularFile, name)
})
.collect();
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
if reply.add(*inode, (i + 1) as i64, *kind, *name) {
reply.ok();
return;
}
}
let base_offset = entries.len();
for (i, (inode, kind, name)) in predictions.iter().enumerate() {
let entry_offset = base_offset + i;
if entry_offset < offset as usize {
continue;
}
if reply.add(*inode, (entry_offset + 1) as i64, *kind, name) {
break;
}
}
reply.ok();
}
pub fn read_status(&self, offset: i64, size: u32, reply: ReplyData) {
let status = self.generate_status();
let start = offset as usize;
let end = std::cmp::min(start + size as usize, status.len());
if start >= status.len() {
reply.data(&[]);
} else {
reply.data(&status.as_bytes()[start..end]);
}
}
pub fn read_hint(&self, inode: u64, offset: i64, size: u32, reply: ReplyData) {
let file_id = self.inode_to_file_id(inode);
let predictions = self.pattern_store.predict_next(file_id, 5);
let content = predictions
.iter()
.map(|id| format!("{}", id.0))
.collect::<Vec<_>>()
.join("\n");
let start = offset as usize;
let end = std::cmp::min(start + size as usize, content.len());
if start >= content.len() {
reply.data(&[]);
} else {
reply.data(&content.as_bytes()[start..end]);
}
}
fn generate_status(&self) -> String {
let engine_status = if let Some(engine) = &self.engine {
format!(
"running: {}\nin_flight: {}",
engine.is_running(),
engine.in_flight_count()
)
} else {
"engine: disabled".to_string()
};
let most_played = self
.pattern_store
.most_played(5)
.unwrap_or_default()
.iter()
.map(|id| format!("{}", id.0))
.collect::<Vec<_>>()
.join(", ");
format!(
"MusicFS Prefetch Status\n\
=======================\n\
{}\n\
most_played: [{}]\n",
engine_status, most_played
)
}
fn hint_name_to_inode(&self, name: &str) -> Option<u64> {
if name.starts_with("hint_") {
let id_str = name.strip_prefix("hint_")?;
let id: i64 = id_str.parse().ok()?;
Some(PREFETCH_HINTS_BASE + id as u64)
} else {
None
}
}
fn inode_to_file_id(&self, inode: u64) -> FileId {
FileId((inode - PREFETCH_HINTS_BASE) as i64)
}
fn dir_attr(&self, inode: u64) -> FileAttr {
FileAttr {
ino: inode,
size: 0,
blocks: 0,
atime: SystemTime::UNIX_EPOCH,
mtime: SystemTime::UNIX_EPOCH,
ctime: SystemTime::UNIX_EPOCH,
crtime: SystemTime::UNIX_EPOCH,
kind: FileType::Directory,
perm: 0o555,
nlink: 2,
uid: self.uid,
gid: self.gid,
rdev: 0,
blksize: 512,
flags: 0,
}
}
fn file_attr(&self, inode: u64, size: u64) -> FileAttr {
FileAttr {
ino: inode,
size,
blocks: (size + 511) / 512,
atime: SystemTime::UNIX_EPOCH,
mtime: SystemTime::UNIX_EPOCH,
ctime: SystemTime::UNIX_EPOCH,
crtime: SystemTime::UNIX_EPOCH,
kind: FileType::RegularFile,
perm: 0o444,
nlink: 1,
uid: self.uid,
gid: self.gid,
rdev: 0,
blksize: 512,
flags: 0,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[test]
fn test_prefetch_ops_new() {
let dir = TempDir::new().unwrap();
let pattern_store =
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
let _ops = PrefetchOps::new(pattern_store, 1000, 1000);
}
#[test]
fn test_is_prefetch_inode() {
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_DIR_INODE));
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_STATUS_INODE));
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_HINTS_BASE));
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_HINTS_BASE + 100));
assert!(!PrefetchOps::is_prefetch_inode(1));
assert!(!PrefetchOps::is_prefetch_inode(1000));
}
#[test]
fn test_hint_name_to_inode() {
let dir = TempDir::new().unwrap();
let pattern_store =
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
let ops = PrefetchOps::new(pattern_store, 1000, 1000);
assert_eq!(
ops.hint_name_to_inode("hint_0001"),
Some(PREFETCH_HINTS_BASE + 1)
);
assert_eq!(
ops.hint_name_to_inode("hint_9999"),
Some(PREFETCH_HINTS_BASE + 9999)
);
assert_eq!(ops.hint_name_to_inode("invalid"), None);
}
}
crates/musicfs-fuse/src/ops/search.rs
-273
View File
@@ -1,273 +0,0 @@
use fuser::{FileAttr, FileType, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry};
use moka::sync::Cache;
use musicfs_search::{SearchHit, SearchIndex};
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
const SEARCH_DIR_INODE: u64 = 0xFFFF_FFFF_0000_0001;
const SEARCH_RESULT_BASE: u64 = 0xFFFF_FFFF_1000_0000;
const RESULT_CACHE_MAX_ENTRIES: u64 = 1000;
const RESULT_CACHE_TTL_SECS: u64 = 300;
const INODE_CACHE_MAX_ENTRIES: u64 = 10000;
const MAX_QUERY_LENGTH: usize = 256;
pub struct SearchOps {
index: Arc<SearchIndex>,
result_cache: Cache<String, Vec<SearchHit>>,
inode_to_result: Cache<u64, (String, usize)>,
mount_point: String,
uid: u32,
gid: u32,
}
impl SearchOps {
pub fn new(index: Arc<SearchIndex>, mount_point: &str, uid: u32, gid: u32) -> Self {
let result_cache = Cache::builder()
.max_capacity(RESULT_CACHE_MAX_ENTRIES)
.time_to_live(Duration::from_secs(RESULT_CACHE_TTL_SECS))
.build();
let inode_to_result = Cache::builder()
.max_capacity(INODE_CACHE_MAX_ENTRIES)
.time_to_live(Duration::from_secs(RESULT_CACHE_TTL_SECS))
.build();
Self {
index,
result_cache,
inode_to_result,
mount_point: mount_point.to_string(),
uid,
gid,
}
}
pub fn is_search_dir_name(name: &str) -> bool {
name == ".search"
}
pub fn is_search_inode(inode: u64) -> bool {
inode == SEARCH_DIR_INODE || inode >= SEARCH_RESULT_BASE
}
pub fn search_dir_inode() -> u64 {
SEARCH_DIR_INODE
}
pub fn lookup_search_dir(&self, reply: ReplyEntry) {
let attr = self.dir_attr(SEARCH_DIR_INODE);
reply.entry(&Duration::from_secs(60), &attr, 0);
}
pub fn lookup_query_dir(&self, query: &str, inode: u64, reply: ReplyEntry) {
let results = self.execute_query(query);
if results.is_empty() {
reply.error(libc::ENOENT);
return;
}
let attr = self.dir_attr(inode);
reply.entry(&Duration::from_secs(1), &attr, 0);
}
pub fn lookup_result(&self, inode: u64, reply: ReplyEntry) {
if self.inode_to_result.contains_key(&inode) {
let attr = self.symlink_attr(inode, 256);
reply.entry(&Duration::from_secs(1), &attr, 0);
} else {
reply.error(libc::ENOENT);
}
}
pub fn getattr_search_dir(&self, reply: ReplyAttr) {
let attr = self.dir_attr(SEARCH_DIR_INODE);
reply.attr(&Duration::from_secs(60), &attr);
}
pub fn getattr_result(&self, inode: u64, reply: ReplyAttr) {
let attr = self.symlink_attr(inode, 256);
reply.attr(&Duration::from_secs(1), &attr);
}
pub fn readdir_search_root(&self, offset: i64, mut reply: ReplyDirectory) {
let entries = vec![
(SEARCH_DIR_INODE, FileType::Directory, "."),
(1, FileType::Directory, ".."),
];
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
if reply.add(*inode, (i + 1) as i64, *kind, name) {
break;
}
}
reply.ok();
}
pub fn readdir_query(&self, query: &str, offset: i64, mut reply: ReplyDirectory) {
let results = self.execute_query(query);
let entries = vec![
(SEARCH_DIR_INODE + 1, FileType::Directory, ".".to_string()),
(SEARCH_DIR_INODE, FileType::Directory, "..".to_string()),
];
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
if reply.add(*inode, (i + 1) as i64, *kind, name) {
reply.ok();
return;
}
}
let base_offset = entries.len();
for (i, hit) in results.iter().enumerate() {
let entry_offset = base_offset + i;
if entry_offset < offset as usize {
continue;
}
let inode = SEARCH_RESULT_BASE + i as u64;
let name = self.result_filename(hit, i);
self.inode_to_result.insert(inode, (query.to_string(), i));
if reply.add(inode, (entry_offset + 1) as i64, FileType::Symlink, &name) {
break;
}
}
reply.ok();
}
pub fn readlink(&self, inode: u64, reply: ReplyData) {
let (query, index) = match self.inode_to_result.get(&inode) {
Some((q, i)) => (q, i),
None => {
reply.error(libc::ENOENT);
return;
}
};
let results = self.execute_query(&query);
if let Some(hit) = results.get(index) {
if let Some(target) = self.safe_symlink_target(hit.virtual_path.as_str()) {
reply.data(target.as_bytes());
} else {
reply.error(libc::EINVAL);
}
} else {
reply.error(libc::ENOENT);
}
}
fn safe_symlink_target(&self, virtual_path: &str) -> Option<String> {
let normalized = Path::new(virtual_path).components().fold(
std::path::PathBuf::new(),
|mut acc, comp| {
match comp {
std::path::Component::Normal(s) => acc.push(s),
std::path::Component::RootDir => acc.push("/"),
_ => {}
}
acc
},
);
let path_str = normalized.to_string_lossy();
if path_str.contains("..") {
return None;
}
Some(format!("{}{}", self.mount_point, path_str))
}
fn execute_query(&self, query: &str) -> Vec<SearchHit> {
let query = if query.len() > MAX_QUERY_LENGTH {
&query[..MAX_QUERY_LENGTH]
} else {
query
};
if let Some(results) = self.result_cache.get(query) {
return results;
}
let results = self.index.search(query, 1000).unwrap_or_default();
self.result_cache.insert(query.to_string(), results.clone());
results
}
fn result_filename(&self, hit: &SearchHit, index: usize) -> String {
let artist = hit.artist.as_deref().unwrap_or("Unknown");
let title = hit.title.as_deref().unwrap_or("Unknown");
let ext = hit
.virtual_path
.as_str()
.rsplit('.')
.next()
.unwrap_or("flac");
format!("{:03}. {} - {}.{}", index + 1, artist, title, ext)
}
fn dir_attr(&self, inode: u64) -> FileAttr {
FileAttr {
ino: inode,
size: 0,
blocks: 0,
atime: SystemTime::UNIX_EPOCH,
mtime: SystemTime::UNIX_EPOCH,
ctime: SystemTime::UNIX_EPOCH,
crtime: SystemTime::UNIX_EPOCH,
kind: FileType::Directory,
perm: 0o555,
nlink: 2,
uid: self.uid,
gid: self.gid,
rdev: 0,
blksize: 512,
flags: 0,
}
}
fn symlink_attr(&self, inode: u64, target_len: u64) -> FileAttr {
FileAttr {
ino: inode,
size: target_len,
blocks: 0,
atime: SystemTime::UNIX_EPOCH,
mtime: SystemTime::UNIX_EPOCH,
ctime: SystemTime::UNIX_EPOCH,
crtime: SystemTime::UNIX_EPOCH,
kind: FileType::Symlink,
perm: 0o777,
nlink: 1,
uid: self.uid,
gid: self.gid,
rdev: 0,
blksize: 512,
flags: 0,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use musicfs_search::SearchIndex;
use tempfile::TempDir;
#[test]
fn test_search_ops_new() {
let dir = TempDir::new().unwrap();
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
let _ops = SearchOps::new(index, "/mnt/music", 1000, 1000);
}
#[test]
fn test_is_search_inode() {
assert!(SearchOps::is_search_inode(SEARCH_DIR_INODE));
assert!(SearchOps::is_search_inode(SEARCH_RESULT_BASE));
assert!(SearchOps::is_search_inode(SEARCH_RESULT_BASE + 100));
assert!(!SearchOps::is_search_inode(1));
assert!(!SearchOps::is_search_inode(1000));
}
}
crates/musicfs-grpc/Cargo.toml
-32
View File
@@ -1,32 +0,0 @@
[package]
name = "musicfs-grpc"
version.workspace = true
edition.workspace = true
[dependencies]
musicfs-cache = { path = "../musicfs-cache" }
musicfs-cas = { path = "../musicfs-cas" }
musicfs-metadata = { path = "../musicfs-metadata" }
musicfs-search = { path = "../musicfs-search" }
musicfs-core = { path = "../musicfs-core" }
parking_lot.workspace = true
tonic.workspace = true
prost.workspace = true
tokio.workspace = true
tokio-stream.workspace = true
tracing.workspace = true
thiserror.workspace = true
serde.workspace = true
serde_json.workspace = true
chrono.workspace = true
csv = "1.3"
reqwest = { version = "0.11", features = ["json"] }
hmac = "0.12"
sha2 = "0.10"
hex.workspace = true
[build-dependencies]
tonic-build.workspace = true
[dev-dependencies]
tempfile.workspace = true
crates/musicfs-grpc/build.rs
-4
View File
@@ -1,4 +0,0 @@
fn main() -> Result<(), Box<dyn std::error::Error>> {
tonic_build::compile_protos("proto/musicfs.proto")?;
Ok(())
}
crates/musicfs-grpc/proto/musicfs.proto
-322
View File
@@ -1,322 +0,0 @@
syntax = "proto3";
package musicfs.v1;
option go_package = "homelab.lan/music-agregator/gen/musicfs/v1;musicfsv1";
service MusicFS {
rpc Search(SearchRequest) returns (SearchResponse);
rpc SearchStream(SearchRequest) returns (stream SearchResult);
rpc GetStatus(Empty) returns (StatusResponse);
rpc Shutdown(ShutdownRequest) returns (Empty);
rpc GetCacheStats(Empty) returns (CacheStats);
rpc ClearCache(ClearCacheRequest) returns (ClearCacheResponse);
rpc Prefetch(PrefetchRequest) returns (stream PrefetchProgress);
rpc ListOrigins(Empty) returns (OriginsResponse);
rpc GetOriginHealth(OriginRequest) returns (OriginHealthResponse);
rpc RescanOrigin(OriginRequest) returns (stream SyncProgress);
rpc SubscribeEvents(EventFilter) returns (stream Event);
}
service MetadataService {
rpc GetMetadata(GetMetadataRequest) returns (MetadataResponse);
rpc UpdateMetadata(UpdateMetadataRequest) returns (UpdateMetadataResponse);
rpc ClearOverlay(ClearOverlayRequest) returns (ClearOverlayResponse);
rpc BatchUpdateMetadata(BatchUpdateRequest) returns (stream BatchUpdateProgress);
rpc ImportMetadata(ImportMetadataRequest) returns (stream ImportProgress);
}
message Empty {}
message SearchRequest {
string query = 1;
optional uint32 limit = 2;
optional uint32 offset = 3;
optional string origin_id = 4;
}
message SearchResponse {
repeated SearchResult results = 1;
uint64 total_matches = 2;
uint32 query_time_ms = 3;
}
message SearchResult {
int64 file_id = 1;
string virtual_path = 2;
optional string artist = 3;
optional string album = 4;
optional string title = 5;
float score = 6;
map<string, string> highlights = 7;
}
enum MountState {
MOUNT_UNKNOWN = 0;
MOUNT_MOUNTING = 1;
MOUNT_READY = 2;
MOUNT_SYNCING = 3;
MOUNT_DEGRADED = 4;
MOUNT_UNMOUNTING = 5;
}
message StatusResponse {
string version = 1;
uint64 uptime_secs = 2;
string mount_point = 3;
MountState state = 4;
uint32 open_file_handles = 5;
uint64 fuse_ops_total = 6;
uint64 files_indexed = 7;
uint64 cache_size_bytes = 8;
repeated OriginStatus origins = 9;
}
message OriginStatus {
string id = 1;
string origin_type = 2;
HealthStatus health = 3;
uint64 files_count = 4;
}
enum HealthStatus {
HEALTH_UNKNOWN = 0;
HEALTH_HEALTHY = 1;
HEALTH_DEGRADED = 2;
HEALTH_UNHEALTHY = 3;
}
message ShutdownRequest {
bool graceful = 1;
uint32 timeout_secs = 2;
}
message TierStats {
uint64 entries = 1;
uint64 size_bytes = 2;
uint64 hits = 3;
uint64 misses = 4;
}
message CacheStats {
uint64 total_size_bytes = 1;
uint64 used_size_bytes = 2;
uint64 size_limit_bytes = 3;
uint64 chunk_count = 4;
uint64 chunks_unique = 5;
double dedup_ratio = 6;
uint64 hit_count = 7;
uint64 miss_count = 8;
double hit_ratio = 9;
uint64 metadata_entries = 10;
uint64 metadata_bytes = 11;
TierStats l1_metadata = 12;
TierStats l2_headers = 13;
TierStats l3_chunks = 14;
}
message ClearCacheRequest {
optional string origin_id = 1;
bool clear_metadata = 2;
bool clear_chunks = 3;
}
message ClearCacheResponse {
uint64 bytes_cleared = 1;
uint64 chunks_cleared = 2;
}
message PrefetchRequest {
repeated string paths = 1;
optional string origin_id = 2;
}
message PrefetchProgress {
string current_path = 1;
uint32 completed = 2;
uint32 total = 3;
uint64 bytes_fetched = 4;
}
message OriginsResponse {
repeated OriginInfo origins = 1;
}
message OriginInfo {
string id = 1;
string origin_type = 2;
string display_name = 3;
string root_path = 4;
HealthStatus health = 5;
uint64 files_count = 6;
uint64 total_size_bytes = 7;
}
message OriginRequest {
string origin_id = 1;
// Optional subdirectory to scope the scan (relative to origin root).
// If empty, scans the entire origin.
// Example: "Metallica - Master of Puppets (1986) [FLAC]"
optional string subdir = 2;
}
message OriginHealthResponse {
string origin_id = 1;
HealthStatus status = 2;
optional string message = 3;
uint64 last_check_secs = 4;
}
message SyncProgress {
string phase = 1;
uint32 current = 2;
uint32 total = 3;
string current_path = 4;
uint64 bytes_synced = 5;
repeated SyncedFile new_files = 6;
}
message SyncedFile {
string path = 1;
int64 file_id = 2;
string virtual_path = 3;
}
message EventFilter {
repeated string event_types = 1;
optional string origin_id = 2;
}
message Event {
string event_type = 1;
int64 timestamp_ms = 2;
optional string origin_id = 3;
optional string path = 4;
optional int64 file_id = 5;
map<string, string> metadata = 6;
}
// MetadataService messages
message GetMetadataRequest {
string virtual_path = 1;
}
message MetadataResponse {
int64 file_id = 1;
optional string title = 2;
optional string artist = 3;
optional string album = 4;
optional string album_artist = 5;
optional uint32 year = 6;
optional uint32 track = 7;
optional uint32 disc = 8;
optional string genre = 9;
optional string format = 10;
optional uint64 duration_ms = 11;
optional uint64 bitrate = 12;
optional uint32 track_total = 13;
optional uint32 disc_total = 14;
optional string date = 15;
optional string composer = 16;
optional string comment = 17;
optional string lyrics = 18;
optional string copyright = 19;
optional bool compilation = 20;
optional string artist_sort = 21;
optional string album_artist_sort = 22;
optional string album_sort = 23;
optional string title_sort = 24;
optional string mb_recording_id = 25;
optional string mb_album_id = 26;
optional string mb_artist_id = 27;
optional string mb_album_artist_id = 28;
optional string mb_release_group_id = 29;
optional float replaygain_track_gain = 30;
optional float replaygain_track_peak = 31;
optional float replaygain_album_gain = 32;
optional float replaygain_album_peak = 33;
optional uint32 channels = 34;
optional uint32 bits_per_sample = 35;
optional string encoder = 36;
optional string label = 40;
optional string album_type = 41;
optional string cover_url = 42;
map<string, string> custom_tags = 50;
}
message UpdateMetadataRequest {
int64 file_id = 1;
optional string title = 2;
optional string artist = 3;
optional string album = 4;
optional string album_artist = 5;
optional uint32 track_number = 6;
optional uint32 disc_number = 7;
optional string date = 8;
optional string genre = 9;
optional string composer = 10;
optional string comment = 11;
optional string lyrics = 12;
optional string copyright = 13;
optional bool compilation = 14;
optional string artist_sort = 15;
optional string album_artist_sort = 16;
optional string album_sort = 17;
optional string title_sort = 18;
optional string mb_recording_id = 20;
optional string mb_album_id = 21;
optional string mb_artist_id = 22;
optional float replaygain_track_gain = 30;
optional float replaygain_track_peak = 31;
optional float replaygain_album_gain = 32;
optional float replaygain_album_peak = 33;
optional string label = 40;
optional string album_type = 41;
optional string cover_url = 42;
map<string, string> custom_tags = 50;
}
message UpdateMetadataResponse {
int64 file_id = 1;
bool success = 2;
optional string error_message = 3;
}
message ClearOverlayRequest {
int64 file_id = 1;
}
message ClearOverlayResponse {
int64 file_id = 1;
bool success = 2;
optional string error_message = 3;
}
message BatchUpdateRequest {
repeated BatchUpdateItem items = 1;
}
message BatchUpdateItem {
int64 file_id = 1;
UpdateMetadataRequest metadata = 2;
}
message BatchUpdateProgress {
uint32 completed = 1;
uint32 total = 2;
optional int64 current_file_id = 3;
optional string error_message = 4;
}
message ImportMetadataRequest {
string source_path = 1;
optional string format = 2;
}
message ImportProgress {
uint32 imported = 1;
uint32 total = 2;
optional string current_file = 3;
optional string error_message = 4;
}
crates/musicfs-grpc/src/lib.rs
-21
View File
@@ -1,21 +0,0 @@
pub mod proto {
pub mod musicfs {
pub mod v1 {
tonic::include_proto!("musicfs.v1");
}
}
}
mod metadata;
pub mod scanner;
mod search_service;
mod server;
mod webhook;
pub use metadata::MetadataServiceImpl;
pub use proto::musicfs::v1::metadata_service_server::MetadataServiceServer;
pub use proto::musicfs::v1::music_fs_server::{MusicFs, MusicFsServer as MusicFsGrpcServer};
pub use proto::musicfs::v1::*;
pub use search_service::SearchService;
pub use server::MusicFsServer;
pub use webhook::{WebhookConfig, WebhookHandler, WebhookPayload};
crates/musicfs-grpc/src/metadata.rs
-794
View File
@@ -1,794 +0,0 @@
//! MetadataService gRPC handlers for metadata overlay operations.
use crate::proto::musicfs::v1::{
metadata_service_server::MetadataService, BatchUpdateProgress, BatchUpdateRequest,
ClearOverlayRequest, ClearOverlayResponse, GetMetadataRequest, ImportMetadataRequest,
ImportProgress, MetadataResponse, UpdateMetadataRequest, UpdateMetadataResponse,
};
use musicfs_cache::{Database, EnrichmentUpdate};
use musicfs_core::{AudioMeta, FileId, VirtualPath};
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use tonic::{Request, Response, Status};
use tracing::{debug, info, instrument, warn};
/// gRPC service implementation for metadata operations.
pub struct MetadataServiceImpl {
db: Arc<Database>,
}
impl MetadataServiceImpl {
/// Create a new MetadataServiceImpl with the given database.
pub fn new(db: Arc<Database>) -> Self {
Self { db }
}
/// Convert AudioMeta to MetadataResponse proto message.
fn audio_meta_to_response(file_id: FileId, meta: &AudioMeta) -> MetadataResponse {
MetadataResponse {
file_id: file_id.0,
title: meta.title.clone(),
artist: meta.artist.clone(),
album: meta.album.clone(),
album_artist: meta.album_artist.clone(),
year: meta.year,
track: meta.track,
disc: meta.disc,
genre: meta.genre.clone(),
format: Some(format!("{:?}", meta.format)),
duration_ms: meta.duration_ms,
bitrate: meta.bitrate.map(|b| b as u64),
track_total: meta.track_total,
disc_total: meta.disc_total,
date: meta.date.clone(),
composer: meta.composer.clone(),
comment: meta.comment.clone(),
lyrics: meta.lyrics.clone(),
copyright: meta.copyright.clone(),
compilation: meta.compilation,
artist_sort: meta.artist_sort.clone(),
album_artist_sort: meta.album_artist_sort.clone(),
album_sort: meta.album_sort.clone(),
title_sort: meta.title_sort.clone(),
mb_recording_id: meta.mb_recording_id.clone(),
mb_album_id: meta.mb_album_id.clone(),
mb_artist_id: meta.mb_artist_id.clone(),
mb_album_artist_id: meta.mb_album_artist_id.clone(),
mb_release_group_id: meta.mb_release_group_id.clone(),
replaygain_track_gain: meta.replaygain_track_gain,
replaygain_track_peak: meta.replaygain_track_peak,
replaygain_album_gain: meta.replaygain_album_gain,
replaygain_album_peak: meta.replaygain_album_peak,
channels: meta.channels,
bits_per_sample: meta.bits_per_sample,
encoder: meta.encoder.clone(),
label: None,
album_type: None,
cover_url: None,
custom_tags: Default::default(),
}
}
/// Convert UpdateMetadataRequest to AudioMeta for database update.
fn request_to_audio_meta(req: &UpdateMetadataRequest) -> AudioMeta {
AudioMeta {
title: req.title.clone(),
artist: req.artist.clone(),
album: req.album.clone(),
album_artist: req.album_artist.clone(),
genre: req.genre.clone(),
year: None,
track: req.track_number,
disc: req.disc_number,
duration_ms: None,
bitrate: None,
sample_rate: None,
format: musicfs_core::AudioFormat::Unknown,
track_total: None,
disc_total: None,
date: req.date.clone(),
composer: req.composer.clone(),
comment: req.comment.clone(),
lyrics: req.lyrics.clone(),
copyright: req.copyright.clone(),
compilation: req.compilation,
artist_sort: req.artist_sort.clone(),
album_artist_sort: req.album_artist_sort.clone(),
album_sort: req.album_sort.clone(),
title_sort: req.title_sort.clone(),
mb_recording_id: req.mb_recording_id.clone(),
mb_album_id: req.mb_album_id.clone(),
mb_artist_id: req.mb_artist_id.clone(),
mb_album_artist_id: None,
mb_release_group_id: None,
replaygain_track_gain: req.replaygain_track_gain,
replaygain_track_peak: req.replaygain_track_peak,
replaygain_album_gain: req.replaygain_album_gain,
replaygain_album_peak: req.replaygain_album_peak,
channels: None,
bits_per_sample: None,
encoder: None,
}
}
}
#[tonic::async_trait]
impl MetadataService for MetadataServiceImpl {
#[instrument(level = "debug", skip(self, request), fields(method = "get_metadata"))]
async fn get_metadata(
&self,
request: Request<GetMetadataRequest>,
) -> Result<Response<MetadataResponse>, Status> {
let req = request.into_inner();
debug!(virtual_path = %req.virtual_path, "GetMetadata request");
if req.virtual_path.is_empty() {
return Err(Status::invalid_argument("virtual_path cannot be empty"));
}
let vpath = VirtualPath::new(&req.virtual_path);
let file_meta = self
.db
.get_file_by_virtual_path(&vpath)
.map_err(|e| Status::internal(format!("Database error: {}", e)))?
.ok_or_else(|| Status::not_found(format!("File not found: {}", req.virtual_path)))?;
let audio_meta = self
.db
.get_file_metadata_row(file_meta.id)
.map_err(|e| Status::internal(format!("Failed to get metadata: {}", e)))?;
let response = Self::audio_meta_to_response(file_meta.id, &audio_meta);
Ok(Response::new(response))
}
#[instrument(
level = "info",
skip(self, request),
fields(method = "update_metadata")
)]
async fn update_metadata(
&self,
request: Request<UpdateMetadataRequest>,
) -> Result<Response<UpdateMetadataResponse>, Status> {
let req = request.into_inner();
let file_id = FileId(req.file_id);
info!(file_id = req.file_id, "UpdateMetadata request");
if req.file_id <= 0 {
return Err(Status::invalid_argument("file_id must be positive"));
}
let audio_meta = Self::request_to_audio_meta(&req);
if let Err(e) = self.db.update_metadata(file_id, &audio_meta) {
warn!(file_id = req.file_id, error = %e, "Failed to update metadata");
return Ok(Response::new(UpdateMetadataResponse {
file_id: req.file_id,
success: false,
error_message: Some(e.to_string()),
}));
}
if req.label.is_some() || req.album_type.is_some() || req.cover_url.is_some() {
let enrichment = EnrichmentUpdate {
label: req.label.clone(),
album_type: req.album_type.clone(),
cover_url: req.cover_url.clone(),
genres_json: None,
primary_genre: None,
source: "orchestrator".to_string(),
};
if let Err(e) = self.db.update_enrichment(file_id, &enrichment) {
warn!(file_id = req.file_id, error = %e, "Failed to update enrichment");
return Ok(Response::new(UpdateMetadataResponse {
file_id: req.file_id,
success: false,
error_message: Some(e.to_string()),
}));
}
}
debug!(file_id = req.file_id, "Metadata updated successfully");
Ok(Response::new(UpdateMetadataResponse {
file_id: req.file_id,
success: true,
error_message: None,
}))
}
#[instrument(level = "info", skip(self, request), fields(method = "clear_overlay"))]
async fn clear_overlay(
&self,
request: Request<ClearOverlayRequest>,
) -> Result<Response<ClearOverlayResponse>, Status> {
let req = request.into_inner();
let file_id = FileId(req.file_id);
info!(file_id = req.file_id, "ClearOverlay request");
if req.file_id <= 0 {
return Err(Status::invalid_argument("file_id must be positive"));
}
match self.db.clear_overlay(file_id) {
Ok(()) => {
debug!(file_id = req.file_id, "Overlay cleared successfully");
Ok(Response::new(ClearOverlayResponse {
file_id: req.file_id,
success: true,
error_message: None,
}))
}
Err(e) => {
warn!(file_id = req.file_id, error = %e, "Failed to clear overlay");
Ok(Response::new(ClearOverlayResponse {
file_id: req.file_id,
success: false,
error_message: Some(e.to_string()),
}))
}
}
}
type BatchUpdateMetadataStream = ReceiverStream<Result<BatchUpdateProgress, Status>>;
#[instrument(
level = "info",
skip(self, request),
fields(method = "batch_update_metadata")
)]
async fn batch_update_metadata(
&self,
request: Request<BatchUpdateRequest>,
) -> Result<Response<Self::BatchUpdateMetadataStream>, Status> {
let req = request.into_inner();
let total = req.items.len() as u32;
info!(item_count = total, "BatchUpdateMetadata request");
let (tx, rx) = mpsc::channel(32);
let db = Arc::clone(&self.db);
tokio::spawn(async move {
for (i, item) in req.items.into_iter().enumerate() {
let file_id = FileId(item.file_id);
let completed = (i + 1) as u32;
let error_message = if let Some(ref metadata_req) = item.metadata {
let audio_meta = MetadataServiceImpl::request_to_audio_meta(metadata_req);
match db.update_metadata(file_id, &audio_meta) {
Ok(()) => {
if metadata_req.label.is_some()
|| metadata_req.album_type.is_some()
|| metadata_req.cover_url.is_some()
{
let enrichment = EnrichmentUpdate {
label: metadata_req.label.clone(),
album_type: metadata_req.album_type.clone(),
cover_url: metadata_req.cover_url.clone(),
genres_json: None,
primary_genre: None,
source: "orchestrator".to_string(),
};
if let Err(e) = db.update_enrichment(file_id, &enrichment) {
Some(e.to_string())
} else {
None
}
} else {
None
}
}
Err(e) => Some(e.to_string()),
}
} else {
Some("Missing metadata in batch item".to_string())
};
let progress = BatchUpdateProgress {
completed,
total,
current_file_id: Some(item.file_id),
error_message,
};
if tx.send(Ok(progress)).await.is_err() {
break;
}
}
});
Ok(Response::new(ReceiverStream::new(rx)))
}
type ImportMetadataStream = ReceiverStream<Result<ImportProgress, Status>>;
#[instrument(
level = "info",
skip(self, request),
fields(method = "import_metadata")
)]
async fn import_metadata(
&self,
request: Request<ImportMetadataRequest>,
) -> Result<Response<Self::ImportMetadataStream>, Status> {
let req = request.into_inner();
info!(source_path = %req.source_path, format = ?req.format, "ImportMetadata request");
if req.source_path.is_empty() {
return Err(Status::invalid_argument("source_path cannot be empty"));
}
let (tx, rx) = mpsc::channel(32);
let db = Arc::clone(&self.db);
let source_path = req.source_path.clone();
let format = req.format.clone();
tokio::spawn(async move {
let file_format = format.as_deref().unwrap_or_else(|| {
if source_path.ends_with(".csv") {
"csv"
} else if source_path.ends_with(".json") {
"json"
} else {
"unknown"
}
});
let content = match tokio::fs::read_to_string(&source_path).await {
Ok(c) => c,
Err(e) => {
let _ = tx
.send(Ok(ImportProgress {
imported: 0,
total: 0,
current_file: None,
error_message: Some(format!("Failed to read file: {}", e)),
}))
.await;
return;
}
};
let entries: Vec<ImportEntry> = match file_format {
"json" => match serde_json::from_str::<Vec<ImportEntry>>(&content) {
Ok(e) => e,
Err(e) => {
let _ = tx
.send(Ok(ImportProgress {
imported: 0,
total: 0,
current_file: None,
error_message: Some(format!("Failed to parse JSON: {}", e)),
}))
.await;
return;
}
},
"csv" => match parse_csv_entries(&content) {
Ok(e) => e,
Err(e) => {
let _ = tx
.send(Ok(ImportProgress {
imported: 0,
total: 0,
current_file: None,
error_message: Some(format!("Failed to parse CSV: {}", e)),
}))
.await;
return;
}
},
_ => {
let _ = tx
.send(Ok(ImportProgress {
imported: 0,
total: 0,
current_file: None,
error_message: Some(format!("Unsupported format: {}", file_format)),
}))
.await;
return;
}
};
let total = entries.len() as u32;
let mut imported = 0u32;
for entry in entries {
let vpath = VirtualPath::new(&entry.virtual_path);
let file_meta = match db.get_file_by_virtual_path(&vpath) {
Ok(Some(f)) => f,
Ok(None) => {
let progress = ImportProgress {
imported,
total,
current_file: Some(entry.virtual_path.clone()),
error_message: Some(format!("File not found: {}", entry.virtual_path)),
};
if tx.send(Ok(progress)).await.is_err() {
break;
}
continue;
}
Err(e) => {
let progress = ImportProgress {
imported,
total,
current_file: Some(entry.virtual_path.clone()),
error_message: Some(format!("Database error: {}", e)),
};
if tx.send(Ok(progress)).await.is_err() {
break;
}
continue;
}
};
let audio_meta = entry.to_audio_meta();
let error_message = match db.update_metadata(file_meta.id, &audio_meta) {
Ok(()) => {
imported += 1;
None
}
Err(e) => Some(e.to_string()),
};
let progress = ImportProgress {
imported,
total,
current_file: Some(entry.virtual_path),
error_message,
};
if tx.send(Ok(progress)).await.is_err() {
break;
}
}
});
Ok(Response::new(ReceiverStream::new(rx)))
}
}
/// Entry from import file (CSV or JSON).
#[derive(Debug, Clone, serde::Deserialize)]
struct ImportEntry {
virtual_path: String,
#[serde(default)]
title: Option<String>,
#[serde(default)]
artist: Option<String>,
#[serde(default)]
album: Option<String>,
#[serde(default)]
album_artist: Option<String>,
#[serde(default)]
genre: Option<String>,
#[serde(default)]
year: Option<u32>,
#[serde(default)]
track: Option<u32>,
#[serde(default)]
disc: Option<u32>,
#[serde(default)]
date: Option<String>,
#[serde(default)]
composer: Option<String>,
#[serde(default)]
comment: Option<String>,
}
impl ImportEntry {
fn to_audio_meta(&self) -> AudioMeta {
AudioMeta {
title: self.title.clone(),
artist: self.artist.clone(),
album: self.album.clone(),
album_artist: self.album_artist.clone(),
genre: self.genre.clone(),
year: self.year,
track: self.track,
disc: self.disc,
duration_ms: None,
bitrate: None,
sample_rate: None,
format: musicfs_core::AudioFormat::Unknown,
track_total: None,
disc_total: None,
date: self.date.clone(),
composer: self.composer.clone(),
comment: self.comment.clone(),
lyrics: None,
copyright: None,
compilation: None,
artist_sort: None,
album_artist_sort: None,
album_sort: None,
title_sort: None,
mb_recording_id: None,
mb_album_id: None,
mb_artist_id: None,
mb_album_artist_id: None,
mb_release_group_id: None,
replaygain_track_gain: None,
replaygain_track_peak: None,
replaygain_album_gain: None,
replaygain_album_peak: None,
channels: None,
bits_per_sample: None,
encoder: None,
}
}
}
/// Parse CSV content into ImportEntry list.
fn parse_csv_entries(content: &str) -> Result<Vec<ImportEntry>, String> {
let mut reader = csv::Reader::from_reader(content.as_bytes());
let mut entries = Vec::new();
for result in reader.deserialize() {
let entry: ImportEntry = result.map_err(|e| format!("CSV parse error: {}", e))?;
entries.push(entry);
}
Ok(entries)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::proto::musicfs::v1::BatchUpdateItem;
use musicfs_core::{AudioFormat, OriginId};
use std::path::Path;
use std::time::UNIX_EPOCH;
use tempfile::TempDir;
use tokio_stream::StreamExt;
fn create_test_db() -> (TempDir, Arc<Database>) {
let dir = TempDir::new().unwrap();
let db = Arc::new(Database::open_memory().unwrap());
(dir, db)
}
fn insert_test_file(db: &Database, vpath: &str) -> FileId {
let real_path = format!("/music{}", vpath);
db.upsert_file(
&OriginId::from("local"),
Path::new(&real_path),
&VirtualPath::new(vpath),
&AudioMeta {
title: Some("Test Track".to_string()),
artist: Some("Test Artist".to_string()),
album: Some("Test Album".to_string()),
format: AudioFormat::Flac,
..Default::default()
},
UNIX_EPOCH,
1000,
)
.unwrap()
}
#[tokio::test]
async fn test_get_metadata_success() {
let (_dir, db) = create_test_db();
let vpath = "/Artist/Album/Track.flac";
insert_test_file(&db, vpath);
let service = MetadataServiceImpl::new(db);
let request = Request::new(GetMetadataRequest {
virtual_path: vpath.to_string(),
});
let response = service.get_metadata(request).await.unwrap();
let meta = response.into_inner();
assert_eq!(meta.title, Some("Test Track".to_string()));
assert_eq!(meta.artist, Some("Test Artist".to_string()));
assert_eq!(meta.album, Some("Test Album".to_string()));
}
#[tokio::test]
async fn test_get_metadata_not_found() {
let (_dir, db) = create_test_db();
let service = MetadataServiceImpl::new(db);
let request = Request::new(GetMetadataRequest {
virtual_path: "/nonexistent.flac".to_string(),
});
let result = service.get_metadata(request).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);
}
#[tokio::test]
async fn test_get_metadata_empty_path() {
let (_dir, db) = create_test_db();
let service = MetadataServiceImpl::new(db);
let request = Request::new(GetMetadataRequest {
virtual_path: String::new(),
});
let result = service.get_metadata(request).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
}
#[tokio::test]
async fn test_update_metadata_success() {
let (_dir, db) = create_test_db();
let vpath = "/Artist/Album/Track.flac";
let file_id = insert_test_file(&db, vpath);
let service = MetadataServiceImpl::new(db.clone());
let request = Request::new(UpdateMetadataRequest {
file_id: file_id.0,
title: Some("Updated Title".to_string()),
artist: Some("Updated Artist".to_string()),
..Default::default()
});
let response = service.update_metadata(request).await.unwrap();
let result = response.into_inner();
assert!(result.success);
assert!(result.error_message.is_none());
let meta = db.get_file_metadata_row(file_id).unwrap();
assert_eq!(meta.title, Some("Updated Title".to_string()));
assert_eq!(meta.artist, Some("Updated Artist".to_string()));
}
#[tokio::test]
async fn test_update_metadata_invalid_id() {
let (_dir, db) = create_test_db();
let service = MetadataServiceImpl::new(db);
let request = Request::new(UpdateMetadataRequest {
file_id: 0,
title: Some("Title".to_string()),
..Default::default()
});
let result = service.update_metadata(request).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
}
#[tokio::test]
async fn test_clear_overlay_success() {
let (_dir, db) = create_test_db();
let vpath = "/Artist/Album/Track.flac";
let file_id = insert_test_file(&db, vpath);
let service = MetadataServiceImpl::new(db.clone());
let request = Request::new(ClearOverlayRequest { file_id: file_id.0 });
let response = service.clear_overlay(request).await.unwrap();
let result = response.into_inner();
assert!(result.success);
assert!(result.error_message.is_none());
let meta = db.get_file_metadata_row(file_id).unwrap();
assert!(meta.title.is_none());
assert!(meta.artist.is_none());
}
#[tokio::test]
async fn test_clear_overlay_invalid_id() {
let (_dir, db) = create_test_db();
let service = MetadataServiceImpl::new(db);
let request = Request::new(ClearOverlayRequest { file_id: -1 });
let result = service.clear_overlay(request).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
}
#[tokio::test]
async fn test_batch_update_metadata() {
let (_dir, db) = create_test_db();
let file_id1 = insert_test_file(&db, "/Track1.flac");
let file_id2 = insert_test_file(&db, "/Track2.flac");
let service = MetadataServiceImpl::new(db.clone());
let request = Request::new(BatchUpdateRequest {
items: vec![
BatchUpdateItem {
file_id: file_id1.0,
metadata: Some(UpdateMetadataRequest {
file_id: file_id1.0,
title: Some("Batch Title 1".to_string()),
..Default::default()
}),
},
BatchUpdateItem {
file_id: file_id2.0,
metadata: Some(UpdateMetadataRequest {
file_id: file_id2.0,
title: Some("Batch Title 2".to_string()),
..Default::default()
}),
},
],
});
let response = service.batch_update_metadata(request).await.unwrap();
let mut stream = response.into_inner();
let mut progress_count = 0;
while let Some(Ok(result)) = stream.next().await {
progress_count += 1;
assert!(result.error_message.is_none());
}
assert_eq!(progress_count, 2);
let meta1 = db.get_file_metadata_row(file_id1).unwrap();
assert_eq!(meta1.title, Some("Batch Title 1".to_string()));
let meta2 = db.get_file_metadata_row(file_id2).unwrap();
assert_eq!(meta2.title, Some("Batch Title 2".to_string()));
}
#[tokio::test]
async fn test_import_metadata_empty_path() {
let (_dir, db) = create_test_db();
let service = MetadataServiceImpl::new(db);
let request = Request::new(ImportMetadataRequest {
source_path: String::new(),
format: None,
});
let result = service.import_metadata(request).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
}
#[test]
fn test_parse_csv_entries() {
let csv_content = r#"virtual_path,title,artist,album
/Track1.flac,Title 1,Artist 1,Album 1
/Track2.flac,Title 2,Artist 2,Album 2"#;
let entries = parse_csv_entries(csv_content).unwrap();
assert_eq!(entries.len(), 2);
assert_eq!(entries[0].virtual_path, "/Track1.flac");
assert_eq!(entries[0].title, Some("Title 1".to_string()));
assert_eq!(entries[1].virtual_path, "/Track2.flac");
assert_eq!(entries[1].artist, Some("Artist 2".to_string()));
}
#[test]
fn test_import_entry_to_audio_meta() {
let entry = ImportEntry {
virtual_path: "/test.flac".to_string(),
title: Some("Test".to_string()),
artist: Some("Artist".to_string()),
album: None,
album_artist: None,
genre: Some("Rock".to_string()),
year: Some(2024),
track: Some(1),
disc: None,
date: None,
composer: None,
comment: None,
};
let meta = entry.to_audio_meta();
assert_eq!(meta.title, Some("Test".to_string()));
assert_eq!(meta.artist, Some("Artist".to_string()));
assert_eq!(meta.genre, Some("Rock".to_string()));
assert_eq!(meta.year, Some(2024));
assert_eq!(meta.track, Some(1));
}
}
crates/musicfs-grpc/src/scanner.rs
-261
View File
@@ -1,261 +0,0 @@
use musicfs_cache::{Database, VirtualTree};
use musicfs_cas::ContentFetcher;
use musicfs_core::{
AudioMeta, Error, Event, EventBus, FileId, FileMeta, OriginId, RealPath, Result, VirtualPath,
};
use musicfs_metadata::MetadataParser;
use parking_lot::RwLock;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::UNIX_EPOCH;
use tokio::sync::mpsc;
use tracing::{info, warn};
pub struct ScanResult {
pub new_files: Vec<SyncedFileInfo>,
pub changed: u32,
pub deleted: u32,
pub unchanged: u32,
pub bytes_synced: u64,
}
pub struct SyncedFileInfo {
pub path: String,
pub file_id: FileId,
pub virtual_path: String,
}
#[derive(Debug, Clone)]
pub struct ScanProgress {
pub phase: String,
pub current: u32,
pub total: u32,
pub current_path: String,
pub bytes_synced: u64,
}
pub struct OriginScanner {
db: Arc<Database>,
event_bus: Arc<EventBus>,
tree: Arc<RwLock<VirtualTree>>,
fetcher: Arc<ContentFetcher>,
parser: MetadataParser,
}
impl OriginScanner {
pub fn new(
db: Arc<Database>,
event_bus: Arc<EventBus>,
tree: Arc<RwLock<VirtualTree>>,
fetcher: Arc<ContentFetcher>,
) -> Self {
Self {
db,
event_bus,
tree,
fetcher,
parser: MetadataParser,
}
}
pub async fn scan(
&self,
origin_id: &OriginId,
origin_root: &Path,
subdir: Option<&str>,
progress_tx: mpsc::Sender<ScanProgress>,
) -> Result<ScanResult> {
let scan_root = match subdir {
Some(sub) if !sub.is_empty() => origin_root.join(sub),
_ => origin_root.to_path_buf(),
};
if !scan_root.exists() {
return Err(Error::Origin(format!(
"scan path does not exist: {}",
scan_root.display()
)));
}
// Phase 1: Scanning
let audio_files = self.collect_audio_files(&scan_root, &progress_tx)?;
let total_files = audio_files.len() as u32;
info!(files = total_files, "scan phase complete");
// Phase 2: Hashing + categorization
let mut new_files = Vec::new();
let mut unchanged = 0u32;
for (i, abs_path) in audio_files.iter().enumerate() {
let _ = progress_tx.try_send(ScanProgress {
phase: "hashing".to_string(),
current: i as u32 + 1,
total: total_files,
current_path: abs_path.display().to_string(),
bytes_synced: 0,
});
let rel_path = abs_path.strip_prefix(origin_root).unwrap_or(abs_path);
let existing = self.db.get_file_by_real_path(origin_id, rel_path)?;
if existing.is_some() {
unchanged += 1;
continue;
}
let size = std::fs::metadata(abs_path).map(|m| m.len()).unwrap_or(0);
new_files.push(DiscoveredFile {
abs_path: abs_path.clone(),
rel_path: rel_path.to_path_buf(),
size,
});
}
info!(
new = new_files.len(),
unchanged = unchanged,
"hash phase complete"
);
// Phase 3: Indexing
let mut synced = Vec::new();
let mut bytes_synced = 0u64;
let ingest_total = new_files.len() as u32;
for (i, file) in new_files.iter().enumerate() {
let _ = progress_tx.try_send(ScanProgress {
phase: "indexing".to_string(),
current: i as u32 + 1,
total: ingest_total,
current_path: file.abs_path.display().to_string(),
bytes_synced,
});
let audio_meta = match self.parser.parse_file(&file.abs_path) {
Ok(meta) => meta,
Err(e) => {
warn!(path = %file.abs_path.display(), error = %e, "parse failed, using defaults");
AudioMeta::default()
}
};
let virtual_path = derive_virtual_path(&audio_meta, &file.rel_path);
let file_id = self.db.upsert_file(
origin_id,
&file.rel_path,
&virtual_path,
&audio_meta,
UNIX_EPOCH,
file.size,
)?;
let file_meta = FileMeta {
id: file_id,
virtual_path: virtual_path.clone(),
real_path: RealPath {
origin_id: origin_id.clone(),
path: file.rel_path.clone(),
},
size: file.size,
mtime: UNIX_EPOCH,
content_hash: None,
audio: Some(audio_meta),
};
{
let mut tree = self.tree.write();
tree.insert_file(&file_meta);
}
self.fetcher.register_file(file_meta.clone());
self.event_bus.publish(Event::FileAdded {
path: virtual_path.clone(),
origin_id: origin_id.clone(),
});
bytes_synced += file.size;
synced.push(SyncedFileInfo {
path: file.abs_path.display().to_string(),
file_id,
virtual_path: virtual_path.as_str().to_string(),
});
}
Ok(ScanResult {
new_files: synced,
changed: 0,
deleted: 0,
unchanged,
bytes_synced,
})
}
fn collect_audio_files(
&self,
scan_root: &Path,
progress_tx: &mpsc::Sender<ScanProgress>,
) -> Result<Vec<PathBuf>> {
let mut files = Vec::new();
self.walk_dir(scan_root, &mut files, progress_tx)?;
Ok(files)
}
fn walk_dir(
&self,
dir: &Path,
files: &mut Vec<PathBuf>,
progress_tx: &mpsc::Sender<ScanProgress>,
) -> Result<()> {
let entries = std::fs::read_dir(dir)
.map_err(|e| Error::Origin(format!("read_dir {}: {}", dir.display(), e)))?;
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
self.walk_dir(&path, files, progress_tx)?;
} else if is_audio_file(&path) {
files.push(path.clone());
let _ = progress_tx.try_send(ScanProgress {
phase: "scanning".to_string(),
current: files.len() as u32,
total: 0,
current_path: path.display().to_string(),
bytes_synced: 0,
});
}
}
Ok(())
}
}
fn derive_virtual_path(meta: &AudioMeta, rel_path: &Path) -> VirtualPath {
let artist = meta.artist.as_deref().unwrap_or("Unknown Artist");
let album = meta.album.as_deref().unwrap_or("Unknown Album");
let filename = rel_path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown");
VirtualPath::new(format!("/{}/{}/{}", artist, album, filename))
}
fn is_audio_file(path: &Path) -> bool {
matches!(
path.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_lowercase())
.as_deref(),
Some("flac" | "mp3" | "ogg" | "wav" | "m4a" | "aac" | "opus")
)
}
struct DiscoveredFile {
abs_path: PathBuf,
rel_path: PathBuf,
size: u64,
}
crates/musicfs-grpc/src/search_service.rs
-251
View File
@@ -1,251 +0,0 @@
use crate::proto::musicfs::v1::{
music_fs_server::MusicFs, CacheStats, ClearCacheRequest, ClearCacheResponse, Empty, Event,
EventFilter, OriginHealthResponse, OriginRequest, OriginsResponse, PrefetchProgress,
PrefetchRequest, SearchRequest, SearchResponse, SearchResult, ShutdownRequest, StatusResponse,
SyncProgress,
};
use musicfs_search::SearchIndex;
use std::sync::Arc;
use std::time::Instant;
use tokio_stream::wrappers::ReceiverStream;
use tonic::{Request, Response, Status};
use tracing::debug;
pub struct SearchService {
index: Arc<SearchIndex>,
}
impl SearchService {
pub fn new(index: Arc<SearchIndex>) -> Self {
Self { index }
}
}
#[tonic::async_trait]
impl MusicFs for SearchService {
async fn search(
&self,
request: Request<SearchRequest>,
) -> Result<Response<SearchResponse>, Status> {
let start = Instant::now();
let req = request.into_inner();
if req.query.is_empty() {
return Err(Status::invalid_argument("Query cannot be empty"));
}
if req.query.len() > 256 {
return Err(Status::invalid_argument(
"Query exceeds maximum length (256)",
));
}
let limit = req.limit.unwrap_or(100).min(10000) as usize;
let offset = req.offset.unwrap_or(0) as usize;
let results = self
.index
.search(&req.query, limit + offset)
.map_err(|e| Status::internal(format!("Search failed: {}", e)))?;
let hits: Vec<SearchResult> = results
.into_iter()
.skip(offset)
.take(limit)
.map(|hit| SearchResult {
file_id: hit.file_id.0,
virtual_path: hit.virtual_path.as_str().to_string(),
artist: hit.artist,
album: hit.album,
title: hit.title,
score: hit.score,
highlights: Default::default(),
})
.collect();
let total_matches = self.index.count();
let query_time_ms = start.elapsed().as_millis() as u32;
debug!(
"Search '{}' returned {} results in {}ms",
req.query,
hits.len(),
query_time_ms
);
Ok(Response::new(SearchResponse {
results: hits,
total_matches,
query_time_ms,
}))
}
type SearchStreamStream = ReceiverStream<Result<SearchResult, Status>>;
async fn search_stream(
&self,
request: Request<SearchRequest>,
) -> Result<Response<Self::SearchStreamStream>, Status> {
let req = request.into_inner();
if req.query.is_empty() {
return Err(Status::invalid_argument("Query cannot be empty"));
}
let limit = req.limit.unwrap_or(1000).min(10000) as usize;
let results = self
.index
.search(&req.query, limit)
.map_err(|e| Status::internal(format!("Search failed: {}", e)))?;
let (tx, rx) = tokio::sync::mpsc::channel(100);
tokio::spawn(async move {
for hit in results {
let result = SearchResult {
file_id: hit.file_id.0,
virtual_path: hit.virtual_path.as_str().to_string(),
artist: hit.artist,
album: hit.album,
title: hit.title,
score: hit.score,
highlights: Default::default(),
};
if tx.send(Ok(result)).await.is_err() {
break;
}
}
});
Ok(Response::new(ReceiverStream::new(rx)))
}
async fn get_status(
&self,
_request: Request<Empty>,
) -> Result<Response<StatusResponse>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
async fn shutdown(
&self,
_request: Request<ShutdownRequest>,
) -> Result<Response<Empty>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
async fn get_cache_stats(
&self,
_request: Request<Empty>,
) -> Result<Response<CacheStats>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
async fn clear_cache(
&self,
_request: Request<ClearCacheRequest>,
) -> Result<Response<ClearCacheResponse>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
type PrefetchStream = ReceiverStream<Result<PrefetchProgress, Status>>;
async fn prefetch(
&self,
_request: Request<PrefetchRequest>,
) -> Result<Response<Self::PrefetchStream>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
async fn list_origins(
&self,
_request: Request<Empty>,
) -> Result<Response<OriginsResponse>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
async fn get_origin_health(
&self,
_request: Request<OriginRequest>,
) -> Result<Response<OriginHealthResponse>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
type RescanOriginStream = ReceiverStream<Result<SyncProgress, Status>>;
async fn rescan_origin(
&self,
_request: Request<OriginRequest>,
) -> Result<Response<Self::RescanOriginStream>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
async fn subscribe_events(
&self,
_request: Request<EventFilter>,
) -> Result<Response<Self::SubscribeEventsStream>, Status> {
Err(Status::unimplemented(
"Use MusicFsServer for control operations",
))
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[tokio::test]
async fn test_grpc_search_empty_query() {
let dir = TempDir::new().unwrap();
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
let service = SearchService::new(index);
let request = Request::new(SearchRequest {
query: String::new(),
limit: Some(10),
offset: None,
origin_id: None,
});
let result = service.search(request).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
}
#[tokio::test]
async fn test_grpc_search_returns_response() {
let dir = TempDir::new().unwrap();
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
let service = SearchService::new(index);
let request = Request::new(SearchRequest {
query: "test".to_string(),
limit: Some(10),
offset: None,
origin_id: None,
});
let response = service.search(request).await.unwrap();
assert!(response.get_ref().results.is_empty());
}
}
crates/musicfs-grpc/src/server.rs
-561
View File
@@ -1,561 +0,0 @@
use crate::proto::musicfs::v1::{
music_fs_server::MusicFs, CacheStats, ClearCacheRequest, ClearCacheResponse, Empty, Event,
EventFilter, HealthStatus, MountState, OriginHealthResponse, OriginRequest, OriginsResponse,
PrefetchProgress, PrefetchRequest, SearchRequest, SearchResponse, SearchResult,
ShutdownRequest, StatusResponse, SyncProgress, SyncedFile, TierStats,
};
use musicfs_core::{Event as CoreEvent, EventBus};
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use tonic::{Request, Response, Status};
use tracing::{debug, info, instrument};
pub struct MusicFsServer {
start_time: Instant,
event_bus: Arc<EventBus>,
version: String,
scanner: Arc<crate::scanner::OriginScanner>,
origin_root: std::path::PathBuf,
}
impl MusicFsServer {
pub fn new(
event_bus: Arc<EventBus>,
db: Arc<musicfs_cache::Database>,
tree: Arc<parking_lot::RwLock<musicfs_cache::VirtualTree>>,
fetcher: Arc<musicfs_cas::ContentFetcher>,
origin_root: std::path::PathBuf,
) -> Self {
let scanner = Arc::new(crate::scanner::OriginScanner::new(
db,
event_bus.clone(),
tree,
fetcher,
));
Self {
start_time: Instant::now(),
event_bus,
version: env!("CARGO_PKG_VERSION").to_string(),
scanner,
origin_root,
}
}
fn event_to_proto(event: &CoreEvent) -> Event {
let (event_type, origin_id, path, file_id) = match event {
CoreEvent::FileAccessed {
file_id,
origin_id,
path,
..
} => (
"file_accessed".to_string(),
Some(origin_id.to_string()),
Some(path.as_str().to_string()),
Some(file_id.0),
),
CoreEvent::FileAdded { path, origin_id } => (
"file_added".to_string(),
Some(origin_id.to_string()),
Some(path.as_str().to_string()),
None,
),
CoreEvent::FileRemoved { path, file_id } => (
"file_removed".to_string(),
None,
Some(path.as_str().to_string()),
file_id.map(|id| id.0),
),
CoreEvent::FileModified { path } => (
"file_modified".to_string(),
None,
Some(path.as_str().to_string()),
None,
),
CoreEvent::SyncStarted { origin_id } => (
"sync_started".to_string(),
Some(origin_id.to_string()),
None,
None,
),
CoreEvent::SyncCompleted {
origin_id,
files_changed,
} => {
let mut metadata = std::collections::HashMap::new();
metadata.insert("files_changed".to_string(), files_changed.to_string());
return Event {
event_type: "sync_completed".to_string(),
timestamp_ms: chrono::Utc::now().timestamp_millis(),
origin_id: Some(origin_id.to_string()),
path: None,
file_id: None,
metadata,
};
}
CoreEvent::OriginHealthChanged { origin_id, healthy } => {
let mut metadata = std::collections::HashMap::new();
metadata.insert("healthy".to_string(), healthy.to_string());
return Event {
event_type: "origin_health_changed".to_string(),
timestamp_ms: chrono::Utc::now().timestamp_millis(),
origin_id: Some(origin_id.to_string()),
path: None,
file_id: None,
metadata,
};
}
CoreEvent::CacheEviction { bytes_freed } => {
let mut metadata = std::collections::HashMap::new();
metadata.insert("bytes_freed".to_string(), bytes_freed.to_string());
return Event {
event_type: "cache_eviction".to_string(),
timestamp_ms: chrono::Utc::now().timestamp_millis(),
origin_id: None,
path: None,
file_id: None,
metadata,
};
}
CoreEvent::OriginConnected { origin_id } => (
"origin_connected".to_string(),
Some(origin_id.to_string()),
None,
None,
),
CoreEvent::OriginDisconnected { origin_id } => (
"origin_disconnected".to_string(),
Some(origin_id.to_string()),
None,
None,
),
CoreEvent::AllOriginsUnhealthy { candidate_count } => {
let mut metadata = std::collections::HashMap::new();
metadata.insert("candidate_count".to_string(), candidate_count.to_string());
return Event {
event_type: "all_origins_unhealthy".to_string(),
timestamp_ms: chrono::Utc::now().timestamp_millis(),
origin_id: None,
path: None,
file_id: None,
metadata,
};
}
};
Event {
event_type,
timestamp_ms: chrono::Utc::now().timestamp_millis(),
origin_id,
path,
file_id,
metadata: Default::default(),
}
}
fn matches_filter(event: &CoreEvent, filter: &EventFilter) -> bool {
if !filter.event_types.is_empty() {
let event_type = match event {
CoreEvent::FileAccessed { .. } => "file_accessed",
CoreEvent::FileAdded { .. } => "file_added",
CoreEvent::FileRemoved { .. } => "file_removed",
CoreEvent::FileModified { .. } => "file_modified",
CoreEvent::SyncStarted { .. } => "sync_started",
CoreEvent::SyncCompleted { .. } => "sync_completed",
CoreEvent::OriginHealthChanged { .. } => "origin_health_changed",
CoreEvent::CacheEviction { .. } => "cache_eviction",
CoreEvent::OriginConnected { .. } => "origin_connected",
CoreEvent::OriginDisconnected { .. } => "origin_disconnected",
CoreEvent::AllOriginsUnhealthy { .. } => "all_origins_unhealthy",
};
if !filter.event_types.iter().any(|t| t == event_type) {
return false;
}
}
if let Some(ref origin_filter) = filter.origin_id {
let event_origin = match event {
CoreEvent::FileAccessed { origin_id, .. }
| CoreEvent::FileAdded { origin_id, .. }
| CoreEvent::SyncStarted { origin_id }
| CoreEvent::SyncCompleted { origin_id, .. }
| CoreEvent::OriginHealthChanged { origin_id, .. }
| CoreEvent::OriginConnected { origin_id }
| CoreEvent::OriginDisconnected { origin_id } => Some(origin_id.to_string()),
CoreEvent::FileRemoved { .. }
| CoreEvent::FileModified { .. }
| CoreEvent::CacheEviction { .. }
| CoreEvent::AllOriginsUnhealthy { .. } => None,
};
if event_origin.as_ref() != Some(origin_filter) {
return false;
}
}
true
}
}
#[tonic::async_trait]
impl MusicFs for MusicFsServer {
async fn search(
&self,
_request: Request<SearchRequest>,
) -> Result<Response<SearchResponse>, Status> {
Err(Status::unimplemented(
"Use SearchService for search operations",
))
}
type SearchStreamStream = ReceiverStream<Result<SearchResult, Status>>;
async fn search_stream(
&self,
_request: Request<SearchRequest>,
) -> Result<Response<Self::SearchStreamStream>, Status> {
Err(Status::unimplemented(
"Use SearchService for search operations",
))
}
#[instrument(level = "debug", skip(self, _request), fields(method = "get_status"))]
async fn get_status(
&self,
_request: Request<Empty>,
) -> Result<Response<StatusResponse>, Status> {
debug!("gRPC get_status called");
let uptime = self.start_time.elapsed().as_secs();
Ok(Response::new(StatusResponse {
version: self.version.clone(),
uptime_secs: uptime,
mount_point: String::new(),
state: MountState::MountReady as i32,
open_file_handles: 0,
fuse_ops_total: 0,
files_indexed: 0,
cache_size_bytes: 0,
origins: vec![],
}))
}
#[instrument(level = "info", skip(self, request), fields(method = "shutdown"))]
async fn shutdown(&self, request: Request<ShutdownRequest>) -> Result<Response<Empty>, Status> {
let req = request.into_inner();
info!(
graceful = req.graceful,
timeout_secs = req.timeout_secs,
"gRPC shutdown requested"
);
Ok(Response::new(Empty {}))
}
#[instrument(
level = "debug",
skip(self, _request),
fields(method = "get_cache_stats")
)]
async fn get_cache_stats(
&self,
_request: Request<Empty>,
) -> Result<Response<CacheStats>, Status> {
debug!("gRPC get_cache_stats called");
Ok(Response::new(CacheStats {
total_size_bytes: 0,
used_size_bytes: 0,
size_limit_bytes: 0,
chunk_count: 0,
chunks_unique: 0,
dedup_ratio: 0.0,
hit_count: 0,
miss_count: 0,
hit_ratio: 0.0,
metadata_entries: 0,
metadata_bytes: 0,
l1_metadata: Some(TierStats {
entries: 0,
size_bytes: 0,
hits: 0,
misses: 0,
}),
l2_headers: Some(TierStats {
entries: 0,
size_bytes: 0,
hits: 0,
misses: 0,
}),
l3_chunks: Some(TierStats {
entries: 0,
size_bytes: 0,
hits: 0,
misses: 0,
}),
}))
}
#[instrument(level = "info", skip(self, request), fields(method = "clear_cache"))]
async fn clear_cache(
&self,
request: Request<ClearCacheRequest>,
) -> Result<Response<ClearCacheResponse>, Status> {
let req = request.into_inner();
info!(
origin_id = ?req.origin_id,
clear_metadata = req.clear_metadata,
clear_chunks = req.clear_chunks,
"gRPC clear_cache"
);
Ok(Response::new(ClearCacheResponse {
bytes_cleared: 0,
chunks_cleared: 0,
}))
}
type PrefetchStream = ReceiverStream<Result<PrefetchProgress, Status>>;
#[instrument(level = "debug", skip(self, request), fields(method = "prefetch"))]
async fn prefetch(
&self,
request: Request<PrefetchRequest>,
) -> Result<Response<Self::PrefetchStream>, Status> {
let req = request.into_inner();
let total = req.paths.len() as u32;
debug!(file_count = total, "gRPC prefetch started");
let (tx, rx) = mpsc::channel(32);
tokio::spawn(async move {
for (i, path) in req.paths.into_iter().enumerate() {
let progress = PrefetchProgress {
current_path: path,
completed: i as u32 + 1,
total,
bytes_fetched: 0,
};
if tx.send(Ok(progress)).await.is_err() {
break;
}
}
});
Ok(Response::new(ReceiverStream::new(rx)))
}
#[instrument(level = "debug", skip(self, _request), fields(method = "list_origins"))]
async fn list_origins(
&self,
_request: Request<Empty>,
) -> Result<Response<OriginsResponse>, Status> {
debug!("gRPC list_origins called");
Ok(Response::new(OriginsResponse { origins: vec![] }))
}
#[instrument(
level = "debug",
skip(self, request),
fields(method = "get_origin_health")
)]
async fn get_origin_health(
&self,
request: Request<OriginRequest>,
) -> Result<Response<OriginHealthResponse>, Status> {
let req = request.into_inner();
debug!(origin_id = %req.origin_id, "gRPC get_origin_health");
Ok(Response::new(OriginHealthResponse {
origin_id: req.origin_id,
status: HealthStatus::HealthUnknown as i32,
message: None,
last_check_secs: 0,
}))
}
type RescanOriginStream = ReceiverStream<Result<SyncProgress, Status>>;
#[instrument(level = "info", skip(self, request), fields(method = "rescan_origin"))]
async fn rescan_origin(
&self,
request: Request<OriginRequest>,
) -> Result<Response<Self::RescanOriginStream>, Status> {
let req = request.into_inner();
let subdir = req.subdir.as_deref().filter(|s| !s.is_empty());
info!(
origin_id = %req.origin_id,
subdir = ?subdir,
"gRPC rescan_origin started"
);
let (tx, rx) = mpsc::channel(32);
let (progress_tx, mut progress_rx) = mpsc::channel::<crate::scanner::ScanProgress>(64);
let origin_id = musicfs_core::OriginId::from(req.origin_id.as_str());
let scanner = self.scanner.clone();
let origin_root = self.origin_root.clone();
let subdir_owned = subdir.map(|s| s.to_string());
tokio::spawn(async move {
let forward_handle = {
let tx = tx.clone();
tokio::spawn(async move {
while let Some(progress) = progress_rx.recv().await {
let proto = SyncProgress {
phase: progress.phase,
current: progress.current,
total: progress.total,
current_path: progress.current_path,
bytes_synced: progress.bytes_synced,
new_files: vec![],
};
if tx.send(Ok(proto)).await.is_err() {
break;
}
}
})
};
let result = scanner
.scan(
&origin_id,
&origin_root,
subdir_owned.as_deref(),
progress_tx,
)
.await;
forward_handle.abort();
match result {
Ok(scan_result) => {
let synced_files: Vec<SyncedFile> = scan_result
.new_files
.iter()
.map(|f| SyncedFile {
path: f.path.clone(),
file_id: f.file_id.0,
virtual_path: f.virtual_path.clone(),
})
.collect();
let _ = tx
.send(Ok(SyncProgress {
phase: "complete".to_string(),
current: scan_result.new_files.len() as u32
+ scan_result.changed
+ scan_result.deleted,
total: scan_result.new_files.len() as u32
+ scan_result.changed
+ scan_result.deleted
+ scan_result.unchanged,
current_path: String::new(),
bytes_synced: scan_result.bytes_synced,
new_files: synced_files,
}))
.await;
}
Err(e) => {
let _ = tx
.send(Err(Status::internal(format!("rescan failed: {}", e))))
.await;
}
}
});
Ok(Response::new(ReceiverStream::new(rx)))
}
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
#[instrument(
level = "info",
skip(self, request),
fields(method = "subscribe_events")
)]
async fn subscribe_events(
&self,
request: Request<EventFilter>,
) -> Result<Response<Self::SubscribeEventsStream>, Status> {
info!("gRPC subscribe_events: client connected");
let filter = request.into_inner();
let mut rx = self.event_bus.subscribe();
let (tx, out_rx) = mpsc::channel(100);
tokio::spawn(async move {
loop {
match rx.recv().await {
Ok(event) => {
if Self::matches_filter(&event, &filter) {
let proto_event = Self::event_to_proto(&event);
if tx.send(Ok(proto_event)).await.is_err() {
break;
}
}
}
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
tracing::warn!(skipped = n, "Event subscriber lagged, skipped events");
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
tracing::debug!("Event channel closed");
break;
}
}
}
});
Ok(Response::new(ReceiverStream::new(out_rx)))
}
}
#[cfg(test)]
mod tests {
use super::*;
async fn make_test_server() -> (MusicFsServer, tempfile::TempDir) {
let event_bus = Arc::new(EventBus::new(16));
let db = Arc::new(musicfs_cache::Database::open_memory().unwrap());
let tree = Arc::new(parking_lot::RwLock::new(
musicfs_cache::TreeBuilder::new().build(),
));
let dir = tempfile::tempdir().unwrap();
let cfg = musicfs_cas::CasConfig {
chunks_dir: dir.path().join("chunks"),
..Default::default()
};
let store = Arc::new(musicfs_cas::CasStore::open(cfg).await.unwrap());
let fetcher = Arc::new(musicfs_cas::ContentFetcher::new(store));
let origin_root = std::path::PathBuf::from("/tmp/test-origin");
(
MusicFsServer::new(event_bus, db, tree, fetcher, origin_root),
dir,
)
}
#[tokio::test]
async fn test_get_status() {
let (server, _dir) = make_test_server().await;
let response = server.get_status(Request::new(Empty {})).await.unwrap();
let status = response.into_inner();
assert!(!status.version.is_empty());
assert!(status.uptime_secs < 5);
}
#[tokio::test]
async fn test_get_cache_stats() {
let (server, _dir) = make_test_server().await;
let response = server
.get_cache_stats(Request::new(Empty {}))
.await
.unwrap();
let stats = response.into_inner();
assert_eq!(stats.hit_ratio, 0.0);
}
}
crates/musicfs-grpc/src/webhook.rs
-327
View File
@@ -1,327 +0,0 @@
use musicfs_core::Event;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use tokio::sync::broadcast;
use tracing::{debug, error, warn};
#[derive(Debug, Clone, Serialize)]
pub struct WebhookPayload {
pub event_type: String,
pub timestamp: i64,
pub data: serde_json::Value,
}
#[derive(Clone, Deserialize)]
pub struct WebhookConfig {
pub url: String,
#[serde(skip_serializing)]
pub secret: Option<String>,
pub events: Vec<String>,
#[serde(default = "default_retry_count")]
pub retry_count: u32,
#[serde(default = "default_timeout_ms")]
pub timeout_ms: u64,
}
impl std::fmt::Debug for WebhookConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WebhookConfig")
.field("url", &self.url)
.field("secret", &self.secret.as_ref().map(|_| "[REDACTED]"))
.field("events", &self.events)
.field("retry_count", &self.retry_count)
.field("timeout_ms", &self.timeout_ms)
.finish()
}
}
fn default_retry_count() -> u32 {
3
}
fn default_timeout_ms() -> u64 {
5000
}
#[derive(Debug, thiserror::Error)]
pub enum WebhookError {
#[error("Failed to initialize HTTP client: {0}")]
ClientInit(String),
}
pub struct WebhookHandler {
client: reqwest::Client,
configs: Vec<WebhookConfig>,
}
impl WebhookHandler {
pub fn new(configs: Vec<WebhookConfig>) -> Result<Self, WebhookError> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(30))
.build()
.map_err(|e| {
error!(error = %e, "Failed to create webhook HTTP client");
WebhookError::ClientInit(e.to_string())
})?;
Ok(Self { client, configs })
}
pub async fn run(&self, mut rx: broadcast::Receiver<Event>) {
loop {
match rx.recv().await {
Ok(event) => {
for config in &self.configs {
if self.matches_filter(&event, config) {
self.dispatch(config, &event).await;
}
}
}
Err(broadcast::error::RecvError::Lagged(n)) => {
warn!(skipped = n, "Webhook handler lagged, skipped events");
}
Err(broadcast::error::RecvError::Closed) => {
debug!("Event channel closed, webhook handler stopping");
break;
}
}
}
}
async fn dispatch(&self, config: &WebhookConfig, event: &Event) {
let payload = WebhookPayload {
event_type: self.event_type_name(event),
timestamp: chrono::Utc::now().timestamp_millis(),
data: self.event_to_json(event),
};
let signature = self.sign(&payload, config);
let mut attempts = 0u32;
loop {
let result = self
.client
.post(&config.url)
.timeout(Duration::from_millis(config.timeout_ms))
.header("Content-Type", "application/json")
.header("X-MusicFS-Signature", &signature)
.header("X-MusicFS-Event", &payload.event_type)
.json(&payload)
.send()
.await;
match result {
Ok(resp) if resp.status().is_success() => {
debug!(
"Webhook delivered to {} for {}",
config.url, payload.event_type
);
break;
}
Ok(resp) => {
warn!(
"Webhook to {} returned status {}, attempt {}/{}",
config.url,
resp.status(),
attempts + 1,
config.retry_count + 1
);
}
Err(e) => {
warn!(
"Webhook to {} failed: {}, attempt {}/{}",
config.url,
e,
attempts + 1,
config.retry_count + 1
);
}
}
if attempts >= config.retry_count {
warn!(
"Webhook delivery to {} failed after {} attempts",
config.url,
attempts + 1
);
break;
}
attempts += 1;
let delay = Duration::from_millis(100 * 2u64.pow(attempts));
tokio::time::sleep(delay).await;
}
}
fn sign(&self, payload: &WebhookPayload, config: &WebhookConfig) -> String {
match &config.secret {
Some(secret) => {
use hmac::{Hmac, Mac};
use sha2::Sha256;
type HmacSha256 = Hmac<Sha256>;
let body = serde_json::to_string(payload).unwrap_or_default();
let mac = match HmacSha256::new_from_slice(secret.as_bytes()) {
Ok(m) => m,
Err(e) => {
error!(error = %e, "Invalid HMAC key for webhook signature");
return String::new();
}
};
let mut mac = mac;
mac.update(body.as_bytes());
let result = mac.finalize();
format!("sha256={}", hex::encode(result.into_bytes()))
}
None => String::new(),
}
}
fn matches_filter(&self, event: &Event, config: &WebhookConfig) -> bool {
if config.events.is_empty() {
return true;
}
let event_type = self.event_type_name(event);
config.events.iter().any(|e| e == &event_type)
}
fn event_type_name(&self, event: &Event) -> String {
match event {
Event::FileAccessed { .. } => "file_accessed",
Event::FileAdded { .. } => "file_added",
Event::FileRemoved { .. } => "file_removed",
Event::FileModified { .. } => "file_modified",
Event::SyncStarted { .. } => "sync_started",
Event::SyncCompleted { .. } => "sync_completed",
Event::OriginHealthChanged { .. } => "origin_health_changed",
Event::CacheEviction { .. } => "cache_eviction",
Event::OriginConnected { .. } => "origin_connected",
Event::OriginDisconnected { .. } => "origin_disconnected",
Event::AllOriginsUnhealthy { .. } => "all_origins_unhealthy",
}
.to_string()
}
fn event_to_json(&self, event: &Event) -> serde_json::Value {
match event {
Event::FileAccessed {
file_id,
origin_id,
path,
offset,
size,
} => serde_json::json!({
"file_id": file_id.0,
"origin_id": origin_id.to_string(),
"path": path.as_str(),
"offset": offset,
"size": size,
}),
Event::FileAdded { path, origin_id } => serde_json::json!({
"path": path.as_str(),
"origin_id": origin_id.to_string(),
}),
Event::FileRemoved { path, file_id } => serde_json::json!({
"path": path.as_str(),
"file_id": file_id.map(|id| id.0),
}),
Event::FileModified { path } => serde_json::json!({
"path": path.as_str(),
}),
Event::SyncStarted { origin_id } => serde_json::json!({
"origin_id": origin_id.to_string(),
}),
Event::SyncCompleted {
origin_id,
files_changed,
} => serde_json::json!({
"origin_id": origin_id.to_string(),
"files_changed": files_changed,
}),
Event::OriginHealthChanged { origin_id, healthy } => serde_json::json!({
"origin_id": origin_id.to_string(),
"healthy": healthy,
}),
Event::CacheEviction { bytes_freed } => serde_json::json!({
"bytes_freed": bytes_freed,
}),
Event::OriginConnected { origin_id } => serde_json::json!({
"origin_id": origin_id.to_string(),
}),
Event::OriginDisconnected { origin_id } => serde_json::json!({
"origin_id": origin_id.to_string(),
}),
Event::AllOriginsUnhealthy { candidate_count } => serde_json::json!({
"candidate_count": candidate_count,
}),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use musicfs_core::OriginId;
#[test]
fn test_webhook_config_defaults() {
let json = r#"{"url": "http://example.com", "events": []}"#;
let config: WebhookConfig = serde_json::from_str(json).unwrap();
assert_eq!(config.retry_count, 3);
assert_eq!(config.timeout_ms, 5000);
}
#[test]
fn test_event_type_name() {
let handler = WebhookHandler::new(vec![]).unwrap();
let event = Event::SyncStarted {
origin_id: OriginId::from("test"),
};
assert_eq!(handler.event_type_name(&event), "sync_started");
}
#[test]
fn test_matches_filter_empty() {
let handler = WebhookHandler::new(vec![]).unwrap();
let config = WebhookConfig {
url: "http://example.com".to_string(),
secret: None,
events: vec![],
retry_count: 3,
timeout_ms: 5000,
};
let event = Event::SyncStarted {
origin_id: OriginId::from("test"),
};
assert!(handler.matches_filter(&event, &config));
}
#[test]
fn test_matches_filter_specific() {
let handler = WebhookHandler::new(vec![]).unwrap();
let config = WebhookConfig {
url: "http://example.com".to_string(),
secret: None,
events: vec!["sync_started".to_string()],
retry_count: 3,
timeout_ms: 5000,
};
let event = Event::SyncStarted {
origin_id: OriginId::from("test"),
};
assert!(handler.matches_filter(&event, &config));
let event2 = Event::SyncCompleted {
origin_id: OriginId::from("test"),
files_changed: 0,
};
assert!(!handler.matches_filter(&event2, &config));
}
}
crates/musicfs-metadata/Cargo.toml
-11
View File
@@ -1,11 +0,0 @@
[package]
name = "musicfs-metadata"
version.workspace = true
edition.workspace = true
[dependencies]
musicfs-core = { path = "../musicfs-core" }
symphonia.workspace = true
thiserror.workspace = true
tracing.workspace = true
image.workspace = true
crates/musicfs-metadata/src/artwork.rs
-116
View File
@@ -1,116 +0,0 @@
use image::ImageFormat;
use std::io::Cursor;
use symphonia::core::meta::Visual;
use tracing::debug;
#[derive(Debug, Clone)]
pub struct Artwork {
pub art_type: ArtType,
pub mime_type: String,
pub width: u32,
pub height: u32,
pub data: Vec<u8>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArtType {
Front,
Back,
Other,
}
#[derive(Debug, Clone, Copy)]
pub enum ArtSize {
Thumbnail,
Medium,
Full,
}
impl ArtSize {
pub fn max_dimension(&self) -> Option<u32> {
match self {
ArtSize::Thumbnail => Some(150),
ArtSize::Medium => Some(300),
ArtSize::Full => None,
}
}
}
pub struct ArtworkExtractor;
impl ArtworkExtractor {
pub fn extract_from_visual(visual: &Visual) -> Option<Artwork> {
let data = visual.data.to_vec();
let img = image::load_from_memory(&data).ok()?;
let art_type = match visual.usage {
Some(symphonia::core::meta::StandardVisualKey::FrontCover) => ArtType::Front,
Some(symphonia::core::meta::StandardVisualKey::BackCover) => ArtType::Back,
_ => ArtType::Other,
};
let mime_type = if visual.media_type.is_empty() {
"image/jpeg".to_string()
} else {
visual.media_type.clone()
};
Some(Artwork {
art_type,
mime_type,
width: img.width(),
height: img.height(),
data,
})
}
pub fn resize(artwork: &Artwork, size: ArtSize) -> Option<Artwork> {
let max_dim = size.max_dimension()?;
if artwork.width <= max_dim && artwork.height <= max_dim {
return Some(artwork.clone());
}
let img = image::load_from_memory(&artwork.data).ok()?;
let resized = img.thumbnail(max_dim, max_dim);
let mut output = Vec::new();
let mut cursor = Cursor::new(&mut output);
resized.write_to(&mut cursor, ImageFormat::Jpeg).ok()?;
debug!(
"Resized artwork from {}x{} to {}x{}",
artwork.width,
artwork.height,
resized.width(),
resized.height()
);
Some(Artwork {
art_type: artwork.art_type,
mime_type: "image/jpeg".to_string(),
width: resized.width(),
height: resized.height(),
data: output,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_art_size_dimensions() {
assert_eq!(ArtSize::Thumbnail.max_dimension(), Some(150));
assert_eq!(ArtSize::Medium.max_dimension(), Some(300));
assert_eq!(ArtSize::Full.max_dimension(), None);
}
#[test]
fn test_art_type_equality() {
assert_eq!(ArtType::Front, ArtType::Front);
assert_ne!(ArtType::Front, ArtType::Back);
}
}
crates/musicfs-metadata/src/lib.rs
-5
View File
@@ -1,5 +0,0 @@
pub mod artwork;
mod parser;
pub use artwork::{ArtSize, ArtType, Artwork, ArtworkExtractor};
pub use parser::MetadataParser;
crates/musicfs-metadata/src/parser.rs
-209
View File
@@ -1,209 +0,0 @@
use musicfs_core::{AudioFormat, AudioMeta, Error, Result};
use std::fs::File;
use std::path::Path;
use symphonia::core::codecs::CODEC_TYPE_NULL;
use symphonia::core::formats::FormatOptions;
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
use tracing::debug;
pub struct MetadataParser;
impl MetadataParser {
pub fn new() -> Self {
Self
}
pub fn parse_file(&self, path: &Path) -> Result<AudioMeta> {
let file = File::open(path)?;
let mss = MediaSourceStream::new(Box::new(file), Default::default());
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
let mut hint = Hint::new();
if !ext.is_empty() {
hint.with_extension(ext);
}
let fmt_opts = FormatOptions::default();
let meta_opts = MetadataOptions::default();
let probed = symphonia::default::get_probe()
.format(&hint, mss, &fmt_opts, &meta_opts)
.map_err(|e| Error::Metadata(format!("Failed to probe format: {}", e)))?;
let mut format = probed.format;
let mut audio_meta = AudioMeta {
format: AudioFormat::from_extension(ext),
..Default::default()
};
if let Some(metadata) = format.metadata().current() {
self.extract_tags(&mut audio_meta, metadata);
}
if let Some(track) = format
.tracks()
.iter()
.find(|t| t.codec_params.codec != CODEC_TYPE_NULL)
{
let params = &track.codec_params;
if let Some(n_frames) = params.n_frames {
if let Some(sample_rate) = params.sample_rate {
audio_meta.duration_ms = Some((n_frames as u64 * 1000) / sample_rate as u64);
audio_meta.sample_rate = Some(sample_rate);
}
}
if let Some(channels) = params.channels {
audio_meta.channels = Some(channels.count() as u32);
}
if let Some(bits_per_sample) = params.bits_per_sample {
audio_meta.bits_per_sample = Some(bits_per_sample);
if let Some(sample_rate) = params.sample_rate {
if let Some(channels) = params.channels {
audio_meta.bitrate =
Some(bits_per_sample * sample_rate * channels.count() as u32 / 1000);
}
}
}
}
debug!(?audio_meta, "Parsed metadata");
Ok(audio_meta)
}
fn extract_tags(
&self,
meta: &mut AudioMeta,
metadata: &symphonia::core::meta::MetadataRevision,
) {
use symphonia::core::meta::StandardTagKey;
for tag in metadata.tags() {
if let Some(std_key) = tag.std_key {
let value = tag.value.to_string();
match std_key {
// Basic metadata
StandardTagKey::TrackTitle => meta.title = Some(value),
StandardTagKey::Artist => meta.artist = Some(value),
StandardTagKey::Album => meta.album = Some(value),
StandardTagKey::AlbumArtist => meta.album_artist = Some(value),
StandardTagKey::Genre => meta.genre = Some(value),
// Track/disc with totals (parse "X/Y" format)
StandardTagKey::TrackNumber => {
let parts: Vec<&str> = value.split('/').collect();
meta.track = parts.first().and_then(|s| s.trim().parse().ok());
if parts.len() > 1 {
meta.track_total = parts.get(1).and_then(|s| s.trim().parse().ok());
}
}
StandardTagKey::DiscNumber => {
let parts: Vec<&str> = value.split('/').collect();
meta.disc = parts.first().and_then(|s| s.trim().parse().ok());
if parts.len() > 1 {
meta.disc_total = parts.get(1).and_then(|s| s.trim().parse().ok());
}
}
StandardTagKey::TrackTotal => {
meta.track_total = value.trim().parse().ok();
}
StandardTagKey::DiscTotal => {
meta.disc_total = value.trim().parse().ok();
}
// Date handling: store full date string, extract year
StandardTagKey::Date | StandardTagKey::ReleaseDate => {
meta.date = Some(value.clone());
meta.year = value.chars().take(4).collect::<String>().parse().ok();
}
// Additional metadata
StandardTagKey::Composer => meta.composer = Some(value),
StandardTagKey::Comment => meta.comment = Some(value),
StandardTagKey::Lyrics => meta.lyrics = Some(value),
StandardTagKey::Copyright => meta.copyright = Some(value),
StandardTagKey::Compilation => {
meta.compilation = Some(value == "1" || value.eq_ignore_ascii_case("true"));
}
StandardTagKey::Encoder => meta.encoder = Some(value),
// Sort keys
StandardTagKey::SortTrackTitle => meta.title_sort = Some(value),
StandardTagKey::SortArtist => meta.artist_sort = Some(value),
StandardTagKey::SortAlbum => meta.album_sort = Some(value),
StandardTagKey::SortAlbumArtist => meta.album_artist_sort = Some(value),
// MusicBrainz IDs
StandardTagKey::MusicBrainzRecordingId => meta.mb_recording_id = Some(value),
StandardTagKey::MusicBrainzAlbumId => meta.mb_album_id = Some(value),
StandardTagKey::MusicBrainzArtistId => meta.mb_artist_id = Some(value),
StandardTagKey::MusicBrainzAlbumArtistId => {
meta.mb_album_artist_id = Some(value)
}
StandardTagKey::MusicBrainzReleaseGroupId => {
meta.mb_release_group_id = Some(value)
}
// ReplayGain (parse as f32, values may have "dB" suffix)
StandardTagKey::ReplayGainTrackGain => {
meta.replaygain_track_gain = parse_replaygain(&value);
}
StandardTagKey::ReplayGainTrackPeak => {
meta.replaygain_track_peak = value.trim().parse().ok();
}
StandardTagKey::ReplayGainAlbumGain => {
meta.replaygain_album_gain = parse_replaygain(&value);
}
StandardTagKey::ReplayGainAlbumPeak => {
meta.replaygain_album_peak = value.trim().parse().ok();
}
_ => {}
}
}
}
}
}
/// Parse ReplayGain value, stripping optional "dB" suffix
fn parse_replaygain(value: &str) -> Option<f32> {
let trimmed = value.trim();
let without_db = trimmed
.strip_suffix("dB")
.or_else(|| trimmed.strip_suffix(" dB"))
.unwrap_or(trimmed);
without_db.trim().parse().ok()
}
impl Default for MetadataParser {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_audio_format_detection() {
assert_eq!(AudioFormat::from_extension("flac"), AudioFormat::Flac);
assert_eq!(AudioFormat::from_extension("mp3"), AudioFormat::Mp3);
assert_eq!(AudioFormat::from_extension("opus"), AudioFormat::Opus);
assert_eq!(AudioFormat::from_extension("ogg"), AudioFormat::Vorbis);
assert_eq!(AudioFormat::from_extension("m4a"), AudioFormat::Aac);
assert_eq!(AudioFormat::from_extension("wav"), AudioFormat::Wav);
}
#[test]
fn test_parser_creation() {
let parser = MetadataParser::new();
let default_parser = MetadataParser::default();
assert!(std::mem::size_of_val(&parser) == std::mem::size_of_val(&default_parser));
}
}
crates/musicfs-origins/Cargo.toml
-23
View File
@@ -1,23 +0,0 @@
[package]
name = "musicfs-origins"
version.workspace = true
edition.workspace = true
[features]
default = []
s3 = []
sftp = []
[dependencies]
musicfs-core = { path = "../musicfs-core" }
async-trait.workspace = true
dashmap.workspace = true
futures.workspace = true
libc.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["fs", "sync", "time"] }
tracing.workspace = true
parking_lot.workspace = true
[dev-dependencies]
tempfile.workspace = true
crates/musicfs-origins/src/failover.rs
-226
View File
@@ -1,226 +0,0 @@
use crate::registry::OriginRegistry;
use crate::traits::Origin;
use musicfs_core::{Error, RealPath, Result};
use std::sync::Arc;
use std::time::Duration;
use tracing::{trace, warn};
#[derive(Debug, Clone)]
pub struct RetryConfig {
pub max_attempts: u32,
pub delays: Vec<Duration>,
}
impl Default for RetryConfig {
fn default() -> Self {
Self::spec_compliant()
}
}
impl RetryConfig {
pub fn spec_compliant() -> Self {
Self {
max_attempts: 3,
delays: vec![
Duration::from_millis(100),
Duration::from_millis(500),
Duration::from_millis(2000),
],
}
}
pub fn with_delays(delays: Vec<Duration>) -> Self {
Self {
max_attempts: delays.len() as u32,
delays,
}
}
fn delay_for_attempt(&self, attempt: u32) -> Duration {
self.delays
.get(attempt as usize)
.copied()
.unwrap_or(*self.delays.last().unwrap_or(&Duration::from_millis(100)))
}
}
pub struct FailoverExecutor {
registry: Arc<OriginRegistry>,
retry_config: RetryConfig,
}
impl FailoverExecutor {
pub fn new(registry: Arc<OriginRegistry>, retry_config: RetryConfig) -> Self {
Self {
registry,
retry_config,
}
}
pub async fn read_with_failover(
&self,
path: &RealPath,
offset: u64,
size: u32,
) -> Result<Vec<u8>> {
let origins = self.registry.route_all(path);
if origins.is_empty() {
if let Some(origin) = self.registry.route_with_fallback(path) {
warn!("No healthy origins, using fallback origin {}", origin.id());
return self
.read_with_retry(&origin, &path.path, offset, size)
.await;
}
return Err(Error::NoOriginAvailable);
}
let mut last_error = None;
for origin in origins {
trace!(origin_id = %origin.id(), "Attempting read from origin");
let start = std::time::Instant::now();
match self
.read_with_retry(&origin, &path.path, offset, size)
.await
{
Ok(data) => {
let latency = start.elapsed().as_millis() as u64;
self.registry.record_latency(origin.id(), latency);
return Ok(data);
}
Err(e) => {
warn!(origin_id = %origin.id(), error = %e, "Origin failed, trying next");
last_error = Some(e);
}
}
}
Err(last_error.unwrap_or(Error::NoOriginAvailable))
}
async fn read_with_retry(
&self,
origin: &Arc<dyn Origin>,
path: &std::path::Path,
offset: u64,
size: u32,
) -> Result<Vec<u8>> {
for attempt in 0..self.retry_config.max_attempts {
match origin.read(path, offset, size).await {
Ok(data) => return Ok(data),
Err(e) if attempt + 1 < self.retry_config.max_attempts => {
let delay = self.retry_config.delay_for_attempt(attempt);
warn!(
origin_id = %origin.id(),
attempt = attempt + 1,
max_attempts = self.retry_config.max_attempts,
error = %e,
delay_ms = delay.as_millis() as u64,
"Retrying read operation"
);
tokio::time::sleep(delay).await;
}
Err(e) => return Err(e),
}
}
Err(Error::MaxRetriesExceeded)
}
pub async fn read_full_with_failover(&self, path: &RealPath) -> Result<Vec<u8>> {
let origins = self.registry.route_all(path);
if origins.is_empty() {
if let Some(origin) = self.registry.route_with_fallback(path) {
warn!(
"No healthy origins for full read, using fallback {}",
origin.id()
);
return self.read_full_with_retry(&origin, &path.path).await;
}
return Err(Error::NoOriginAvailable);
}
let mut last_error = None;
for origin in origins {
trace!(origin_id = %origin.id(), "Attempting full read from origin");
let start = std::time::Instant::now();
match self.read_full_with_retry(&origin, &path.path).await {
Ok(data) => {
let latency = start.elapsed().as_millis() as u64;
self.registry.record_latency(origin.id(), latency);
return Ok(data);
}
Err(e) => {
warn!(origin_id = %origin.id(), error = %e, "Origin failed full read, trying next");
last_error = Some(e);
}
}
}
Err(last_error.unwrap_or(Error::NoOriginAvailable))
}
async fn read_full_with_retry(
&self,
origin: &Arc<dyn Origin>,
path: &std::path::Path,
) -> Result<Vec<u8>> {
for attempt in 0..self.retry_config.max_attempts {
match origin.read_full(path).await {
Ok(data) => return Ok(data),
Err(e) if attempt + 1 < self.retry_config.max_attempts => {
let delay = self.retry_config.delay_for_attempt(attempt);
warn!(
origin_id = %origin.id(),
attempt = attempt + 1,
max_attempts = self.retry_config.max_attempts,
error = %e,
delay_ms = delay.as_millis() as u64,
"Retrying full read operation"
);
tokio::time::sleep(delay).await;
}
Err(e) => return Err(e),
}
}
Err(Error::MaxRetriesExceeded)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_retry_config_default() {
let config = RetryConfig::default();
assert_eq!(config.max_attempts, 3);
assert_eq!(config.delays[0], Duration::from_millis(100));
assert_eq!(config.delays[1], Duration::from_millis(500));
assert_eq!(config.delays[2], Duration::from_millis(2000));
}
#[test]
fn test_delay_for_attempt() {
let config = RetryConfig::spec_compliant();
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(100));
assert_eq!(config.delay_for_attempt(1), Duration::from_millis(500));
assert_eq!(config.delay_for_attempt(2), Duration::from_millis(2000));
assert_eq!(config.delay_for_attempt(10), Duration::from_millis(2000));
}
#[test]
fn test_custom_delays() {
let config =
RetryConfig::with_delays(vec![Duration::from_millis(50), Duration::from_millis(100)]);
assert_eq!(config.max_attempts, 2);
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(50));
assert_eq!(config.delay_for_attempt(1), Duration::from_millis(100));
}
}
crates/musicfs-origins/src/health.rs
-400
View File
@@ -1,400 +0,0 @@
use crate::traits::Origin;
use dashmap::DashMap;
use futures::future::join_all;
use musicfs_core::{Event, EventBus, HealthStatus, OriginId, OriginType};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::mpsc;
use tracing::{debug, info, info_span, warn, Instrument};
pub struct HealthMonitor {
origins: DashMap<OriginId, Arc<dyn Origin>>,
state: DashMap<OriginId, OriginHealthState>,
check_interval: Duration,
default_threshold: u32,
per_type_thresholds: HashMap<OriginType, u32>,
event_bus: Option<Arc<EventBus>>,
}
#[derive(Debug, Clone)]
pub struct OriginHealthState {
pub status: HealthStatus,
pub last_check: Instant,
pub consecutive_failures: u32,
pub last_latency_ms: Option<u64>,
}
impl Default for OriginHealthState {
fn default() -> Self {
Self {
status: HealthStatus::Unknown,
last_check: Instant::now(),
consecutive_failures: 0,
last_latency_ms: None,
}
}
}
#[derive(Debug, Clone)]
pub struct HealthSnapshot {
pub healthy: Vec<OriginId>,
pub degraded: Vec<OriginId>,
pub unhealthy: Vec<OriginId>,
pub failure_counts: HashMap<OriginId, u32>,
}
impl HealthSnapshot {
pub fn is_healthy(&self, id: &OriginId) -> bool {
self.healthy.contains(id)
}
pub fn is_degraded(&self, id: &OriginId) -> bool {
self.degraded.contains(id)
}
pub fn is_unhealthy(&self, id: &OriginId) -> bool {
self.unhealthy.contains(id)
}
pub fn failure_count(&self, id: &OriginId) -> Option<u32> {
self.failure_counts.get(id).copied()
}
pub fn all_unhealthy(&self) -> bool {
self.healthy.is_empty() && self.degraded.is_empty()
}
pub fn total_candidates(&self) -> usize {
self.healthy.len() + self.degraded.len() + self.unhealthy.len()
}
}
impl HealthMonitor {
pub fn new(check_interval: Duration) -> Self {
let mut per_type = HashMap::new();
per_type.insert(OriginType::Local, 1);
per_type.insert(OriginType::Nfs, 3);
per_type.insert(OriginType::Smb, 3);
per_type.insert(OriginType::S3, 3);
per_type.insert(OriginType::Sftp, 3);
Self {
origins: DashMap::new(),
state: DashMap::new(),
check_interval,
default_threshold: 3,
per_type_thresholds: per_type,
event_bus: None,
}
}
pub fn with_threshold(mut self, threshold: u32) -> Self {
self.default_threshold = threshold;
self
}
pub fn with_per_type_thresholds(mut self, thresholds: HashMap<OriginType, u32>) -> Self {
self.per_type_thresholds = thresholds;
self
}
pub fn with_event_bus(mut self, bus: Arc<EventBus>) -> Self {
self.event_bus = Some(bus);
self
}
fn threshold_for(&self, origin_type: OriginType) -> u32 {
self.per_type_thresholds
.get(&origin_type)
.copied()
.unwrap_or(self.default_threshold)
}
pub fn add_origin(&self, origin: Arc<dyn Origin>) {
let id = origin.id().clone();
self.origins.insert(id.clone(), origin);
self.state.insert(id, OriginHealthState::default());
}
pub fn remove_origin(&self, id: &OriginId) {
self.origins.remove(id);
self.state.remove(id);
}
pub fn snapshot(&self) -> HealthSnapshot {
let mut healthy = Vec::new();
let mut degraded = Vec::new();
let mut unhealthy = Vec::new();
let mut failure_counts = HashMap::new();
for entry in self.state.iter() {
let id = entry.key().clone();
failure_counts.insert(id.clone(), entry.value().consecutive_failures);
match entry.value().status {
HealthStatus::Healthy => healthy.push(id),
HealthStatus::Degraded => degraded.push(id),
HealthStatus::Unhealthy => unhealthy.push(id),
HealthStatus::Unknown => degraded.push(id),
}
}
HealthSnapshot {
healthy,
degraded,
unhealthy,
failure_counts,
}
}
pub fn start(self: Arc<Self>) -> HealthCheckHandle {
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
let monitor = self.clone();
let interval_secs = monitor.check_interval.as_secs();
info!(
interval_secs = interval_secs,
origin_count = monitor.origins.len(),
"Health monitor starting"
);
tokio::spawn(
async move {
let mut interval = tokio::time::interval(monitor.check_interval);
loop {
tokio::select! {
_ = interval.tick() => {
monitor.check_all().await;
}
_ = stop_rx.recv() => {
info!("Health monitor stopping");
break;
}
}
}
}
.instrument(info_span!("health_monitor")),
);
HealthCheckHandle { stop_tx }
}
pub async fn check_all(&self) {
let origins: Vec<_> = self
.origins
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let checks: Vec<_> = origins
.iter()
.map(|(id, origin)| self.check_one(id, origin))
.collect();
join_all(checks).await;
}
async fn check_one(&self, id: &OriginId, origin: &Arc<dyn Origin>) {
let start = Instant::now();
let health_timeout = Duration::from_millis(1500);
let status = match tokio::time::timeout(health_timeout, origin.health()).await {
Ok(status) => status,
Err(_) => {
warn!(
origin_id = %id,
timeout_ms = health_timeout.as_millis() as u64,
"Health check timed out"
);
HealthStatus::Unhealthy
}
};
let latency_ms = start.elapsed().as_millis() as u64;
let threshold = self.threshold_for(origin.origin_type());
let prev_healthy = self
.state
.get(id)
.map(|s| s.status == HealthStatus::Healthy)
.unwrap_or(false);
let mut state = self.state.entry(id.clone()).or_default();
match status {
HealthStatus::Healthy => {
if state.status != HealthStatus::Healthy {
info!(
origin_id = %id,
previous_status = ?state.status,
duration_ms = latency_ms,
"Origin health state transition to healthy"
);
}
state.status = HealthStatus::Healthy;
state.consecutive_failures = 0;
}
HealthStatus::Degraded => {
if state.status != HealthStatus::Degraded {
info!(
origin_id = %id,
previous_status = ?state.status,
duration_ms = latency_ms,
"Origin health state transition to degraded"
);
}
state.status = HealthStatus::Degraded;
}
HealthStatus::Unhealthy => {
state.consecutive_failures += 1;
if state.consecutive_failures >= threshold {
if state.status != HealthStatus::Unhealthy {
info!(
origin_id = %id,
previous_status = ?state.status,
consecutive_failures = state.consecutive_failures,
threshold = threshold,
duration_ms = latency_ms,
"Origin health state transition to unhealthy"
);
}
state.status = HealthStatus::Unhealthy;
} else {
debug!(
origin_id = %id,
consecutive_failures = state.consecutive_failures,
threshold = threshold,
"Origin health check failed"
);
state.status = HealthStatus::Degraded;
}
}
HealthStatus::Unknown => {
state.status = HealthStatus::Unknown;
}
}
state.last_check = Instant::now();
state.last_latency_ms = Some(latency_ms);
let now_healthy = state.status == HealthStatus::Healthy;
if prev_healthy != now_healthy {
if let Some(bus) = &self.event_bus {
bus.publish(Event::OriginHealthChanged {
origin_id: id.clone(),
healthy: now_healthy,
});
}
}
}
pub async fn check_now(&self, id: &OriginId) {
if let Some(origin) = self.origins.get(id) {
self.check_one(id, &origin.clone()).await;
}
}
pub fn get_state(&self, id: &OriginId) -> Option<OriginHealthState> {
self.state.get(id).map(|e| e.value().clone())
}
}
pub struct HealthCheckHandle {
stop_tx: mpsc::Sender<()>,
}
impl HealthCheckHandle {
pub async fn stop(self) {
let _ = self.stop_tx.send(()).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::LocalOrigin;
use tempfile::TempDir;
#[tokio::test]
async fn test_health_monitor_basic() {
let monitor = HealthMonitor::new(Duration::from_secs(30));
let dir = TempDir::new().unwrap();
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
monitor.add_origin(origin);
let snapshot = monitor.snapshot();
assert!(!snapshot.is_healthy(&OriginId::from("test")));
}
#[tokio::test]
async fn test_health_check() {
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
let dir = TempDir::new().unwrap();
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
monitor.add_origin(origin);
monitor.check_now(&OriginId::from("test")).await;
let snapshot = monitor.snapshot();
assert!(snapshot.is_healthy(&OriginId::from("test")));
}
#[tokio::test]
async fn test_failure_tracking() {
let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 3);
let monitor =
HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds);
let origin = Arc::new(LocalOrigin::new(
"missing",
std::path::Path::new("/nonexistent"),
));
monitor.add_origin(origin);
monitor.check_now(&OriginId::from("missing")).await;
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.status, HealthStatus::Degraded);
monitor.check_now(&OriginId::from("missing")).await;
monitor.check_now(&OriginId::from("missing")).await;
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
assert_eq!(state.consecutive_failures, 3);
assert_eq!(state.status, HealthStatus::Unhealthy);
}
#[tokio::test]
async fn test_local_origin_threshold_is_one() {
let monitor = HealthMonitor::new(Duration::from_secs(30));
let origin = Arc::new(LocalOrigin::new(
"missing",
std::path::Path::new("/nonexistent"),
));
monitor.add_origin(origin);
monitor.check_now(&OriginId::from("missing")).await;
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.status, HealthStatus::Unhealthy);
}
#[test]
fn test_snapshot_all_unhealthy() {
let snapshot = HealthSnapshot {
healthy: Vec::new(),
degraded: Vec::new(),
unhealthy: vec![OriginId::from("a")],
failure_counts: HashMap::new(),
};
assert!(snapshot.all_unhealthy());
}
}
crates/musicfs-origins/src/lib.rs
-20
View File
@@ -1,20 +0,0 @@
mod failover;
mod health;
mod local;
mod nfs;
mod registry;
mod router;
mod s3;
mod sftp;
mod smb;
mod traits;
pub use failover::{FailoverExecutor, RetryConfig};
pub use health::{HealthCheckHandle, HealthMonitor, HealthSnapshot, OriginHealthState};
pub use local::LocalOrigin;
pub use musicfs_core::OriginType;
pub use nfs::NfsOrigin;
pub use registry::OriginRegistry;
pub use router::{LatencyStats, Router};
pub use smb::SmbOrigin;
pub use traits::{Origin, WatchCallback, WatchEvent, WatchHandle};
crates/musicfs-origins/src/local.rs
-218
View File
@@ -1,218 +0,0 @@
use crate::traits::{Origin, WatchCallback, WatchHandle};
use async_trait::async_trait;
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
use std::path::{Path, PathBuf};
use tokio::fs;
use tokio::io::AsyncRead;
use tracing::debug;
pub struct LocalOrigin {
id: OriginId,
root: PathBuf,
display_name: String,
}
impl LocalOrigin {
pub fn new(id: impl Into<OriginId>, root: impl Into<PathBuf>) -> Self {
let root = root.into();
let display_name = format!("Local: {}", root.display());
Self {
id: id.into(),
root,
display_name,
}
}
fn full_path(&self, path: &Path) -> PathBuf {
if path.as_os_str().is_empty() || path == Path::new("/") {
self.root.clone()
} else {
self.root.join(path.strip_prefix("/").unwrap_or(path))
}
}
}
#[async_trait]
impl Origin for LocalOrigin {
fn id(&self) -> &OriginId {
&self.id
}
fn origin_type(&self) -> OriginType {
OriginType::Local
}
fn display_name(&self) -> &str {
&self.display_name
}
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
let full_path = self.full_path(path);
debug!("LocalOrigin::readdir({:?})", full_path);
let mut entries = Vec::new();
let mut dir = fs::read_dir(&full_path).await?;
while let Some(entry) = dir.next_entry().await? {
let metadata = entry.metadata().await?;
let name = entry.file_name().to_string_lossy().into_owned();
entries.push(DirEntry {
name,
is_dir: metadata.is_dir(),
size: metadata.len(),
mtime: metadata.modified().unwrap_or(std::time::UNIX_EPOCH),
});
}
Ok(entries)
}
async fn stat(&self, path: &Path) -> Result<FileStat> {
let full_path = self.full_path(path);
debug!("LocalOrigin::stat({:?})", full_path);
let metadata = fs::metadata(&full_path).await?;
Ok(FileStat {
size: metadata.len(),
mtime: metadata.modified().unwrap_or(std::time::UNIX_EPOCH),
is_dir: metadata.is_dir(),
})
}
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
use tokio::io::{AsyncReadExt, AsyncSeekExt};
let full_path = self.full_path(path);
debug!(
"LocalOrigin::read({:?}, offset={}, size={})",
full_path, offset, size
);
let mut file = fs::File::open(&full_path).await?;
file.seek(std::io::SeekFrom::Start(offset)).await?;
// FIX: Loop until all requested bytes are read or EOF
// Single read() only returns kernel buffer (~2MB), not full request
let mut buffer = Vec::with_capacity(size as usize);
let mut temp_buf = vec![0u8; 64 * 1024]; // 64KB chunks
let mut total_read = 0usize;
while total_read < size as usize {
let to_read = std::cmp::min(temp_buf.len(), size as usize - total_read);
let n = file.read(&mut temp_buf[..to_read]).await?;
if n == 0 {
break; // EOF
}
buffer.extend_from_slice(&temp_buf[..n]);
total_read += n;
}
Ok(buffer)
}
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
let full_path = self.full_path(path);
debug!("LocalOrigin::read_full({:?})", full_path);
Ok(fs::read(&full_path).await?)
}
async fn exists(&self, path: &Path) -> Result<bool> {
let full_path = self.full_path(path);
Ok(fs::try_exists(&full_path).await?)
}
async fn health(&self) -> HealthStatus {
match fs::try_exists(&self.root).await {
Ok(true) => HealthStatus::Healthy,
Ok(false) => HealthStatus::Unhealthy,
Err(_) => HealthStatus::Unhealthy,
}
}
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
let full_path = self.full_path(path);
let file = fs::File::open(&full_path).await?;
Ok(Box::new(file))
}
async fn watch(&self, path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
debug!("LocalOrigin::watch({:?}) - stub implementation", path);
let (tx, _rx) = tokio::sync::oneshot::channel();
Ok(WatchHandle::new(tx))
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[tokio::test]
async fn test_local_origin_readdir() {
let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("test.txt"), "hello").unwrap();
std::fs::create_dir(dir.path().join("subdir")).unwrap();
let origin = LocalOrigin::new("test", dir.path());
let entries = origin.readdir(Path::new("/")).await.unwrap();
assert_eq!(entries.len(), 2);
assert!(entries.iter().any(|e| e.name == "test.txt" && !e.is_dir));
assert!(entries.iter().any(|e| e.name == "subdir" && e.is_dir));
}
#[tokio::test]
async fn test_local_origin_stat() {
let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
let origin = LocalOrigin::new("test", dir.path());
let stat = origin.stat(Path::new("/test.txt")).await.unwrap();
assert_eq!(stat.size, 11);
assert!(!stat.is_dir);
}
#[tokio::test]
async fn test_local_origin_read() {
let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
let origin = LocalOrigin::new("test", dir.path());
let data = origin.read(Path::new("/test.txt"), 0, 5).await.unwrap();
assert_eq!(data, b"hello");
}
#[tokio::test]
async fn test_local_origin_read_offset() {
let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
let origin = LocalOrigin::new("test", dir.path());
let data = origin.read(Path::new("/test.txt"), 6, 5).await.unwrap();
assert_eq!(data, b"world");
}
#[tokio::test]
async fn test_local_origin_exists() {
let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("test.txt"), "hello").unwrap();
let origin = LocalOrigin::new("test", dir.path());
assert!(origin.exists(Path::new("/test.txt")).await.unwrap());
assert!(!origin.exists(Path::new("/nonexistent.txt")).await.unwrap());
}
#[tokio::test]
async fn test_local_origin_health() {
let dir = TempDir::new().unwrap();
let origin = LocalOrigin::new("test", dir.path());
assert_eq!(origin.health().await, HealthStatus::Healthy);
}
}
crates/musicfs-origins/src/nfs.rs
-162
View File
@@ -1,162 +0,0 @@
use crate::local::LocalOrigin;
use crate::traits::{Origin, WatchCallback, WatchHandle};
use async_trait::async_trait;
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
use std::path::{Path, PathBuf};
use std::time::Duration;
use tokio::time::sleep;
use tracing::{debug, warn};
pub struct NfsOrigin {
inner: LocalOrigin,
max_retries: u32,
display_name: String,
}
impl NfsOrigin {
pub fn new(id: impl Into<OriginId>, mount_point: impl Into<PathBuf>) -> Self {
let mount_point = mount_point.into();
let display_name = format!("NFS: {}", mount_point.display());
Self {
inner: LocalOrigin::new(id, &mount_point),
max_retries: 3,
display_name,
}
}
pub fn with_max_retries(mut self, retries: u32) -> Self {
self.max_retries = retries;
self
}
async fn retry_on_stale<T, F, Fut>(&self, op: F) -> Result<T>
where
F: Fn() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
let mut delay = Duration::from_millis(100);
for attempt in 0..self.max_retries {
match op().await {
Ok(result) => return Ok(result),
Err(e) => {
if let Some(io_err) = e.downcast_io() {
#[cfg(unix)]
if io_err.raw_os_error() == Some(libc::ESTALE) {
warn!(
"NFS stale handle (attempt {}/{}), retrying after {:?}",
attempt + 1,
self.max_retries,
delay
);
sleep(delay).await;
delay *= 2;
continue;
}
}
return Err(e);
}
}
}
Err(musicfs_core::Error::NfsStaleHandle)
}
}
#[async_trait]
impl Origin for NfsOrigin {
fn id(&self) -> &OriginId {
self.inner.id()
}
fn origin_type(&self) -> OriginType {
OriginType::Nfs
}
fn display_name(&self) -> &str {
&self.display_name
}
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
self.retry_on_stale(|| self.inner.readdir(path)).await
}
async fn stat(&self, path: &Path) -> Result<FileStat> {
self.retry_on_stale(|| self.inner.stat(path)).await
}
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
self.retry_on_stale(|| self.inner.read(path, offset, size))
.await
}
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
self.retry_on_stale(|| self.inner.read_full(path)).await
}
async fn exists(&self, path: &Path) -> Result<bool> {
self.retry_on_stale(|| self.inner.exists(path)).await
}
async fn health(&self) -> HealthStatus {
let health_timeout = Duration::from_secs(5);
match tokio::time::timeout(health_timeout, self.inner.stat(Path::new("/"))).await {
Ok(Ok(_)) => HealthStatus::Healthy,
Ok(Err(_)) | Err(_) => HealthStatus::Unhealthy,
}
}
async fn open_read(&self, path: &Path) -> Result<Box<dyn tokio::io::AsyncRead + Send + Unpin>> {
self.inner.open_read(path).await
}
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
debug!("NFS watch - inotify may be unreliable over NFS, consider polling");
self.inner.watch(path, callback).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[tokio::test]
async fn test_nfs_origin_basic() {
let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("test.flac"), b"audio").unwrap();
let origin = NfsOrigin::new("nfs-test", dir.path());
let entries = origin.readdir(Path::new("/")).await.unwrap();
assert_eq!(entries.len(), 1);
let data = origin.read(Path::new("/test.flac"), 0, 5).await.unwrap();
assert_eq!(&data, b"audio");
}
#[tokio::test]
async fn test_nfs_origin_health() {
let dir = TempDir::new().unwrap();
let origin = NfsOrigin::new("nfs-test", dir.path());
assert_eq!(origin.health().await, HealthStatus::Healthy);
}
#[tokio::test]
async fn test_nfs_origin_type() {
let dir = TempDir::new().unwrap();
let origin = NfsOrigin::new("nfs-test", dir.path());
assert_eq!(origin.origin_type(), OriginType::Nfs);
}
#[test]
fn test_retry_uses_fn_not_fnmut() {
fn assert_fn<F: Fn() -> Fut, Fut>(_: F) {}
let closure = || async { Ok::<_, musicfs_core::Error>(()) };
assert_fn(closure);
}
}
crates/musicfs-origins/src/registry.rs
-215
View File
@@ -1,215 +0,0 @@
use crate::health::{HealthMonitor, HealthSnapshot};
use crate::router::Router;
use crate::traits::{Origin, WatchHandle};
use musicfs_core::{OriginId, RealPath};
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::Arc;
use tracing::{info, warn};
pub struct OriginRegistry {
origins: RwLock<HashMap<OriginId, Arc<dyn Origin>>>,
router: Router,
health_monitor: Arc<HealthMonitor>,
watch_handles: RwLock<HashMap<OriginId, Vec<WatchHandle>>>,
}
impl OriginRegistry {
pub fn new(health_monitor: Arc<HealthMonitor>) -> Self {
Self {
origins: RwLock::new(HashMap::new()),
router: Router::new(),
health_monitor,
watch_handles: RwLock::new(HashMap::new()),
}
}
pub fn register(&self, origin: Arc<dyn Origin>, priority: u8) {
let id = origin.id().clone();
info!("Registering origin {} with priority {}", id, priority);
self.router.set_priority(id.clone(), priority);
self.health_monitor.add_origin(origin.clone());
self.origins.write().insert(id, origin);
}
pub fn unregister(&self, id: &OriginId) {
info!("Unregistering origin {}", id);
if let Some(handles) = self.watch_handles.write().remove(id) {
info!("Dropping {} watch handles for origin {}", handles.len(), id);
}
self.origins.write().remove(id);
self.router.remove_priority(id);
self.health_monitor.remove_origin(id);
}
pub fn register_watch(&self, origin_id: &OriginId, handle: WatchHandle) {
self.watch_handles
.write()
.entry(origin_id.clone())
.or_default()
.push(handle);
}
pub fn get(&self, id: &OriginId) -> Option<Arc<dyn Origin>> {
self.origins.read().get(id).cloned()
}
pub fn list(&self) -> Vec<Arc<dyn Origin>> {
self.origins.read().values().cloned().collect()
}
pub fn route(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
let origins = self.origins.read();
let health = self.health_monitor.snapshot();
let candidates: Vec<_> = origins
.iter()
.filter(|(id, _)| self.can_serve(id, path))
.map(|(id, origin)| (id.clone(), origin.clone()))
.collect();
if candidates.is_empty() {
warn!("No origin can serve path: {:?}", path);
return None;
}
let candidate_ids: Vec<_> = candidates.iter().map(|(id, _)| id.clone()).collect();
let selected = self.router.select(&candidate_ids, &health)?;
candidates
.into_iter()
.find(|(id, _)| id == &selected)
.map(|(_, origin)| origin)
}
pub fn route_with_fallback(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
let origins = self.origins.read();
let health = self.health_monitor.snapshot();
let candidates: Vec<_> = origins
.iter()
.filter(|(id, _)| self.can_serve(id, path))
.map(|(id, origin)| (id.clone(), origin.clone()))
.collect();
if candidates.is_empty() {
return None;
}
let candidate_ids: Vec<_> = candidates.iter().map(|(id, _)| id.clone()).collect();
let selected = self.router.select_with_fallback(&candidate_ids, &health)?;
candidates
.into_iter()
.find(|(id, _)| id == &selected)
.map(|(_, origin)| origin)
}
pub fn route_all(&self, path: &RealPath) -> Vec<Arc<dyn Origin>> {
let origins = self.origins.read();
let health = self.health_monitor.snapshot();
let mut result: Vec<_> = origins
.iter()
.filter(|(id, _)| self.can_serve(id, path) && health.is_healthy(id))
.map(|(_, origin)| origin.clone())
.collect();
result.sort_by_key(|o| self.router.get_priority(o.id()));
result
}
fn can_serve(&self, origin_id: &OriginId, path: &RealPath) -> bool {
path.origin_id == *origin_id
}
pub fn health(&self) -> HealthSnapshot {
self.health_monitor.snapshot()
}
pub fn record_latency(&self, id: &OriginId, latency_ms: u64) {
self.router.record_latency(id, latency_ms);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::LocalOrigin;
use std::path::PathBuf;
use std::time::Duration;
use tempfile::TempDir;
#[test]
fn test_register_and_get() {
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
let registry = OriginRegistry::new(monitor);
let dir = TempDir::new().unwrap();
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
registry.register(origin.clone(), 1);
let retrieved = registry.get(&OriginId::from("test"));
assert!(retrieved.is_some());
}
#[test]
fn test_unregister() {
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
let registry = OriginRegistry::new(monitor);
let dir = TempDir::new().unwrap();
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
registry.register(origin, 1);
registry.unregister(&OriginId::from("test"));
assert!(registry.get(&OriginId::from("test")).is_none());
}
#[tokio::test]
async fn test_route_by_priority() {
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
let registry = OriginRegistry::new(monitor.clone());
let dir1 = TempDir::new().unwrap();
let dir2 = TempDir::new().unwrap();
let origin1 = Arc::new(LocalOrigin::new("primary", dir1.path()));
let origin2 = Arc::new(LocalOrigin::new("backup", dir2.path()));
registry.register(origin1, 1);
registry.register(origin2, 2);
monitor.check_now(&OriginId::from("primary")).await;
monitor.check_now(&OriginId::from("backup")).await;
let path = RealPath {
origin_id: OriginId::from("primary"),
path: PathBuf::from("/test.flac"),
};
let routed = registry.route(&path);
assert!(routed.is_some());
assert_eq!(routed.unwrap().id(), &OriginId::from("primary"));
}
#[test]
fn test_list_origins() {
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
let registry = OriginRegistry::new(monitor);
let dir1 = TempDir::new().unwrap();
let dir2 = TempDir::new().unwrap();
registry.register(Arc::new(LocalOrigin::new("a", dir1.path())), 1);
registry.register(Arc::new(LocalOrigin::new("b", dir2.path())), 2);
let list = registry.list();
assert_eq!(list.len(), 2);
}
}
crates/musicfs-origins/src/router.rs
-255
View File
@@ -1,255 +0,0 @@
use crate::health::HealthSnapshot;
use dashmap::DashMap;
use musicfs_core::{Event, EventBus, OriginId};
use std::sync::Arc;
use std::time::Instant;
use tracing::{debug, trace, warn};
pub struct Router {
priorities: DashMap<OriginId, u8>,
latency_stats: DashMap<OriginId, LatencyStats>,
event_bus: Option<Arc<EventBus>>,
}
#[derive(Debug, Clone, Default)]
pub struct LatencyStats {
pub samples: Vec<u64>,
pub p50_ms: u64,
pub p99_ms: u64,
pub last_update: Option<Instant>,
}
impl LatencyStats {
pub fn record(&mut self, latency_ms: u64) {
self.samples.push(latency_ms);
if self.samples.len() > 100 {
self.samples.remove(0);
}
if !self.samples.is_empty() {
let mut sorted = self.samples.clone();
sorted.sort_unstable();
let p50_idx = sorted.len() / 2;
let p99_idx = (sorted.len() * 99) / 100;
self.p50_ms = sorted[p50_idx];
self.p99_ms = sorted.get(p99_idx).copied().unwrap_or(self.p50_ms);
}
self.last_update = Some(Instant::now());
}
}
impl Router {
pub fn new() -> Self {
Self {
priorities: DashMap::new(),
latency_stats: DashMap::new(),
event_bus: None,
}
}
pub fn with_event_bus(mut self, bus: Arc<EventBus>) -> Self {
self.event_bus = Some(bus);
self
}
pub fn set_priority(&self, id: OriginId, priority: u8) {
self.priorities.insert(id, priority);
}
pub fn remove_priority(&self, id: &OriginId) {
self.priorities.remove(id);
self.latency_stats.remove(id);
}
pub fn get_priority(&self, id: &OriginId) -> u8 {
self.priorities.get(id).map(|p| *p).unwrap_or(100)
}
pub fn record_latency(&self, id: &OriginId, latency_ms: u64) {
self.latency_stats
.entry(id.clone())
.or_default()
.record(latency_ms);
}
pub fn select(&self, candidates: &[OriginId], health: &HealthSnapshot) -> Option<OriginId> {
let selected = candidates
.iter()
.filter(|id| health.is_healthy(id))
.min_by_key(|id| {
let priority = self.get_priority(id);
let latency = self.latency_stats.get(*id).map(|s| s.p50_ms).unwrap_or(0);
(priority, latency)
})
.cloned();
if let Some(ref id) = selected {
let priority = self.get_priority(id);
let latency = self.latency_stats.get(id).map(|s| s.p50_ms).unwrap_or(0);
trace!(
origin_id = %id,
priority = priority,
latency_ms = latency,
"Selected healthy origin"
);
}
selected
}
pub fn select_with_fallback(
&self,
candidates: &[OriginId],
health: &HealthSnapshot,
) -> Option<OriginId> {
if let Some(id) = self.select(candidates, health) {
return Some(id);
}
debug!("No healthy origins, trying degraded");
if let Some(id) = candidates
.iter()
.filter(|id| health.is_degraded(id))
.min_by_key(|id| self.get_priority(id))
.cloned()
{
trace!(
origin_id = %id,
priority = self.get_priority(&id),
"Selected degraded origin as fallback"
);
return Some(id);
}
warn!("All origins unhealthy, selecting least-bad by failure count");
if let Some(bus) = &self.event_bus {
bus.publish(Event::AllOriginsUnhealthy {
candidate_count: candidates.len(),
});
}
let selected = candidates
.iter()
.min_by_key(|id| {
let failures = health.failure_count(id).unwrap_or(u32::MAX);
let priority = self.get_priority(id);
(failures, priority)
})
.cloned();
if let Some(ref id) = selected {
let failures = health.failure_count(id).unwrap_or(u32::MAX);
trace!(
origin_id = %id,
failure_count = failures,
priority = self.get_priority(id),
"Selected least-bad unhealthy origin"
);
}
selected
}
}
impl Default for Router {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
fn mock_health(healthy: &[&str], degraded: &[&str]) -> HealthSnapshot {
HealthSnapshot {
healthy: healthy.iter().map(|s| OriginId::from(*s)).collect(),
degraded: degraded.iter().map(|s| OriginId::from(*s)).collect(),
unhealthy: Vec::new(),
failure_counts: HashMap::new(),
}
}
#[test]
fn test_select_by_priority() {
let router = Router::new();
router.set_priority(OriginId::from("high"), 1);
router.set_priority(OriginId::from("low"), 2);
let candidates = vec![OriginId::from("low"), OriginId::from("high")];
let health = mock_health(&["high", "low"], &[]);
let selected = router.select(&candidates, &health);
assert_eq!(selected, Some(OriginId::from("high")));
}
#[test]
fn test_select_skips_unhealthy() {
let router = Router::new();
router.set_priority(OriginId::from("high"), 1);
router.set_priority(OriginId::from("low"), 2);
let candidates = vec![OriginId::from("high"), OriginId::from("low")];
let health = mock_health(&["low"], &[]);
let selected = router.select(&candidates, &health);
assert_eq!(selected, Some(OriginId::from("low")));
}
#[test]
fn test_latency_affects_tiebreak() {
let router = Router::new();
router.set_priority(OriginId::from("a"), 1);
router.set_priority(OriginId::from("b"), 1);
router.record_latency(&OriginId::from("a"), 100);
router.record_latency(&OriginId::from("b"), 10);
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
let health = mock_health(&["a", "b"], &[]);
let selected = router.select(&candidates, &health);
assert_eq!(selected, Some(OriginId::from("b")));
}
#[test]
fn test_fallback_to_degraded() {
let router = Router::new();
router.set_priority(OriginId::from("a"), 1);
router.set_priority(OriginId::from("b"), 2);
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
let health = mock_health(&[], &["b"]);
let selected = router.select_with_fallback(&candidates, &health);
assert_eq!(selected, Some(OriginId::from("b")));
}
#[test]
fn test_fallback_least_bad() {
let router = Router::new();
router.set_priority(OriginId::from("a"), 1);
router.set_priority(OriginId::from("b"), 2);
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
let mut failure_counts = HashMap::new();
failure_counts.insert(OriginId::from("a"), 5);
failure_counts.insert(OriginId::from("b"), 2);
let health = HealthSnapshot {
healthy: Vec::new(),
degraded: Vec::new(),
unhealthy: vec![OriginId::from("a"), OriginId::from("b")],
failure_counts,
};
let selected = router.select_with_fallback(&candidates, &health);
assert_eq!(selected, Some(OriginId::from("b")));
}
}
crates/musicfs-origins/src/s3.rs
-49
View File
@@ -1,49 +0,0 @@
//! S3-compatible object storage origin
//!
//! This module is feature-gated behind the `s3` feature to avoid heavy AWS SDK dependencies.
//!
//! # Oracle Security Fixes (MUST IMPLEMENT)
//!
//! 1. **Range EOF** - Clamp range to `min(requested_end, file_size)` to avoid 416 errors
//! 2. **Health check** - Use `head_bucket` not `list_objects_v2` (lighter operation)
//! 3. **Timeout handling** - Wrap all remote calls with `tokio::time::timeout(30s)`
//!
//! # Example Implementation (when feature enabled)
//!
//! ```ignore
//! async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
//! // Oracle fix: Clamp range to file size to avoid 416 error
//! let file_size = self.stat(path).await?.size;
//! let end = std::cmp::min(offset + size as u64, file_size).saturating_sub(1);
//!
//! if offset >= file_size {
//! return Ok(Vec::new()); // EOF
//! }
//!
//! let range = format!("bytes={}-{}", offset, end);
//!
//! // Oracle fix: Add timeout to prevent hung connections
//! let resp = tokio::time::timeout(
//! Duration::from_secs(30),
//! self.client.get_object().bucket(&self.bucket).key(&key).range(range).send()
//! )
//! .await
//! .map_err(|_| Error::Timeout("S3 read timed out".into()))?
//! .map_err(|e| Error::S3(e.to_string()))?;
//!
//! // ...
//! }
//!
//! async fn health(&self) -> HealthStatus {
//! // Oracle fix: Use head_bucket instead of list_objects_v2 (lighter)
//! match self.client.head_bucket().bucket(&self.bucket).send().await {
//! Ok(_) => HealthStatus::Healthy,
//! Err(_) => HealthStatus::Unhealthy,
//! }
//! }
//! ```
#[cfg(feature = "s3")]
mod implementation {
// Full S3 implementation would go here when aws-sdk-s3 is enabled
}
crates/musicfs-origins/src/sftp.rs
-12
View File
@@ -1,12 +0,0 @@
#![allow(dead_code)]
//! SFTP origin - feature-gated to avoid russh/deadpool dependencies
#[cfg(feature = "sftp")]
mod implementation {
// Full SFTP implementation with connection pooling
// Oracle fixes to implement:
// 1. Use deadpool connection pool, not Arc<Mutex<SftpSession>>
// 2. Verify SSH host keys against ~/.ssh/known_hosts
// 3. Wrap all operations with tokio::time::timeout(30s)
// 4. Cap open_read to actual file size, not u32::MAX
}
crates/musicfs-origins/src/smb.rs
-156
View File
@@ -1,156 +0,0 @@
use crate::local::LocalOrigin;
use crate::traits::{Origin, WatchCallback, WatchHandle};
use async_trait::async_trait;
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
use std::future::Future;
use std::path::{Path, PathBuf};
use tracing::{debug, warn};
pub struct SmbOrigin {
inner: LocalOrigin,
share_path: String,
}
impl SmbOrigin {
pub fn from_mount(
id: impl Into<OriginId>,
mount_point: impl Into<PathBuf>,
share_path: impl Into<String>,
) -> Self {
let mount_point = mount_point.into();
let share_path = share_path.into();
Self {
inner: LocalOrigin::new(id, &mount_point),
share_path,
}
}
pub async fn is_mounted(&self) -> bool {
self.inner.exists(Path::new("/")).await.unwrap_or(false)
}
async fn retry_on_disconnect<T, F, Fut>(&self, op: F) -> Result<T>
where
F: Fn() -> Fut,
Fut: Future<Output = Result<T>>,
{
const MAX_RETRIES: u32 = 3;
for attempt in 0..MAX_RETRIES {
match op().await {
Ok(val) => return Ok(val),
Err(e) => {
if Self::is_enotconn(&e) && attempt < MAX_RETRIES - 1 {
debug!(attempt, "SMB ENOTCONN, retrying");
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
continue;
}
return Err(e);
}
}
}
unreachable!()
}
#[cfg(unix)]
fn is_enotconn(err: &musicfs_core::Error) -> bool {
if let musicfs_core::Error::Io(io_err) = err {
io_err.raw_os_error() == Some(libc::ENOTCONN)
} else {
false
}
}
#[cfg(not(unix))]
fn is_enotconn(_err: &musicfs_core::Error) -> bool {
false
}
}
#[async_trait]
impl Origin for SmbOrigin {
fn id(&self) -> &OriginId {
self.inner.id()
}
fn origin_type(&self) -> OriginType {
OriginType::Smb
}
fn display_name(&self) -> &str {
&self.share_path
}
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
self.retry_on_disconnect(|| self.inner.readdir(path)).await
}
async fn stat(&self, path: &Path) -> Result<FileStat> {
self.retry_on_disconnect(|| self.inner.stat(path)).await
}
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
self.retry_on_disconnect(|| self.inner.read(path, offset, size))
.await
}
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
self.retry_on_disconnect(|| self.inner.read_full(path))
.await
}
async fn exists(&self, path: &Path) -> Result<bool> {
self.retry_on_disconnect(|| self.inner.exists(path)).await
}
async fn health(&self) -> HealthStatus {
let health_timeout = std::time::Duration::from_secs(5);
match tokio::time::timeout(health_timeout, self.is_mounted()).await {
Ok(true) => HealthStatus::Healthy,
Ok(false) | Err(_) => HealthStatus::Unhealthy,
}
}
async fn open_read(&self, path: &Path) -> Result<Box<dyn tokio::io::AsyncRead + Send + Unpin>> {
self.inner.open_read(path).await
}
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
warn!("SMB watch using inotify - may be unreliable. Consider polling for remote mounts.");
self.inner.watch(path, callback).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[tokio::test]
async fn test_smb_origin_basic() {
let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("test.flac"), b"audio").unwrap();
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/share");
let entries = origin.readdir(Path::new("/")).await.unwrap();
assert_eq!(entries.len(), 1);
}
#[tokio::test]
async fn test_smb_origin_type() {
let dir = TempDir::new().unwrap();
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/share");
assert_eq!(origin.origin_type(), OriginType::Smb);
}
#[tokio::test]
async fn test_smb_display_name() {
let dir = TempDir::new().unwrap();
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/music");
assert_eq!(origin.display_name(), "//server/music");
}
}
crates/musicfs-origins/src/traits.rs
-49
View File
@@ -1,49 +0,0 @@
use async_trait::async_trait;
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
use std::path::{Path, PathBuf};
use tokio::io::AsyncRead;
#[async_trait]
pub trait Origin: Send + Sync {
fn id(&self) -> &OriginId;
fn origin_type(&self) -> OriginType;
fn display_name(&self) -> &str;
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>>;
async fn stat(&self, path: &Path) -> Result<FileStat>;
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>>;
/// Read entire file content (for CDC chunking of files <4GB)
async fn read_full(&self, path: &Path) -> Result<Vec<u8>>;
async fn exists(&self, path: &Path) -> Result<bool>;
async fn health(&self) -> HealthStatus;
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>>;
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle>;
}
pub type WatchCallback = Box<dyn Fn(WatchEvent) + Send + Sync>;
pub struct WatchHandle {
_cancel: tokio::sync::oneshot::Sender<()>,
}
impl WatchHandle {
pub fn new(cancel: tokio::sync::oneshot::Sender<()>) -> Self {
Self { _cancel: cancel }
}
}
#[derive(Debug, Clone)]
pub enum WatchEvent {
Created(PathBuf),
Modified(PathBuf),
Deleted(PathBuf),
}
crates/musicfs-plugins/Cargo.toml
-23
View File
@@ -1,23 +0,0 @@
[package]
name = "musicfs-plugins"
version.workspace = true
edition.workspace = true
[dependencies]
musicfs-core = { path = "../musicfs-core" }
async-trait.workspace = true
tokio.workspace = true
thiserror.workspace = true
serde.workspace = true
serde_json.workspace = true
tracing.workspace = true
libloading = "0.8"
wasmtime = { version = "19", optional = true }
semver = "1"
[features]
default = []
wasm = ["wasmtime"]
[dev-dependencies]
tempfile.workspace = true
crates/musicfs-plugins/src/error.rs
-42
View File
@@ -1,42 +0,0 @@
use thiserror::Error;
#[derive(Debug, Error)]
pub enum PluginError {
#[error("Plugin not found: {0}")]
NotFound(String),
#[error("Plugin load failed: {0}")]
LoadFailed(String),
#[error("Plugin initialization failed: {0}")]
InitFailed(String),
#[error("Plugin API version mismatch: expected {expected}, got {actual}")]
VersionMismatch { expected: String, actual: String },
#[error("Plugin already loaded: {0}")]
AlreadyLoaded(String),
#[error("Plugin symbol not found: {0}")]
SymbolNotFound(String),
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("Plugin execution error: {0}")]
Execution(String),
#[error("Plugin shutdown error: {0}")]
Shutdown(String),
#[error("Configuration error: {0}")]
Config(String),
#[error("WASM error: {0}")]
Wasm(String),
#[error("Resource limit exceeded: {0}")]
ResourceLimit(String),
}
pub type Result<T> = std::result::Result<T, PluginError>;
crates/musicfs-plugins/src/lib.rs
-15
View File
@@ -1,15 +0,0 @@
pub mod error;
pub mod manager;
pub mod native;
pub mod traits;
pub mod wasm;
pub use error::{PluginError, Result};
pub use manager::{PluginConfig, PluginEntry, PluginManager, WasmConfig};
pub use native::NativePluginHost;
pub use traits::{
ExternalMetadata, FormatPlugin, MetadataPlugin, MetadataQuery, MetadataQueryType,
OriginDirEntry, OriginHealth, OriginInstance, OriginPlugin, OriginStat, Plugin, PluginId,
PluginInfo, PluginType, WatchEvent, WatchHandle, PLUGIN_API_VERSION,
};
pub use wasm::{ResourceLimits, WasmPluginHost};
crates/musicfs-plugins/src/manager.rs
-346
View File
@@ -1,346 +0,0 @@
use crate::error::{PluginError, Result};
use crate::native::NativePluginHost;
use crate::traits::{Plugin, PluginId, PluginInfo, PluginType};
use crate::wasm::{ResourceLimits, WasmPluginHost};
use serde::Deserialize;
use serde_json::Value;
use std::collections::HashMap;
use std::path::PathBuf;
use tracing::{debug, info};
#[derive(Debug, Clone, Deserialize, Default)]
pub struct PluginConfig {
#[serde(default)]
pub enabled: bool,
#[serde(default)]
pub search_paths: Vec<PathBuf>,
#[serde(default)]
pub plugins: HashMap<String, PluginEntry>,
#[serde(default)]
pub wasm: WasmConfig,
}
#[derive(Debug, Clone, Deserialize)]
pub struct PluginEntry {
pub path: PathBuf,
#[serde(default)]
pub enabled: bool,
#[serde(default)]
pub config: Value,
}
impl Default for PluginEntry {
fn default() -> Self {
Self {
path: PathBuf::new(),
enabled: true,
config: Value::Null,
}
}
}
#[derive(Debug, Clone, Deserialize, Default)]
pub struct WasmConfig {
#[serde(default)]
pub enabled: bool,
#[serde(default)]
pub max_memory_mb: Option<u32>,
#[serde(default)]
pub max_cpu_time_ms: Option<u32>,
}
pub struct PluginManager {
native_host: NativePluginHost,
wasm_host: WasmPluginHost,
registry: PluginRegistry,
config: PluginConfig,
}
struct PluginRegistry {
origin_plugins: Vec<PluginId>,
metadata_plugins: Vec<PluginId>,
format_plugins: Vec<PluginId>,
}
impl PluginRegistry {
fn new() -> Self {
Self {
origin_plugins: Vec::new(),
metadata_plugins: Vec::new(),
format_plugins: Vec::new(),
}
}
fn register(&mut self, id: PluginId, plugin_type: PluginType) {
match plugin_type {
PluginType::Origin => {
if !self.origin_plugins.contains(&id) {
self.origin_plugins.push(id);
}
}
PluginType::Metadata => {
if !self.metadata_plugins.contains(&id) {
self.metadata_plugins.push(id);
}
}
PluginType::Format => {
if !self.format_plugins.contains(&id) {
self.format_plugins.push(id);
}
}
}
}
fn unregister(&mut self, id: PluginId) {
self.origin_plugins.retain(|&x| x != id);
self.metadata_plugins.retain(|&x| x != id);
self.format_plugins.retain(|&x| x != id);
}
}
impl PluginManager {
pub fn new() -> Result<Self> {
Ok(Self {
native_host: NativePluginHost::new(),
wasm_host: WasmPluginHost::new()?,
registry: PluginRegistry::new(),
config: PluginConfig::default(),
})
}
pub fn init(config: &PluginConfig) -> Result<Self> {
let mut manager = Self::new()?;
manager.config = config.clone();
if !config.enabled {
info!("Plugin system disabled");
return Ok(manager);
}
info!("Initializing plugin system");
for path in &config.search_paths {
manager.native_host.add_search_path(path.clone());
}
if config.wasm.enabled {
let limits = ResourceLimits {
max_memory_mb: config.wasm.max_memory_mb.unwrap_or(64),
max_cpu_time_ms: config.wasm.max_cpu_time_ms.unwrap_or(5000),
..Default::default()
};
manager.wasm_host.set_limits(limits);
}
for (name, entry) in &config.plugins {
if !entry.enabled {
debug!("Skipping disabled plugin: {}", name);
continue;
}
match manager.load_and_init(&entry.path, &entry.config) {
Ok(id) => {
info!("Loaded plugin '{}' with id {:?}", name, id);
}
Err(e) => {
tracing::warn!("Failed to load plugin '{}': {}", name, e);
}
}
}
let discovered = manager.native_host.discover()?;
for id in discovered {
if let Some(info) = manager.native_host.list().iter().find(|i| i.id == id) {
manager.registry.register(id, info.plugin_type);
}
}
Ok(manager)
}
pub fn load_and_init(&mut self, path: &PathBuf, config: &Value) -> Result<PluginId> {
let id = self.native_host.load(path)?;
if let Some(plugin) = self.native_host.get_mut(id) {
plugin.init(config.clone())?;
}
if let Some(info) = self.native_host.list().iter().find(|i| i.id == id) {
self.registry.register(id, info.plugin_type);
}
Ok(id)
}
pub fn load_wasm(&mut self, wasm_bytes: &[u8]) -> Result<PluginId> {
if !self.config.wasm.enabled {
return Err(PluginError::Config("WASM plugins disabled".to_string()));
}
self.wasm_host.load(wasm_bytes)
}
pub fn unload(&mut self, id: PluginId) -> Result<()> {
self.registry.unregister(id);
if let Err(native_err) = self.native_host.unload(id) {
if let Err(wasm_err) = self.wasm_host.unload(id) {
return Err(PluginError::NotFound(format!(
"Plugin {:?} not found in native ({}) or WASM ({}) hosts",
id, native_err, wasm_err
)));
}
}
Ok(())
}
pub fn reload(&mut self, id: PluginId) -> Result<()> {
self.native_host.reload(id)
}
pub fn reload_all(&mut self) -> Result<()> {
let ids: Vec<PluginId> = self.native_host.list().iter().map(|i| i.id).collect();
for id in ids {
self.reload(id)?;
}
Ok(())
}
pub fn list(&self) -> Vec<PluginInfo> {
let mut all = self.native_host.list();
for (id, name) in self.wasm_host.list() {
all.push(PluginInfo {
id,
name: name.to_string(),
version: semver::Version::new(0, 0, 0),
description: "WASM plugin".to_string(),
plugin_type: PluginType::Origin,
});
}
all
}
pub fn get(&self, id: PluginId) -> Option<&dyn Plugin> {
self.native_host.get(id)
}
pub fn get_mut(&mut self, id: PluginId) -> Option<&mut dyn Plugin> {
self.native_host.get_mut(id)
}
pub fn origin_plugin_ids(&self) -> &[PluginId] {
&self.registry.origin_plugins
}
pub fn metadata_plugin_ids(&self) -> &[PluginId] {
&self.registry.metadata_plugins
}
pub fn format_plugin_ids(&self) -> &[PluginId] {
&self.registry.format_plugins
}
pub fn shutdown(&mut self) -> Result<()> {
info!("Shutting down plugin system");
let ids: Vec<PluginId> = self.list().iter().map(|i| i.id).collect();
for id in ids {
if let Err(e) = self.unload(id) {
tracing::warn!("Failed to unload plugin {:?}: {}", id, e);
}
}
Ok(())
}
}
impl Default for PluginManager {
fn default() -> Self {
Self::new().expect("Failed to create plugin manager")
}
}
impl Drop for PluginManager {
fn drop(&mut self) {
debug!(plugin_count = self.list().len(), "PluginManager dropping");
let _ = self.shutdown();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_plugin_manager_new() {
let manager = PluginManager::new();
assert!(manager.is_ok());
}
#[test]
fn test_plugin_manager_disabled() {
let config = PluginConfig {
enabled: false,
..Default::default()
};
let manager = PluginManager::init(&config);
assert!(manager.is_ok());
let manager = manager.unwrap();
assert!(manager.list().is_empty());
}
#[test]
fn test_registry() {
let mut registry = PluginRegistry::new();
let id1 = PluginId::new(1);
let id2 = PluginId::new(2);
registry.register(id1, PluginType::Origin);
registry.register(id2, PluginType::Metadata);
assert_eq!(registry.origin_plugins.len(), 1);
assert_eq!(registry.metadata_plugins.len(), 1);
registry.unregister(id1);
assert!(registry.origin_plugins.is_empty());
}
#[test]
fn test_plugin_config_deserialize() {
let json = r#"{
"enabled": true,
"search_paths": ["/usr/lib/musicfs/plugins"],
"plugins": {
"example": {
"path": "/path/to/plugin.so",
"enabled": true,
"config": {"key": "value"}
}
},
"wasm": {
"enabled": false
}
}"#;
let config: PluginConfig = serde_json::from_str(json).unwrap();
assert!(config.enabled);
assert_eq!(config.search_paths.len(), 1);
assert!(config.plugins.contains_key("example"));
}
}
crates/musicfs-plugins/src/native.rs
-300
View File
@@ -1,300 +0,0 @@
use crate::error::{PluginError, Result};
use crate::traits::{Plugin, PluginId, PluginInfo, PluginType, PLUGIN_API_VERSION};
use libloading::{Library, Symbol};
use semver::Version;
use std::collections::HashMap;
use std::ffi::CStr;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use tracing::{debug, info, warn};
static NEXT_PLUGIN_ID: AtomicU64 = AtomicU64::new(1);
fn next_plugin_id() -> PluginId {
PluginId::new(NEXT_PLUGIN_ID.fetch_add(1, Ordering::SeqCst))
}
struct LoadedPlugin {
id: PluginId,
path: PathBuf,
library: Library,
instance: Box<dyn Plugin>,
plugin_type: PluginType,
}
pub struct NativePluginHost {
plugins: HashMap<PluginId, LoadedPlugin>,
search_paths: Vec<PathBuf>,
}
impl NativePluginHost {
pub fn new() -> Self {
Self {
plugins: HashMap::new(),
search_paths: Vec::new(),
}
}
pub fn add_search_path(&mut self, path: PathBuf) {
if !self.search_paths.contains(&path) {
self.search_paths.push(path);
}
}
pub fn load(&mut self, path: &Path) -> Result<PluginId> {
let canonical = path.canonicalize().map_err(|e| {
PluginError::LoadFailed(format!("Cannot resolve path {}: {}", path.display(), e))
})?;
for plugin in self.plugins.values() {
if plugin.path == canonical {
return Err(PluginError::AlreadyLoaded(canonical.display().to_string()));
}
}
info!("Loading native plugin from {:?}", canonical);
let library = unsafe {
Library::new(&canonical)
.map_err(|e| PluginError::LoadFailed(format!("Failed to load library: {}", e)))?
};
self.verify_api_version(&library)?;
let instance = self.create_plugin_instance(&library)?;
let id = next_plugin_id();
let plugin_type = self.detect_plugin_type(&*instance);
debug!(
"Loaded plugin '{}' v{} as {:?}",
instance.name(),
instance.version(),
plugin_type
);
self.plugins.insert(
id,
LoadedPlugin {
id,
path: canonical,
library,
instance,
plugin_type,
},
);
Ok(id)
}
pub fn unload(&mut self, id: PluginId) -> Result<()> {
let mut plugin = self
.plugins
.remove(&id)
.ok_or_else(|| PluginError::NotFound(format!("Plugin {:?}", id)))?;
info!("Unloading plugin '{}'", plugin.instance.name());
plugin.instance.shutdown()?;
drop(plugin.instance);
drop(plugin.library);
Ok(())
}
pub fn reload(&mut self, id: PluginId) -> Result<()> {
let plugin = self
.plugins
.get(&id)
.ok_or_else(|| PluginError::NotFound(format!("Plugin {:?}", id)))?;
let path = plugin.path.clone();
info!("Hot-reloading plugin from {:?}", path);
self.unload(id)?;
let new_id = self.load(&path)?;
if let Some(plugin) = self.plugins.remove(&new_id) {
self.plugins.insert(id, LoadedPlugin { id, ..plugin });
}
Ok(())
}
pub fn get(&self, id: PluginId) -> Option<&dyn Plugin> {
self.plugins.get(&id).map(|p| &*p.instance as &dyn Plugin)
}
pub fn get_mut(&mut self, id: PluginId) -> Option<&mut dyn Plugin> {
self.plugins
.get_mut(&id)
.map(|p| &mut *p.instance as &mut dyn Plugin)
}
pub fn list(&self) -> Vec<PluginInfo> {
self.plugins
.values()
.map(|p| PluginInfo {
id: p.id,
name: p.instance.name().to_string(),
version: p.instance.version(),
description: p.instance.description().to_string(),
plugin_type: p.plugin_type,
})
.collect()
}
pub fn find_by_name(&self, name: &str) -> Option<PluginId> {
self.plugins
.iter()
.find(|(_, p)| p.instance.name() == name)
.map(|(id, _)| *id)
}
pub fn discover(&mut self) -> Result<Vec<PluginId>> {
let mut loaded = Vec::new();
for search_path in self.search_paths.clone() {
if !search_path.exists() {
continue;
}
let entries = std::fs::read_dir(&search_path).map_err(|e| {
PluginError::LoadFailed(format!(
"Cannot read plugin directory {}: {}",
search_path.display(),
e
))
})?;
for entry in entries.flatten() {
let path = entry.path();
if self.is_plugin_library(&path) {
match self.load(&path) {
Ok(id) => loaded.push(id),
Err(e) => {
warn!("Failed to load plugin {:?}: {}", path, e);
}
}
}
}
}
Ok(loaded)
}
fn verify_api_version(&self, library: &Library) -> Result<()> {
let version_fn: Symbol<unsafe extern "C" fn() -> *const std::ffi::c_char> = unsafe {
library.get(b"musicfs_plugin_api_version").map_err(|_| {
PluginError::SymbolNotFound("musicfs_plugin_api_version".to_string())
})?
};
let version_ptr = unsafe { version_fn() };
let version_str = unsafe { CStr::from_ptr(version_ptr) }
.to_str()
.map_err(|_| PluginError::VersionMismatch {
expected: PLUGIN_API_VERSION.to_string(),
actual: "<invalid UTF-8>".to_string(),
})?;
let plugin_version =
Version::parse(version_str).map_err(|_| PluginError::VersionMismatch {
expected: PLUGIN_API_VERSION.to_string(),
actual: version_str.to_string(),
})?;
let expected_version = Version::parse(PLUGIN_API_VERSION).unwrap();
if plugin_version.major != expected_version.major {
return Err(PluginError::VersionMismatch {
expected: PLUGIN_API_VERSION.to_string(),
actual: version_str.to_string(),
});
}
Ok(())
}
fn create_plugin_instance(&self, library: &Library) -> Result<Box<dyn Plugin>> {
let create_fn: Symbol<unsafe extern "C" fn() -> *mut dyn Plugin> = unsafe {
library
.get(b"musicfs_plugin_create")
.map_err(|_| PluginError::SymbolNotFound("musicfs_plugin_create".to_string()))?
};
let plugin_ptr = unsafe { create_fn() };
if plugin_ptr.is_null() {
return Err(PluginError::LoadFailed(
"Plugin factory returned null".to_string(),
));
}
let plugin = unsafe { Box::from_raw(plugin_ptr) };
Ok(plugin)
}
fn detect_plugin_type(&self, plugin: &dyn Plugin) -> PluginType {
plugin.plugin_type()
}
fn is_plugin_library(&self, path: &Path) -> bool {
let extension = path.extension().and_then(|e| e.to_str());
match extension {
Some("so") => true,
Some("dylib") => true,
Some("dll") => true,
_ => false,
}
}
}
impl Default for NativePluginHost {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_native_host_creation() {
let host = NativePluginHost::new();
assert!(host.plugins.is_empty());
assert!(host.search_paths.is_empty());
}
#[test]
fn test_add_search_path() {
let mut host = NativePluginHost::new();
host.add_search_path(PathBuf::from("/usr/lib/musicfs/plugins"));
host.add_search_path(PathBuf::from("/usr/lib/musicfs/plugins"));
assert_eq!(host.search_paths.len(), 1);
}
#[test]
fn test_is_plugin_library() {
let host = NativePluginHost::new();
assert!(host.is_plugin_library(Path::new("plugin.so")));
assert!(host.is_plugin_library(Path::new("plugin.dylib")));
assert!(host.is_plugin_library(Path::new("plugin.dll")));
assert!(!host.is_plugin_library(Path::new("plugin.txt")));
assert!(!host.is_plugin_library(Path::new("plugin")));
}
#[test]
fn test_load_nonexistent() {
let mut host = NativePluginHost::new();
let result = host.load(Path::new("/nonexistent/plugin.so"));
assert!(result.is_err());
}
}
crates/musicfs-plugins/src/traits.rs
-339
View File
@@ -1,339 +0,0 @@
//! Plugin trait definitions (FR-23.1-23.4)
//!
//! Per architecture.md section 4.3.4:
//! - Plugin trait: Base interface for all plugins
//! - OriginPlugin: Creates Origin instances for storage backends
//! - MetadataPlugin: Provides external metadata lookup
//! - FormatPlugin: Handles custom audio format parsing
use crate::error::Result;
use async_trait::async_trait;
use musicfs_core::AudioMeta;
use semver::Version;
use serde_json::Value;
use std::io::Read;
/// Current plugin API version
pub const PLUGIN_API_VERSION: &str = "0.1.0";
/// Unique identifier for a loaded plugin
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct PluginId(pub u64);
impl PluginId {
pub fn new(id: u64) -> Self {
Self(id)
}
}
/// Plugin metadata returned by plugins
#[derive(Debug, Clone)]
pub struct PluginInfo {
pub id: PluginId,
pub name: String,
pub version: Version,
pub description: String,
pub plugin_type: PluginType,
}
/// Type of plugin
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PluginType {
Origin,
Metadata,
Format,
}
/// Base plugin interface (FR-23.1)
///
/// All plugins must implement this trait. It provides:
/// - Plugin identification (name, version)
/// - Lifecycle management (init, shutdown)
pub trait Plugin: Send + Sync {
/// Unique plugin name (e.g., "s3-origin", "musicbrainz-metadata")
fn name(&self) -> &str;
/// Plugin version following semver
fn version(&self) -> Version;
/// Human-readable description
fn description(&self) -> &str {
""
}
/// Plugin type for registry categorization
fn plugin_type(&self) -> PluginType;
/// Initialize plugin with configuration
///
/// Called once after loading. The config value contains
/// plugin-specific configuration from the main config file.
fn init(&mut self, config: Value) -> Result<()>;
/// Shutdown plugin and release resources
///
/// Called before unloading. Plugins should clean up any
/// resources (connections, file handles, etc).
fn shutdown(&mut self) -> Result<()>;
}
/// Origin plugin interface (FR-23.3)
///
/// Per architecture.md section 4.3.4:
/// Origin plugins create `Box<dyn Origin>` instances for custom storage backends.
///
/// Example use cases:
/// - Google Drive origin
/// - Dropbox origin
/// - Custom NAS protocol
#[async_trait]
pub trait OriginPlugin: Plugin {
/// Origin type identifier (e.g., "gdrive", "dropbox")
fn origin_type(&self) -> &str;
/// Create a new Origin instance with the given configuration
///
/// The config contains origin-specific settings (credentials, paths, etc).
/// Returns a boxed Origin that can be used by the OriginRouter.
async fn create_origin(&self, id: &str, config: Value) -> Result<Box<dyn OriginInstance>>;
}
/// Instance created by OriginPlugin
///
/// This is a simplified async interface that maps to the full Origin trait.
/// The plugin host wraps this to provide the full Origin implementation.
#[async_trait]
pub trait OriginInstance: Send + Sync {
/// List directory contents
async fn readdir(&self, path: &str) -> Result<Vec<OriginDirEntry>>;
/// Get file/directory stats
async fn stat(&self, path: &str) -> Result<OriginStat>;
/// Read file data
async fn read(&self, path: &str, offset: u64, size: u32) -> Result<Vec<u8>>;
/// Check if path exists
async fn exists(&self, path: &str) -> Result<bool>;
/// Health check
async fn health(&self) -> OriginHealth;
/// Watch path for changes (FR-10.2)
async fn watch(
&self,
path: &str,
callback: Box<dyn Fn(WatchEvent) + Send + Sync>,
) -> Result<WatchHandle>;
}
pub struct WatchHandle {
_cancel: tokio::sync::oneshot::Sender<()>,
}
impl WatchHandle {
pub fn new(cancel: tokio::sync::oneshot::Sender<()>) -> Self {
Self { _cancel: cancel }
}
}
#[derive(Debug, Clone)]
pub enum WatchEvent {
Created(String),
Modified(String),
Deleted(String),
}
/// Directory entry from plugin origin
#[derive(Debug, Clone)]
pub struct OriginDirEntry {
pub name: String,
pub is_dir: bool,
pub size: u64,
pub mtime_secs: u64,
}
/// File stats from plugin origin
#[derive(Debug, Clone)]
pub struct OriginStat {
pub size: u64,
pub mtime_secs: u64,
pub is_dir: bool,
}
/// Origin health status
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OriginHealth {
Healthy,
Degraded,
Unhealthy,
}
/// Metadata plugin interface (FR-23.3)
///
/// Metadata plugins provide external metadata lookup from services like
/// MusicBrainz, Discogs, Last.fm, etc.
#[async_trait]
pub trait MetadataPlugin: Plugin {
/// Lookup metadata for a query
///
/// Returns enriched metadata if found, None otherwise.
async fn lookup(&self, query: &MetadataQuery) -> Result<Option<ExternalMetadata>>;
/// Supported query types
fn supported_queries(&self) -> &[MetadataQueryType] {
&[MetadataQueryType::ByTitleArtist]
}
}
/// Query for metadata lookup
#[derive(Debug, Clone)]
pub struct MetadataQuery {
pub query_type: MetadataQueryType,
pub title: Option<String>,
pub artist: Option<String>,
pub album: Option<String>,
pub fingerprint: Option<String>,
pub duration_ms: Option<u64>,
}
/// Type of metadata query
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MetadataQueryType {
ByTitleArtist,
ByFingerprint,
ByAlbum,
}
/// External metadata returned by plugins
#[derive(Debug, Clone, Default)]
pub struct ExternalMetadata {
pub title: Option<String>,
pub artist: Option<String>,
pub album: Option<String>,
pub album_artist: Option<String>,
pub genre: Option<String>,
pub year: Option<u32>,
pub track: Option<u32>,
pub disc: Option<u32>,
pub musicbrainz_id: Option<String>,
pub artwork_url: Option<String>,
}
/// Format plugin interface (FR-24.1)
///
/// Format plugins handle custom audio formats not supported by symphonia.
///
/// Example use cases:
/// - Custom lossless codecs
/// - Proprietary formats
/// - Game audio formats
pub trait FormatPlugin: Plugin {
/// File extensions this plugin handles
fn extensions(&self) -> &[&str];
/// Check if plugin can handle a specific extension
fn can_handle(&self, extension: &str) -> bool {
self.extensions()
.iter()
.any(|ext| ext.eq_ignore_ascii_case(extension))
}
/// Parse audio metadata from reader
///
/// The reader provides the raw file bytes. Plugin should parse
/// and return AudioMeta with whatever metadata it can extract.
fn parse(&self, reader: &mut dyn Read) -> Result<AudioMeta>;
/// Synthesize file header with updated metadata (FR-5.3)
///
/// Creates a new file header containing the provided metadata.
/// Used for metadata overlay - serving cached metadata without
/// modifying the original file.
fn synthesize_header(&self, metadata: &AudioMeta) -> Result<Vec<u8>>;
}
/// Declaration macro for native plugins
///
/// Native plugins must export a function with this signature:
/// ```ignore
/// #[no_mangle]
/// pub extern "C" fn musicfs_plugin_create() -> *mut dyn Plugin
/// ```
#[macro_export]
macro_rules! declare_plugin {
($plugin_type:ty, $constructor:expr) => {
#[no_mangle]
pub extern "C" fn musicfs_plugin_create() -> *mut dyn $crate::Plugin {
let plugin = $constructor;
let boxed: Box<dyn $crate::Plugin> = Box::new(plugin);
Box::into_raw(boxed)
}
#[no_mangle]
pub extern "C" fn musicfs_plugin_api_version() -> *const std::ffi::c_char {
concat!($crate::PLUGIN_API_VERSION, "\0").as_ptr() as *const std::ffi::c_char
}
};
}
#[cfg(test)]
mod tests {
use super::*;
struct TestPlugin {
name: String,
initialized: bool,
}
impl Plugin for TestPlugin {
fn name(&self) -> &str {
&self.name
}
fn version(&self) -> Version {
Version::new(1, 0, 0)
}
fn plugin_type(&self) -> PluginType {
PluginType::Origin
}
fn init(&mut self, _config: Value) -> Result<()> {
self.initialized = true;
Ok(())
}
fn shutdown(&mut self) -> Result<()> {
self.initialized = false;
Ok(())
}
}
#[test]
fn test_plugin_lifecycle() {
let mut plugin = TestPlugin {
name: "test".to_string(),
initialized: false,
};
assert_eq!(plugin.name(), "test");
assert!(!plugin.initialized);
plugin.init(Value::Null).unwrap();
assert!(plugin.initialized);
plugin.shutdown().unwrap();
assert!(!plugin.initialized);
}
#[test]
fn test_plugin_id() {
let id1 = PluginId::new(1);
let id2 = PluginId::new(1);
let id3 = PluginId::new(2);
assert_eq!(id1, id2);
assert_ne!(id1, id3);
}
}
crates/musicfs-plugins/src/wasm.rs
-220
View File
@@ -1,220 +0,0 @@
use crate::error::{PluginError, Result};
use crate::traits::PluginId;
#[cfg(feature = "wasm")]
use std::sync::atomic::{AtomicU64, Ordering};
#[cfg(feature = "wasm")]
static NEXT_WASM_PLUGIN_ID: AtomicU64 = AtomicU64::new(1_000_000);
#[cfg(feature = "wasm")]
fn next_wasm_plugin_id() -> PluginId {
PluginId::new(NEXT_WASM_PLUGIN_ID.fetch_add(1, Ordering::SeqCst))
}
#[derive(Debug, Clone)]
pub struct ResourceLimits {
pub max_memory_mb: u32,
pub max_cpu_time_ms: u32,
pub allow_network: bool,
pub allow_filesystem: bool,
}
impl Default for ResourceLimits {
fn default() -> Self {
Self {
max_memory_mb: 64,
max_cpu_time_ms: 5000,
allow_network: false,
allow_filesystem: false,
}
}
}
#[cfg(feature = "wasm")]
mod wasm_impl {
use super::*;
use std::collections::HashMap;
use tracing::info;
use wasmtime::{Config, Engine, Linker, Module, Store};
pub struct PluginState {
limits: ResourceLimits,
}
pub struct WasmPlugin {
id: PluginId,
name: String,
_module: Module,
}
impl WasmPlugin {
pub fn id(&self) -> PluginId {
self.id
}
pub fn name(&self) -> &str {
&self.name
}
}
pub struct WasmPluginHost {
engine: Engine,
linker: Linker<PluginState>,
plugins: HashMap<PluginId, WasmPlugin>,
limits: ResourceLimits,
}
impl WasmPluginHost {
pub fn new() -> Result<Self> {
let mut config = Config::new();
config.consume_fuel(true);
config.epoch_interruption(true);
let engine = Engine::new(&config)
.map_err(|e| PluginError::Wasm(format!("Failed to create WASM engine: {}", e)))?;
let linker = Linker::new(&engine);
Ok(Self {
engine,
linker,
plugins: HashMap::new(),
limits: ResourceLimits::default(),
})
}
pub fn set_limits(&mut self, limits: ResourceLimits) {
self.limits = limits;
}
pub fn load(&mut self, wasm_bytes: &[u8]) -> Result<PluginId> {
info!("Loading WASM plugin ({} bytes)", wasm_bytes.len());
let module = Module::new(&self.engine, wasm_bytes)
.map_err(|e| PluginError::Wasm(format!("Failed to compile WASM module: {}", e)))?;
let id = next_wasm_plugin_id();
let name = module.name().unwrap_or("unnamed").to_string();
let plugin = WasmPlugin {
id,
name,
_module: module,
};
self.plugins.insert(id, plugin);
Ok(id)
}
pub fn unload(&mut self, id: PluginId) -> Result<()> {
self.plugins
.remove(&id)
.ok_or_else(|| PluginError::NotFound(format!("WASM plugin {:?}", id)))?;
Ok(())
}
pub fn get(&self, id: PluginId) -> Option<&WasmPlugin> {
self.plugins.get(&id)
}
pub fn list(&self) -> Vec<(PluginId, &str)> {
self.plugins.iter().map(|(id, p)| (*id, p.name())).collect()
}
fn create_store(&self) -> Store<PluginState> {
let state = PluginState {
limits: self.limits.clone(),
};
let mut store = Store::new(&self.engine, state);
let fuel = (self.limits.max_cpu_time_ms as u64) * 1_000_000;
store.set_fuel(fuel).ok();
store
}
}
impl Default for WasmPluginHost {
fn default() -> Self {
Self::new().expect("Failed to create WASM host")
}
}
}
#[cfg(not(feature = "wasm"))]
mod wasm_stub {
use super::*;
pub struct WasmPluginHost {
limits: ResourceLimits,
}
impl WasmPluginHost {
pub fn new() -> Result<Self> {
Ok(Self {
limits: ResourceLimits::default(),
})
}
pub fn set_limits(&mut self, limits: ResourceLimits) {
self.limits = limits;
}
pub fn load(&mut self, _wasm_bytes: &[u8]) -> Result<PluginId> {
Err(PluginError::Wasm(
"WASM support not enabled. Compile with --features wasm".to_string(),
))
}
pub fn unload(&mut self, _id: PluginId) -> Result<()> {
Err(PluginError::Wasm("WASM support not enabled".to_string()))
}
pub fn list(&self) -> Vec<(PluginId, &str)> {
Vec::new()
}
}
impl Default for WasmPluginHost {
fn default() -> Self {
Self::new().expect("Failed to create WASM stub host")
}
}
}
#[cfg(feature = "wasm")]
pub use wasm_impl::*;
#[cfg(not(feature = "wasm"))]
pub use wasm_stub::*;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_resource_limits_default() {
let limits = ResourceLimits::default();
assert_eq!(limits.max_memory_mb, 64);
assert_eq!(limits.max_cpu_time_ms, 5000);
assert!(!limits.allow_network);
assert!(!limits.allow_filesystem);
}
#[test]
fn test_wasm_host_creation() {
let host = WasmPluginHost::new();
assert!(host.is_ok());
}
#[test]
#[cfg(not(feature = "wasm"))]
fn test_wasm_disabled_load_fails() {
let mut host = WasmPluginHost::new().unwrap();
let result = host.load(&[0x00, 0x61, 0x73, 0x6d]);
assert!(result.is_err());
}
}
crates/musicfs-search/Cargo.toml
-21
View File
@@ -1,21 +0,0 @@
[package]
name = "musicfs-search"
version.workspace = true
edition.workspace = true
[dependencies]
musicfs-core = { path = "../musicfs-core" }
tantivy.workspace = true
moka.workspace = true
parking_lot.workspace = true
tokio = { workspace = true, features = ["sync", "time"] }
tracing.workspace = true
thiserror.workspace = true
rusqlite.workspace = true
serde.workspace = true
serde_json.workspace = true
[dev-dependencies]
tempfile.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
crates/musicfs-search/src/collections.rs
-321
View File
@@ -1,321 +0,0 @@
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::time::{Duration, SystemTime};
use tracing::{debug, info, warn};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SmartCollection {
pub id: i64,
pub name: String,
pub query: CollectionQuery,
pub created_at: SystemTime,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum CollectionQuery {
Match {
field: String,
pattern: String,
},
DateRange {
field: String,
start: i32,
end: i32,
},
RecentlyAdded {
days: u32,
},
RecentlyPlayed {
days: u32,
},
MostPlayed {
limit: u32,
},
Genre {
genre: String,
},
Compound {
op: BoolOp,
children: Vec<CollectionQuery>,
},
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub enum BoolOp {
And,
Or,
}
impl CollectionQuery {
pub fn to_tantivy_query(&self) -> String {
match self {
CollectionQuery::Match { field, pattern } => {
format!("{}:{}", field, pattern)
}
CollectionQuery::DateRange { field, start, end } => {
format!("{}:[{} TO {}]", field, start, end)
}
CollectionQuery::Genre { genre } => {
format!("genre:{}", genre)
}
CollectionQuery::Compound { op, children } => {
let sep = match op {
BoolOp::And => " AND ",
BoolOp::Or => " OR ",
};
let parts: Vec<_> = children
.iter()
.map(|c| format!("({})", c.to_tantivy_query()))
.collect();
parts.join(sep)
}
_ => String::new(),
}
}
pub fn is_dynamic(&self) -> bool {
matches!(
self,
CollectionQuery::RecentlyAdded { .. }
| CollectionQuery::RecentlyPlayed { .. }
| CollectionQuery::MostPlayed { .. }
)
}
}
pub struct CollectionStore {
db: Mutex<rusqlite::Connection>,
}
impl CollectionStore {
pub fn new(db_path: &Path) -> Result<Self, CollectionError> {
let db = rusqlite::Connection::open(db_path)?;
db.execute(
"CREATE TABLE IF NOT EXISTS collections (
id INTEGER PRIMARY KEY,
name TEXT UNIQUE NOT NULL,
query_json TEXT NOT NULL,
created_at INTEGER NOT NULL
)",
[],
)?;
info!(path = ?db_path, "Collection store opened");
Ok(Self { db: Mutex::new(db) })
}
pub fn create(
&self,
name: &str,
query: CollectionQuery,
) -> Result<SmartCollection, CollectionError> {
info!(name = %name, "Creating collection");
let query_json = serde_json::to_string(&query)?;
let now = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs() as i64;
let db = self.db.lock();
db.execute(
"INSERT INTO collections (name, query_json, created_at) VALUES (?1, ?2, ?3)",
rusqlite::params![name, query_json, now],
)?;
let id = db.last_insert_rowid();
debug!(id = id, name = %name, "Collection created");
Ok(SmartCollection {
id,
name: name.to_string(),
query,
created_at: SystemTime::UNIX_EPOCH + Duration::from_secs(now as u64),
})
}
pub fn list(&self) -> Result<Vec<SmartCollection>, CollectionError> {
let db = self.db.lock();
let mut stmt = db.prepare("SELECT id, name, query_json, created_at FROM collections")?;
let collections = stmt.query_map([], |row| {
let query_json: String = row.get(2)?;
let created_secs: i64 = row.get(3)?;
let query = match serde_json::from_str(&query_json) {
Ok(q) => q,
Err(e) => {
warn!("Failed to parse collection query JSON: {}", e);
CollectionQuery::Match {
field: "title".to_string(),
pattern: "*".to_string(),
}
}
};
Ok(SmartCollection {
id: row.get(0)?,
name: row.get(1)?,
query,
created_at: SystemTime::UNIX_EPOCH + Duration::from_secs(created_secs as u64),
})
})?;
collections
.collect::<Result<Vec<_>, _>>()
.map_err(CollectionError::from)
}
pub fn get(&self, name: &str) -> Result<Option<SmartCollection>, CollectionError> {
let db = self.db.lock();
let mut stmt =
db.prepare("SELECT id, name, query_json, created_at FROM collections WHERE name = ?1")?;
let result = stmt
.query_row([name], |row| {
let query_json: String = row.get(2)?;
let created_secs: i64 = row.get(3)?;
let query = match serde_json::from_str(&query_json) {
Ok(q) => q,
Err(e) => {
warn!("Failed to parse collection query JSON: {}", e);
CollectionQuery::Match {
field: "title".to_string(),
pattern: "*".to_string(),
}
}
};
Ok(SmartCollection {
id: row.get(0)?,
name: row.get(1)?,
query,
created_at: SystemTime::UNIX_EPOCH + Duration::from_secs(created_secs as u64),
})
})
.ok();
Ok(result)
}
pub fn delete(&self, name: &str) -> Result<(), CollectionError> {
info!(name = %name, "Deleting collection");
let db = self.db.lock();
db.execute("DELETE FROM collections WHERE name = ?1", [name])?;
Ok(())
}
}
pub fn builtin_collections() -> Vec<SmartCollection> {
vec![
SmartCollection {
id: -1,
name: "Recently Added".to_string(),
query: CollectionQuery::RecentlyAdded { days: 30 },
created_at: SystemTime::UNIX_EPOCH,
},
SmartCollection {
id: -2,
name: "80s Music".to_string(),
query: CollectionQuery::DateRange {
field: "year".to_string(),
start: 1980,
end: 1989,
},
created_at: SystemTime::UNIX_EPOCH,
},
SmartCollection {
id: -3,
name: "90s Music".to_string(),
query: CollectionQuery::DateRange {
field: "year".to_string(),
start: 1990,
end: 1999,
},
created_at: SystemTime::UNIX_EPOCH,
},
]
}
#[derive(Debug, thiserror::Error)]
pub enum CollectionError {
#[error("database error: {0}")]
Database(#[from] rusqlite::Error),
#[error("serialization error: {0}")]
Serialization(#[from] serde_json::Error),
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[test]
fn test_collection_crud() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("collections.db");
let store = CollectionStore::new(&db_path).unwrap();
let collection = store
.create(
"Jazz",
CollectionQuery::Genre {
genre: "Jazz".to_string(),
},
)
.unwrap();
assert_eq!(collection.name, "Jazz");
let collections = store.list().unwrap();
assert_eq!(collections.len(), 1);
store.delete("Jazz").unwrap();
let collections = store.list().unwrap();
assert_eq!(collections.len(), 0);
}
#[test]
fn test_compound_query() {
let query = CollectionQuery::Compound {
op: BoolOp::And,
children: vec![
CollectionQuery::Genre {
genre: "Metal".to_string(),
},
CollectionQuery::DateRange {
field: "year".to_string(),
start: 1980,
end: 1989,
},
],
};
let tantivy_query = query.to_tantivy_query();
assert!(tantivy_query.contains("genre:Metal"));
assert!(tantivy_query.contains("year:[1980 TO 1989]"));
assert!(tantivy_query.contains(" AND "));
}
#[test]
fn test_builtin_collections() {
let builtins = builtin_collections();
assert_eq!(builtins.len(), 3);
assert!(builtins.iter().any(|c| c.name == "Recently Added"));
}
#[test]
fn test_dynamic_query_detection() {
assert!(CollectionQuery::RecentlyAdded { days: 30 }.is_dynamic());
assert!(CollectionQuery::RecentlyPlayed { days: 7 }.is_dynamic());
assert!(CollectionQuery::MostPlayed { limit: 100 }.is_dynamic());
assert!(!CollectionQuery::Genre {
genre: "Rock".to_string()
}
.is_dynamic());
}
}
crates/musicfs-search/src/index.rs
-417
View File
@@ -1,417 +0,0 @@
use musicfs_core::{AudioMeta, FileId, FileMeta, VirtualPath};
use parking_lot::RwLock;
use std::path::Path;
use std::sync::Arc;
use tantivy::collector::TopDocs;
use tantivy::query::{BooleanQuery, FuzzyTermQuery, Occur, Query, QueryParser};
use tantivy::schema::{Field, Schema, Value, INDEXED, STORED, TEXT};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
use tracing::{debug, info, warn};
const SCHEMA_VERSION: u32 = 1;
pub struct SearchIndex {
index: Index,
reader: IndexReader,
writer: Arc<RwLock<IndexWriter>>,
schema: SearchSchema,
pub schema_version: u32,
}
struct SearchSchema {
schema: Schema,
file_id: Field,
virtual_path: Field,
artist: Field,
album: Field,
album_artist: Field,
title: Field,
genre: Field,
composer: Field,
year: Field,
duration_ms: Field,
bitrate: Field,
sample_rate: Field,
}
impl SearchSchema {
fn new() -> Self {
let mut builder = Schema::builder();
Self {
file_id: builder.add_u64_field("file_id", INDEXED | STORED),
virtual_path: builder.add_text_field("virtual_path", STORED),
artist: builder.add_text_field("artist", TEXT | STORED),
album: builder.add_text_field("album", TEXT | STORED),
album_artist: builder.add_text_field("album_artist", TEXT | STORED),
title: builder.add_text_field("title", TEXT | STORED),
genre: builder.add_text_field("genre", TEXT | STORED),
composer: builder.add_text_field("composer", TEXT | STORED),
year: builder.add_u64_field("year", INDEXED | STORED),
duration_ms: builder.add_u64_field("duration_ms", STORED),
bitrate: builder.add_u64_field("bitrate", STORED),
sample_rate: builder.add_u64_field("sample_rate", STORED),
schema: builder.build(),
}
}
}
#[derive(Debug, Clone)]
pub struct SearchHit {
pub file_id: FileId,
pub virtual_path: VirtualPath,
pub artist: Option<String>,
pub album: Option<String>,
pub title: Option<String>,
pub score: f32,
}
impl SearchIndex {
pub fn open(index_path: &Path) -> Result<Self, SearchError> {
let schema_obj = SearchSchema::new();
let index = if index_path.exists() && index_path.join("meta.json").exists() {
Index::open_in_dir(index_path)?
} else {
std::fs::create_dir_all(index_path)?;
Index::create_in_dir(index_path, schema_obj.schema.clone())?
};
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::OnCommitWithDelay)
.try_into()?;
let writer = index.writer(50_000_000)?;
info!("Search index opened at {:?}", index_path);
Ok(Self {
index,
reader,
writer: Arc::new(RwLock::new(writer)),
schema: schema_obj,
schema_version: SCHEMA_VERSION,
})
}
pub fn open_with_recovery(index_path: &Path) -> Result<Self, SearchError> {
match Self::open(index_path) {
Ok(index) => {
let docs = index.reader.searcher().num_docs();
info!(docs, "Search index opened successfully");
Ok(index)
}
Err(e) => {
warn!(
error = %e,
path = ?index_path,
"Search index corrupted, rebuilding from scratch"
);
if index_path.exists() {
std::fs::remove_dir_all(index_path).map_err(SearchError::Io)?;
}
Self::open(index_path)
}
}
}
pub fn index_file(&self, file: &FileMeta) -> Result<(), SearchError> {
let mut doc = TantivyDocument::new();
doc.add_u64(self.schema.file_id, file.id.0 as u64);
doc.add_text(self.schema.virtual_path, file.virtual_path.as_str());
if let Some(ref audio) = file.audio {
Self::add_audio_fields(&mut doc, &self.schema, audio);
}
self.writer.read().add_document(doc)?;
debug!("Indexed file {:?}", file.id);
Ok(())
}
fn add_audio_fields(doc: &mut TantivyDocument, schema: &SearchSchema, audio: &AudioMeta) {
if let Some(ref v) = audio.artist {
doc.add_text(schema.artist, v);
}
if let Some(ref v) = audio.album {
doc.add_text(schema.album, v);
}
if let Some(ref v) = audio.album_artist {
doc.add_text(schema.album_artist, v);
}
if let Some(ref v) = audio.title {
doc.add_text(schema.title, v);
}
if let Some(ref v) = audio.genre {
doc.add_text(schema.genre, v);
}
if let Some(ref v) = audio.year {
doc.add_u64(schema.year, *v as u64);
}
if let Some(v) = audio.duration_ms {
doc.add_u64(schema.duration_ms, v);
}
if let Some(v) = audio.bitrate {
doc.add_u64(schema.bitrate, v as u64);
}
if let Some(v) = audio.sample_rate {
doc.add_u64(schema.sample_rate, v as u64);
}
}
pub fn remove_file(&self, file_id: FileId) -> Result<(), SearchError> {
let term = tantivy::Term::from_field_u64(self.schema.file_id, file_id.0 as u64);
self.writer.read().delete_term(term);
debug!("Removed file {:?} from index", file_id);
Ok(())
}
pub fn remove_by_path(&self, path: &VirtualPath) -> Result<bool, SearchError> {
let searcher = self.reader.searcher();
let query_parser = QueryParser::for_index(&self.index, vec![self.schema.virtual_path]);
let query = query_parser.parse_query(&format!("\"{}\"", path.as_str()))?;
let top_docs = searcher.search(&query, &TopDocs::with_limit(1))?;
if let Some((_, doc_address)) = top_docs.first() {
let doc: TantivyDocument = searcher.doc(*doc_address)?;
if let Some(file_id) = doc.get_first(self.schema.file_id).and_then(|v| v.as_u64()) {
self.remove_file(FileId(file_id as i64))?;
debug!("Removed file by path {:?}", path);
return Ok(true);
}
}
Ok(false)
}
pub fn commit(&self) -> Result<(), SearchError> {
self.writer.write().commit()?;
self.reader.reload()?;
info!("Search index committed");
Ok(())
}
pub fn search(&self, query_str: &str, limit: usize) -> Result<Vec<SearchHit>, SearchError> {
let searcher = self.reader.searcher();
let default_fields = vec![
self.schema.artist,
self.schema.album,
self.schema.album_artist,
self.schema.title,
self.schema.genre,
self.schema.composer,
];
let query: Box<dyn Query> =
if let Some((term, distance)) = Self::parse_fuzzy_query(query_str) {
let subqueries: Vec<(Occur, Box<dyn Query>)> = default_fields
.iter()
.map(|&field| {
let term = Term::from_field_text(field, &term);
let fuzzy = FuzzyTermQuery::new(term, distance, true);
(Occur::Should, Box::new(fuzzy) as Box<dyn Query>)
})
.collect();
Box::new(BooleanQuery::new(subqueries))
} else {
let query_parser = QueryParser::for_index(&self.index, default_fields);
query_parser.parse_query(query_str)?
};
let top_docs = searcher.search(&*query, &TopDocs::with_limit(limit))?;
let mut results = Vec::with_capacity(top_docs.len());
for (score, doc_address) in top_docs {
let doc: TantivyDocument = searcher.doc(doc_address)?;
let file_id = doc
.get_first(self.schema.file_id)
.and_then(|v| v.as_u64())
.map(|id| FileId(id as i64))
.ok_or(SearchError::CorruptedIndex)?;
let virtual_path = doc
.get_first(self.schema.virtual_path)
.and_then(|v| v.as_str())
.map(|s| VirtualPath::new(s))
.ok_or(SearchError::CorruptedIndex)?;
results.push(SearchHit {
file_id,
virtual_path,
artist: doc
.get_first(self.schema.artist)
.and_then(|v| v.as_str())
.map(String::from),
album: doc
.get_first(self.schema.album)
.and_then(|v| v.as_str())
.map(String::from),
title: doc
.get_first(self.schema.title)
.and_then(|v| v.as_str())
.map(String::from),
score,
});
}
debug!("Search '{}' returned {} results", query_str, results.len());
Ok(results)
}
pub fn count(&self) -> u64 {
self.reader.searcher().num_docs()
}
fn parse_fuzzy_query(query_str: &str) -> Option<(String, u8)> {
let query_str = query_str.trim();
if let Some(tilde_pos) = query_str.rfind('~') {
let term = &query_str[..tilde_pos];
let distance_str = &query_str[tilde_pos + 1..];
if !term.is_empty() && !term.contains(':') && !term.contains(' ') {
if let Ok(distance) = distance_str.parse::<u8>() {
if distance <= 2 {
return Some((term.to_lowercase(), distance));
}
}
}
}
None
}
}
#[derive(Debug, thiserror::Error)]
pub enum SearchError {
#[error("tantivy error: {0}")]
Tantivy(#[from] tantivy::TantivyError),
#[error("query parse error: {0}")]
QueryParse(#[from] tantivy::query::QueryParserError),
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("corrupted search index")]
CorruptedIndex,
}
#[cfg(test)]
mod tests {
use super::*;
use musicfs_core::{AudioFormat, OriginId, RealPath};
use std::path::PathBuf;
use tempfile::TempDir;
fn make_file(id: i64, artist: &str, album: &str, title: &str) -> FileMeta {
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(format!("/{}/{}/{}.flac", artist, album, title)),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from("test.flac"),
},
size: 1000,
mtime: std::time::SystemTime::UNIX_EPOCH,
content_hash: None,
audio: Some(AudioMeta {
artist: Some(artist.to_string()),
album: Some(album.to_string()),
title: Some(title.to_string()),
genre: Some("Metal".to_string()),
format: AudioFormat::Flac,
..Default::default()
}),
}
}
#[test]
fn test_search_basic() {
let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap();
index
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
.unwrap();
index
.index_file(&make_file(2, "Metallica", "Master of Puppets", "Battery"))
.unwrap();
index
.index_file(&make_file(3, "Iron Maiden", "Powerslave", "Aces High"))
.unwrap();
index.commit().unwrap();
let results = index.search("metallica", 10).unwrap();
assert_eq!(results.len(), 2);
let results = index.search("sandman", 10).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].title.as_deref(), Some("Enter Sandman"));
}
#[test]
fn test_search_fuzzy() {
let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap();
index
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
.unwrap();
index.commit().unwrap();
let results = index.search("metalica~1", 10).unwrap();
assert_eq!(results.len(), 1);
}
#[test]
fn test_search_genre() {
let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap();
index
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
.unwrap();
index.commit().unwrap();
let results = index.search("genre:Metal", 10).unwrap();
assert_eq!(results.len(), 1);
}
#[test]
fn test_remove_file() {
let dir = TempDir::new().unwrap();
let index = SearchIndex::open(dir.path()).unwrap();
index
.index_file(&make_file(1, "Test", "Album", "Song"))
.unwrap();
index.commit().unwrap();
assert_eq!(index.search("test", 10).unwrap().len(), 1);
index.remove_file(FileId(1)).unwrap();
index.commit().unwrap();
assert_eq!(index.search("test", 10).unwrap().len(), 0);
}
#[test]
fn test_index_persistence() {
let dir = TempDir::new().unwrap();
{
let index = SearchIndex::open(dir.path()).unwrap();
index
.index_file(&make_file(1, "Artist", "Album", "Track"))
.unwrap();
index.commit().unwrap();
}
{
let index = SearchIndex::open(dir.path()).unwrap();
let results = index.search("artist", 10).unwrap();
assert_eq!(results.len(), 1);
}
}
}
crates/musicfs-search/src/indexer.rs
-236
View File
@@ -1,236 +0,0 @@
use crate::index::{SearchError, SearchIndex};
use musicfs_core::{Event, EventBus, FileMeta};
use std::sync::Arc;
use tokio::sync::mpsc;
use tracing::{debug, error, info, info_span, warn, Instrument};
pub trait MetadataLookup: Send + Sync {
fn lookup(&self, path: &musicfs_core::VirtualPath) -> Option<FileMeta>;
}
pub struct Indexer<M: MetadataLookup> {
index: Arc<SearchIndex>,
event_bus: Arc<EventBus>,
metadata_lookup: Arc<M>,
}
impl<M: MetadataLookup + 'static> Indexer<M> {
pub fn new(index: Arc<SearchIndex>, event_bus: Arc<EventBus>, metadata_lookup: Arc<M>) -> Self {
Self {
index,
event_bus,
metadata_lookup,
}
}
pub fn start(self) -> IndexerHandle {
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
let mut event_rx = self.event_bus.subscribe();
info!("Search indexer starting");
tokio::spawn(
async move {
let mut pending_commit = false;
let mut commit_timer = tokio::time::interval(std::time::Duration::from_secs(5));
loop {
tokio::select! {
result = event_rx.recv() => {
match result {
Ok(event) => {
if let Err(e) = self.handle_event(&event) {
error!("Indexer error: {}", e);
}
pending_commit = true;
}
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
warn!(skipped = n, "Indexer lagged, skipped events");
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
debug!("Event channel closed");
break;
}
}
}
_ = commit_timer.tick() => {
if pending_commit {
if let Err(e) = self.index.commit() {
error!("Index commit error: {}", e);
}
pending_commit = false;
}
}
_ = stop_rx.recv() => {
info!("Indexer stopping");
if pending_commit {
let _ = self.index.commit();
}
break;
}
}
}
}
.instrument(info_span!("search_indexer")),
);
IndexerHandle { stop_tx }
}
fn handle_event(&self, event: &Event) -> Result<(), SearchError> {
match event {
Event::FileAdded { path, .. } => {
debug!("Indexing added file: {:?}", path);
if let Some(meta) = self.metadata_lookup.lookup(path) {
self.index.index_file(&meta)?;
} else {
warn!("No metadata found for added file: {:?}", path);
}
}
Event::FileRemoved { path, file_id } => {
debug!("Removing from index: {:?}", path);
if let Some(id) = file_id {
self.index.remove_file(*id)?;
} else if let Some(meta) = self.metadata_lookup.lookup(path) {
self.index.remove_file(meta.id)?;
} else {
self.index.remove_by_path(path)?;
}
}
Event::FileModified { path } => {
debug!("Re-indexing modified file: {:?}", path);
if let Some(meta) = self.metadata_lookup.lookup(path) {
self.index.remove_file(meta.id)?;
self.index.index_file(&meta)?;
}
}
_ => {}
}
Ok(())
}
pub fn index_batch(&self, files: &[FileMeta]) -> Result<usize, SearchError> {
let mut count = 0;
for file in files {
self.index.index_file(file)?;
count += 1;
}
self.index.commit()?;
info!("Indexed {} files", count);
Ok(count)
}
}
pub struct IndexerHandle {
stop_tx: mpsc::Sender<()>,
}
impl IndexerHandle {
pub async fn stop(self) {
let _ = self.stop_tx.send(()).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
use musicfs_core::{AudioFormat, AudioMeta, FileId, OriginId, RealPath, VirtualPath};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::RwLock;
use tempfile::TempDir;
struct MockMetadataLookup {
files: RwLock<HashMap<String, FileMeta>>,
}
impl MockMetadataLookup {
fn new() -> Self {
Self {
files: RwLock::new(HashMap::new()),
}
}
fn insert(&self, meta: FileMeta) {
self.files
.write()
.unwrap()
.insert(meta.virtual_path.as_str().to_string(), meta);
}
}
impl MetadataLookup for MockMetadataLookup {
fn lookup(&self, path: &VirtualPath) -> Option<FileMeta> {
self.files.read().unwrap().get(path.as_str()).cloned()
}
}
fn make_file(id: i64, path: &str, artist: &str, title: &str) -> FileMeta {
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(path),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from("test.flac"),
},
size: 1000,
mtime: std::time::SystemTime::UNIX_EPOCH,
content_hash: None,
audio: Some(AudioMeta {
artist: Some(artist.to_string()),
title: Some(title.to_string()),
format: AudioFormat::Flac,
..Default::default()
}),
}
}
#[tokio::test]
async fn test_indexer_handles_file_added() {
let dir = TempDir::new().unwrap();
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
let event_bus = Arc::new(EventBus::default());
let metadata = Arc::new(MockMetadataLookup::new());
let file = make_file(1, "/Artist/Album/Track.flac", "Artist", "Track");
metadata.insert(file.clone());
let indexer = Indexer::new(index.clone(), event_bus.clone(), metadata);
let handle = indexer.start();
event_bus.publish(Event::FileAdded {
path: VirtualPath::new("/Artist/Album/Track.flac"),
origin_id: OriginId::from("test"),
});
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
index.commit().unwrap();
let results = index.search("artist", 10).unwrap();
assert_eq!(results.len(), 1);
handle.stop().await;
}
#[test]
fn test_index_batch() {
let dir = TempDir::new().unwrap();
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
let event_bus = Arc::new(EventBus::default());
let metadata = Arc::new(MockMetadataLookup::new());
let indexer = Indexer::new(index.clone(), event_bus, metadata);
let files = vec![
make_file(1, "/a.flac", "Artist1", "Song1"),
make_file(2, "/b.flac", "Artist2", "Song2"),
make_file(3, "/c.flac", "Artist3", "Song3"),
];
let count = indexer.index_batch(&files).unwrap();
assert_eq!(count, 3);
let results = index.search("artist1", 10).unwrap();
assert_eq!(results.len(), 1);
}
}
crates/musicfs-search/src/lib.rs
-11
View File
@@ -1,11 +0,0 @@
mod collections;
mod index;
mod indexer;
mod query;
pub use collections::{
builtin_collections, BoolOp, CollectionError, CollectionQuery, CollectionStore, SmartCollection,
};
pub use index::{SearchError, SearchHit, SearchIndex};
pub use indexer::{Indexer, IndexerHandle, MetadataLookup};
pub use query::SearchQueryBuilder;
crates/musicfs-search/src/query.rs
-78
View File
@@ -1,78 +0,0 @@
use tantivy::query::{BooleanQuery, FuzzyTermQuery, Occur, Query};
use tantivy::schema::Field;
use tantivy::Term;
pub struct SearchQueryBuilder {
fields: Vec<Field>,
default_fuzziness: u8,
}
impl SearchQueryBuilder {
pub fn new(fields: Vec<Field>) -> Self {
Self {
fields,
default_fuzziness: 1,
}
}
pub fn with_fuzziness(mut self, fuzziness: u8) -> Self {
self.default_fuzziness = fuzziness;
self
}
pub fn build_fuzzy(&self, query_text: &str) -> Box<dyn Query> {
let terms: Vec<_> = query_text
.split_whitespace()
.filter(|t| !t.is_empty())
.collect();
if terms.is_empty() {
return Box::new(tantivy::query::AllQuery);
}
let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new();
for term in terms {
let mut field_queries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
for field in &self.fields {
let fuzzy = FuzzyTermQuery::new(
Term::from_field_text(*field, &term.to_lowercase()),
self.default_fuzziness,
true,
);
field_queries.push((Occur::Should, Box::new(fuzzy)));
}
let field_union = BooleanQuery::new(field_queries);
clauses.push((Occur::Must, Box::new(field_union)));
}
Box::new(BooleanQuery::new(clauses))
}
}
#[cfg(test)]
mod tests {
use super::*;
use tantivy::schema::{Schema, TEXT};
#[test]
fn test_query_builder() {
let mut schema_builder = Schema::builder();
let artist = schema_builder.add_text_field("artist", TEXT);
let title = schema_builder.add_text_field("title", TEXT);
let builder = SearchQueryBuilder::new(vec![artist, title]);
let _query = builder.build_fuzzy("metallica sandman");
}
#[test]
fn test_empty_query() {
let mut schema_builder = Schema::builder();
let artist = schema_builder.add_text_field("artist", TEXT);
let builder = SearchQueryBuilder::new(vec![artist]);
let _query = builder.build_fuzzy("");
}
}
crates/musicfs-sync/Cargo.toml
-22
View File
@@ -1,22 +0,0 @@
[package]
name = "musicfs-sync"
version.workspace = true
edition.workspace = true
[dependencies]
musicfs-core = { path = "../musicfs-core" }
musicfs-origins = { path = "../musicfs-origins" }
fastcdc = "3"
xxhash-rust = { version = "0.8", features = ["xxh64"] }
notify = "6"
rmp-serde = "1"
tokio = { workspace = true }
tracing = { workspace = true }
thiserror = { workspace = true }
serde = { workspace = true }
async-trait = { workspace = true }
[dev-dependencies]
tempfile = { workspace = true }
crates/musicfs-sync/src/cdc.rs
-232
View File
@@ -1,232 +0,0 @@
use fastcdc::v2020::FastCDC;
use musicfs_core::ChunkHash;
pub struct CdcChunker {
min_size: u32,
avg_size: u32,
max_size: u32,
}
impl Default for CdcChunker {
fn default() -> Self {
Self {
min_size: 16 * 1024,
avg_size: 64 * 1024,
max_size: 256 * 1024,
}
}
}
#[derive(Debug, Clone)]
pub struct Chunk {
pub hash: ChunkHash,
pub offset: u64,
pub length: u32,
pub data: Vec<u8>,
}
#[derive(Debug)]
pub struct ChunkRef<'a> {
pub hash: ChunkHash,
pub offset: u64,
pub length: u32,
pub data: &'a [u8],
}
impl CdcChunker {
pub fn new(min_size: u32, avg_size: u32, max_size: u32) -> Self {
Self {
min_size,
avg_size,
max_size,
}
}
pub fn chunk(&self, data: &[u8]) -> Vec<Chunk> {
let chunker = FastCDC::new(data, self.min_size, self.avg_size, self.max_size);
chunker
.map(|c| {
let chunk_data = &data[c.offset..c.offset + c.length];
Chunk {
hash: ChunkHash::from_bytes(chunk_data),
offset: c.offset as u64,
length: c.length as u32,
data: chunk_data.to_vec(),
}
})
.collect()
}
pub fn chunk_refs<'a>(&self, data: &'a [u8]) -> Vec<ChunkRef<'a>> {
let chunker = FastCDC::new(data, self.min_size, self.avg_size, self.max_size);
chunker
.map(|c| {
let chunk_data = &data[c.offset..c.offset + c.length];
ChunkRef {
hash: ChunkHash::from_bytes(chunk_data),
offset: c.offset as u64,
length: c.length as u32,
data: chunk_data,
}
})
.collect()
}
pub fn chunk_streaming<F>(&self, data: &[u8], mut processor: F) -> usize
where
F: FnMut(ChunkRef<'_>),
{
let chunker = FastCDC::new(data, self.min_size, self.avg_size, self.max_size);
let mut count = 0;
for c in chunker {
let chunk_data = &data[c.offset..c.offset + c.length];
processor(ChunkRef {
hash: ChunkHash::from_bytes(chunk_data),
offset: c.offset as u64,
length: c.length as u32,
data: chunk_data,
});
count += 1;
}
count
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_cdc_basic() {
let chunker = CdcChunker::default();
let data = vec![0u8; 256 * 1024];
let chunks = chunker.chunk(&data);
assert!(!chunks.is_empty());
let total: u64 = chunks.iter().map(|c| c.length as u64).sum();
assert_eq!(total, data.len() as u64);
let mut offset = 0u64;
for chunk in &chunks {
assert_eq!(chunk.offset, offset);
offset += chunk.length as u64;
}
}
#[test]
fn test_cdc_stable_boundaries() {
let chunker = CdcChunker::new(4 * 1024, 16 * 1024, 64 * 1024);
let mut data1 = vec![0u8; 512 * 1024];
for (i, b) in data1.iter_mut().enumerate() {
*b = ((i * 17 + 31) % 256) as u8;
}
let mut data2 = vec![0xFFu8; 1024];
data2.extend_from_slice(&data1);
let chunks1 = chunker.chunk(&data1);
let chunks2 = chunker.chunk(&data2);
let hashes1: std::collections::HashSet<_> = chunks1.iter().map(|c| c.hash).collect();
let hashes2: std::collections::HashSet<_> = chunks2.iter().map(|c| c.hash).collect();
let shared = hashes1.intersection(&hashes2).count();
assert!(
shared > 0,
"CDC should produce stable boundaries, got {} chunks in original, {} after prepend",
chunks1.len(),
chunks2.len()
);
}
#[test]
fn test_cdc_chunk_sizes() {
let chunker = CdcChunker::default();
let data: Vec<u8> = (0..1024 * 1024)
.map(|i| ((i * 17 + 31) % 256) as u8)
.collect();
let chunks = chunker.chunk(&data);
for chunk in &chunks {
if chunk.offset + chunk.length as u64 != data.len() as u64 {
assert!(
chunk.length >= chunker.min_size / 2,
"Chunk too small: {}",
chunk.length
);
assert!(
chunk.length <= chunker.max_size * 2,
"Chunk too large: {}",
chunk.length
);
}
}
}
#[test]
fn test_cdc_streaming() {
let chunker = CdcChunker::default();
let data = vec![0u8; 256 * 1024];
let mut streamed = Vec::new();
let count = chunker.chunk_streaming(&data, |chunk| {
streamed.push((chunk.hash, chunk.offset, chunk.length));
});
let batched = chunker.chunk(&data);
assert_eq!(count, batched.len());
for (i, chunk) in batched.iter().enumerate() {
assert_eq!(streamed[i].0, chunk.hash);
assert_eq!(streamed[i].1, chunk.offset);
assert_eq!(streamed[i].2, chunk.length);
}
}
#[test]
fn test_bandwidth_reduction_metadata_edit() {
let chunker = CdcChunker::new(4 * 1024, 16 * 1024, 64 * 1024);
let mut state = 12345u64;
let original: Vec<u8> = (0..2 * 1024 * 1024)
.map(|_| {
state = state.wrapping_mul(6364136223846793005).wrapping_add(1);
(state >> 56) as u8
})
.collect();
let chunks1 = chunker.chunk(&original);
let hashes1: std::collections::HashSet<_> = chunks1.iter().map(|c| c.hash).collect();
let mut modified = original.clone();
let mid = modified.len() / 2;
for i in mid..mid + 100 {
modified[i] = 0xFF;
}
let chunks2 = chunker.chunk(&modified);
let hashes2: std::collections::HashSet<_> = chunks2.iter().map(|c| c.hash).collect();
let reused = hashes1.intersection(&hashes2).count();
let reuse_ratio = reused as f64 / chunks2.len() as f64;
// NFR-6.4 requires >90% bandwidth reduction for typical edits
assert!(
reuse_ratio > 0.90,
"Expected >90% chunk reuse for mid-file edit (NFR-6.4). Reused {}/{} chunks ({:.1}%, total {} original)",
reused,
chunks2.len(),
reuse_ratio * 100.0,
chunks1.len()
);
}
}
crates/musicfs-sync/src/delta.rs
-338
View File
@@ -1,338 +0,0 @@
use crate::cdc::CdcChunker;
use musicfs_core::{ChunkHash, FileId, FileMeta, OriginId};
use musicfs_origins::Origin;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::time::SystemTime;
use tracing::{debug, info, trace};
#[derive(Debug, Clone)]
pub struct ScannedFile {
pub path: PathBuf,
pub origin_id: OriginId,
pub size: u64,
pub mtime: SystemTime,
}
#[derive(Debug, Default)]
pub struct ChangeSet {
pub added: Vec<ScannedFile>,
pub removed: Vec<FileId>,
pub modified: Vec<(FileId, ManifestDiff)>,
}
impl ChangeSet {
pub fn is_empty(&self) -> bool {
self.added.is_empty() && self.removed.is_empty() && self.modified.is_empty()
}
pub fn total_changes(&self) -> usize {
self.added.len() + self.removed.len() + self.modified.len()
}
}
#[derive(Debug, Clone)]
pub struct ManifestChunk {
pub hash: ChunkHash,
pub offset: u64,
pub size: u32,
}
#[derive(Debug)]
pub struct ManifestDiff {
pub reuse: Vec<ManifestChunk>,
pub fetch: Vec<ManifestChunk>,
pub orphaned: Vec<ChunkHash>,
}
pub struct DeltaDetector {
chunker: CdcChunker,
}
impl DeltaDetector {
pub fn new() -> Self {
Self {
chunker: CdcChunker::default(),
}
}
pub fn with_chunker(chunker: CdcChunker) -> Self {
Self { chunker }
}
pub async fn detect_changes(
&self,
origin: &dyn Origin,
cached: &HashMap<FileId, FileMeta>,
manifests: &HashMap<FileId, Vec<ManifestChunk>>,
) -> Result<ChangeSet, DeltaError> {
let origin_id = origin.id().clone();
info!(origin_id = %origin_id, "Starting delta detection");
let mut changes = ChangeSet::default();
let origin_files = self.scan_origin(origin).await?;
trace!(origin_id = %origin_id, scanned_count = origin_files.len(), "Completed origin scan");
let cached_by_path: HashMap<_, _> = cached
.values()
.map(|m| (m.real_path.path.clone(), m))
.collect();
for scanned in &origin_files {
if let Some(cached_file) = cached_by_path.get(&scanned.path) {
if self.is_modified_scan(cached_file, scanned) {
debug!(origin_id = %origin_id, path = ?scanned.path, "File modified");
if let Some(old_chunks) = manifests.get(&cached_file.id) {
let new_chunks = self.compute_chunks_for_scan(origin, scanned).await?;
let diff = self.compute_diff(old_chunks, &new_chunks);
changes.modified.push((cached_file.id, diff));
}
}
} else {
debug!(origin_id = %origin_id, path = ?scanned.path, "File added");
changes.added.push(scanned.clone());
}
}
let origin_paths: HashSet<_> = origin_files.iter().map(|f| &f.path).collect();
for cached_file in cached.values() {
if !origin_paths.contains(&cached_file.real_path.path) {
debug!(origin_id = %origin_id, path = ?cached_file.real_path.path, "File removed");
changes.removed.push(cached_file.id);
}
}
info!(
origin_id = %origin_id,
files_added = changes.added.len(),
files_removed = changes.removed.len(),
files_modified = changes.modified.len(),
"Delta detection complete"
);
Ok(changes)
}
fn is_modified_scan(&self, cached: &FileMeta, scanned: &ScannedFile) -> bool {
cached.size != scanned.size || cached.mtime != scanned.mtime
}
async fn scan_origin(&self, origin: &dyn Origin) -> Result<Vec<ScannedFile>, DeltaError> {
let mut files = Vec::new();
let mut dirs_to_scan = vec![PathBuf::from("/")];
while let Some(dir) = dirs_to_scan.pop() {
let entries = origin
.readdir(&dir)
.await
.map_err(|e| DeltaError::OriginScan(e.to_string()))?;
for entry in entries {
let entry_path = dir.join(&entry.name);
if entry.is_dir {
dirs_to_scan.push(entry_path);
} else if Self::is_audio_file(&entry.name) {
let stat = origin
.stat(&entry_path)
.await
.map_err(|e| DeltaError::OriginScan(e.to_string()))?;
files.push(ScannedFile {
path: entry_path,
origin_id: origin.id().clone(),
size: stat.size,
mtime: stat.mtime,
});
}
}
}
Ok(files)
}
fn is_audio_file(name: &str) -> bool {
let lower = name.to_lowercase();
lower.ends_with(".flac")
|| lower.ends_with(".mp3")
|| lower.ends_with(".ogg")
|| lower.ends_with(".wav")
|| lower.ends_with(".m4a")
|| lower.ends_with(".aac")
|| lower.ends_with(".opus")
}
async fn compute_chunks_for_scan(
&self,
origin: &dyn Origin,
scanned: &ScannedFile,
) -> Result<Vec<ManifestChunk>, DeltaError> {
let data = origin
.read_full(&scanned.path)
.await
.map_err(|e| DeltaError::OriginRead(e.to_string()))?;
let chunks = self.chunker.chunk_refs(&data);
Ok(chunks
.into_iter()
.map(|c| ManifestChunk {
hash: c.hash,
offset: c.offset,
size: c.length,
})
.collect())
}
fn compute_diff(
&self,
old_chunks: &[ManifestChunk],
new_chunks: &[ManifestChunk],
) -> ManifestDiff {
let old_hashes: HashSet<_> = old_chunks.iter().map(|c| c.hash).collect();
let new_hashes: HashSet<_> = new_chunks.iter().map(|c| c.hash).collect();
ManifestDiff {
reuse: new_chunks
.iter()
.filter(|c| old_hashes.contains(&c.hash))
.cloned()
.collect(),
fetch: new_chunks
.iter()
.filter(|c| !old_hashes.contains(&c.hash))
.cloned()
.collect(),
orphaned: old_chunks
.iter()
.filter(|c| !new_hashes.contains(&c.hash))
.map(|c| c.hash)
.collect(),
}
}
}
impl Default for DeltaDetector {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, thiserror::Error)]
pub enum DeltaError {
#[error("Origin read error: {0}")]
OriginRead(String),
#[error("Origin scan error: {0}")]
OriginScan(String),
}
#[cfg(test)]
mod tests {
use super::*;
use musicfs_core::{OriginId, RealPath, VirtualPath};
use std::time::SystemTime;
fn make_file_meta(id: i64, path: &str, size: u64) -> FileMeta {
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(format!("/test/{}", path)),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from(path),
},
size,
mtime: SystemTime::UNIX_EPOCH,
content_hash: None,
audio: None,
}
}
fn make_scanned_file(path: &str, size: u64) -> ScannedFile {
ScannedFile {
path: PathBuf::from(path),
origin_id: OriginId::from("test"),
size,
mtime: SystemTime::UNIX_EPOCH,
}
}
#[test]
fn test_is_modified_size_change() {
let detector = DeltaDetector::new();
let cached = make_file_meta(1, "test.flac", 1000);
let scanned = make_scanned_file("test.flac", 2000);
assert!(detector.is_modified_scan(&cached, &scanned));
}
#[test]
fn test_is_modified_same() {
let detector = DeltaDetector::new();
let cached = make_file_meta(1, "test.flac", 1000);
let scanned = make_scanned_file("test.flac", 1000);
assert!(!detector.is_modified_scan(&cached, &scanned));
}
#[test]
fn test_is_audio_file() {
assert!(DeltaDetector::is_audio_file("track.flac"));
assert!(DeltaDetector::is_audio_file("song.MP3"));
assert!(DeltaDetector::is_audio_file("audio.ogg"));
assert!(!DeltaDetector::is_audio_file("readme.txt"));
assert!(!DeltaDetector::is_audio_file("cover.jpg"));
}
#[test]
fn test_compute_diff() {
let detector = DeltaDetector::new();
let old_chunks = vec![
ManifestChunk {
hash: ChunkHash::from_bytes(b"A"),
offset: 0,
size: 256,
},
ManifestChunk {
hash: ChunkHash::from_bytes(b"B"),
offset: 256,
size: 256,
},
ManifestChunk {
hash: ChunkHash::from_bytes(b"C"),
offset: 512,
size: 256,
},
];
let new_chunks = vec![
ManifestChunk {
hash: ChunkHash::from_bytes(b"A"),
offset: 0,
size: 256,
},
ManifestChunk {
hash: ChunkHash::from_bytes(b"D"),
offset: 256,
size: 256,
},
ManifestChunk {
hash: ChunkHash::from_bytes(b"C"),
offset: 512,
size: 256,
},
];
let diff = detector.compute_diff(&old_chunks, &new_chunks);
assert_eq!(diff.reuse.len(), 2);
assert_eq!(diff.fetch.len(), 1);
assert_eq!(diff.orphaned.len(), 1);
}
}
crates/musicfs-sync/src/lib.rs
-7
View File
@@ -1,7 +0,0 @@
pub mod cdc;
pub mod delta;
pub mod watcher;
pub use cdc::{CdcChunker, Chunk, ChunkRef};
pub use delta::{ChangeSet, DeltaDetector, DeltaError, ManifestChunk, ManifestDiff};
pub use watcher::{OriginWatcher, WatchError, WatchHandle};
crates/musicfs-sync/src/watcher.rs
-218
View File
@@ -1,218 +0,0 @@
use musicfs_core::{Event, EventBus, OriginId, VirtualPath};
use notify::{Config, RecommendedWatcher, RecursiveMode, Watcher};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::mpsc;
use tracing::{error, info, info_span, trace, Instrument};
const DEBOUNCE_MS: u64 = 200;
pub struct OriginWatcher {
origin_id: OriginId,
root: PathBuf,
event_bus: Arc<EventBus>,
}
impl OriginWatcher {
pub fn new(origin_id: OriginId, root: PathBuf, event_bus: Arc<EventBus>) -> Self {
Self {
origin_id,
root,
event_bus,
}
}
pub fn start(self) -> WatchHandle {
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
let origin_id = self.origin_id.clone();
let root = self.root.clone();
let event_bus = self.event_bus.clone();
let origin_id_str = origin_id.to_string();
tokio::spawn(
async move {
if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await
{
error!("Watcher error: {}", e);
}
}
.instrument(info_span!("file_watcher", origin_id = %origin_id_str)),
);
WatchHandle { stop_tx }
}
async fn watch_loop(
origin_id: &OriginId,
root: &Path,
event_bus: &EventBus,
stop_rx: &mut mpsc::Receiver<()>,
) -> Result<(), WatchError> {
let (tx, mut rx) = mpsc::channel(100);
let mut watcher = RecommendedWatcher::new(
move |res: Result<notify::Event, notify::Error>| {
if let Ok(event) = res {
let _ = tx.blocking_send(event);
}
},
Config::default(),
)
.map_err(|e| WatchError::Init(e.to_string()))?;
watcher
.watch(root, RecursiveMode::Recursive)
.map_err(|e| WatchError::Watch(e.to_string()))?;
info!(origin_id = %origin_id, path = ?root, "Watcher started");
let mut debouncer: HashMap<PathBuf, Instant> = HashMap::new();
loop {
tokio::select! {
Some(event) = rx.recv() => {
Self::handle_notify_event(origin_id, root, event_bus, event, &mut debouncer);
}
_ = stop_rx.recv() => {
info!(origin_id = %origin_id, "Watcher stopped");
break;
}
}
}
Ok(())
}
fn handle_notify_event(
origin_id: &OriginId,
root: &Path,
event_bus: &EventBus,
event: notify::Event,
debouncer: &mut HashMap<PathBuf, Instant>,
) {
use notify::EventKind;
let now = Instant::now();
for path in event.paths {
let relative = match path.strip_prefix(root) {
Ok(p) => p.to_path_buf(),
Err(_) => continue,
};
if !Self::is_audio_file(&path) {
continue;
}
if let Some(last_seen) = debouncer.get(&relative) {
if now.duration_since(*last_seen).as_millis() < DEBOUNCE_MS as u128 {
trace!(origin_id = %origin_id, path = ?relative, "Debouncing event");
continue;
}
}
debouncer.insert(relative.clone(), now);
let vpath = VirtualPath::new(format!("/{}", relative.display()));
match event.kind {
EventKind::Create(_) => {
trace!(origin_id = %origin_id, path = ?relative, "File created");
event_bus.publish(Event::FileAdded {
path: vpath,
origin_id: origin_id.clone(),
});
}
EventKind::Remove(_) => {
trace!(origin_id = %origin_id, path = ?relative, "File removed");
event_bus.publish(Event::FileRemoved {
path: vpath,
file_id: None,
});
}
EventKind::Modify(_) => {
trace!(origin_id = %origin_id, path = ?relative, "File modified");
event_bus.publish(Event::FileModified { path: vpath });
}
_ => {}
}
}
}
fn is_audio_file(path: &Path) -> bool {
matches!(
path.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_lowercase())
.as_deref(),
Some("flac" | "mp3" | "ogg" | "wav" | "m4a" | "aac" | "opus")
)
}
}
pub struct WatchHandle {
stop_tx: mpsc::Sender<()>,
}
impl WatchHandle {
pub async fn stop(self) {
let _ = self.stop_tx.send(()).await;
}
}
impl Drop for WatchHandle {
fn drop(&mut self) {
trace!("WatchHandle dropped");
let _ = self.stop_tx.try_send(());
}
}
#[derive(Debug, thiserror::Error)]
pub enum WatchError {
#[error("Failed to initialize watcher: {0}")]
Init(String),
#[error("Failed to watch path: {0}")]
Watch(String),
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
use tempfile::TempDir;
#[tokio::test]
async fn test_watcher_detects_create() {
let dir = TempDir::new().unwrap();
let event_bus = Arc::new(EventBus::default());
let mut rx = event_bus.subscribe();
let watcher =
OriginWatcher::new(OriginId::from("test"), dir.path().to_path_buf(), event_bus);
let handle = watcher.start();
tokio::time::sleep(Duration::from_millis(100)).await;
std::fs::write(dir.path().join("test.flac"), b"audio").unwrap();
tokio::time::sleep(Duration::from_millis(300)).await;
let event = rx.try_recv();
assert!(matches!(event, Ok(Event::FileAdded { .. })));
handle.stop().await;
}
#[test]
fn test_is_audio_file() {
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.flac")));
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.MP3")));
assert!(!OriginWatcher::is_audio_file(Path::new("/music/cover.jpg")));
assert!(!OriginWatcher::is_audio_file(Path::new(
"/music/readme.txt"
)));
}
}
crates/musicfs-test-utils/Cargo.toml
-43
View File
@@ -1,43 +0,0 @@
[package]
name = "musicfs-test-utils"
version.workspace = true
edition.workspace = true
description = "Test utilities and fixtures for MusicFS resilience testing"
[dependencies]
musicfs-core = { path = "../musicfs-core" }
musicfs-origins = { path = "../musicfs-origins" }
musicfs-cas = { path = "../musicfs-cas" }
musicfs-cache = { path = "../musicfs-cache" }
musicfs-search = { path = "../musicfs-search" }
async-trait.workspace = true
tokio = { workspace = true, features = ["full", "sync", "time"] }
tracing.workspace = true
thiserror.workspace = true
parking_lot.workspace = true
tempfile.workspace = true
bytes.workspace = true
# Fault injection
fail = { version = "0.5", optional = true }
rlimit = { version = "0.10", optional = true }
nix = { version = "0.29", optional = true, features = ["signal", "process"] }
# Docker/network tests
noxious-client = { version = "1.0", optional = true }
reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls"] }
[features]
default = []
failpoints = ["fail/failpoints"]
process-tests = ["nix"]
resource-limits = ["rlimit"]
docker-tests = ["noxious-client", "reqwest"]
full = ["failpoints", "process-tests", "resource-limits", "docker-tests"]
[dev-dependencies]
tokio-test = "0.4"
tokio-util.workspace = true
sd-notify.workspace = true
libc.workspace = true
crates/musicfs-test-utils/src/assertions.rs
-204
View File
@@ -1,204 +0,0 @@
use musicfs_cas::CasError;
use musicfs_core::Error;
use std::time::{Duration, Instant};
pub fn assert_error_contains<T, E: std::fmt::Debug>(result: Result<T, E>, expected_text: &str) {
match result {
Ok(_) => panic!("Expected error containing '{}', but got Ok", expected_text),
Err(e) => {
let error_msg = format!("{:?}", e);
assert!(
error_msg.contains(expected_text),
"Expected error containing '{}', but got: {}",
expected_text,
error_msg
);
}
}
}
pub fn assert_io_error<T>(result: Result<T, Error>) {
match result {
Err(Error::Io(_)) => (),
Err(e) => panic!("Expected Io error, got: {:?}", e),
Ok(_) => panic!("Expected Io error, got Ok"),
}
}
pub fn assert_cas_io_error<T>(result: Result<T, CasError>) {
match result {
Err(CasError::Io(_)) => (),
Err(e) => panic!("Expected CasError::Io, got: {:?}", e),
Ok(_) => panic!("Expected CasError::Io, got Ok"),
}
}
pub fn assert_cas_not_found<T>(result: Result<T, CasError>) {
match result {
Err(CasError::NotFound(_)) => (),
Err(e) => panic!("Expected CasError::NotFound, got: {:?}", e),
Ok(_) => panic!("Expected CasError::NotFound, got Ok"),
}
}
pub fn assert_cas_integrity_error<T>(result: Result<T, CasError>) {
match result {
Err(CasError::IntegrityError { .. }) => (),
Err(e) => panic!("Expected CasError::IntegrityError, got: {:?}", e),
Ok(_) => panic!("Expected CasError::IntegrityError, got Ok"),
}
}
pub fn assert_file_not_found<T>(result: Result<T, Error>) {
match result {
Err(Error::FileNotFound(_)) => (),
Err(e) => panic!("Expected FileNotFound error, got: {:?}", e),
Ok(_) => panic!("Expected FileNotFound error, got Ok"),
}
}
pub fn assert_origin_error<T>(result: Result<T, Error>) {
match result {
Err(Error::Origin(_)) => (),
Err(e) => panic!("Expected Origin error, got: {:?}", e),
Ok(_) => panic!("Expected Origin error, got Ok"),
}
}
pub fn assert_timeout_error<T>(result: Result<T, Error>) {
match result {
Err(Error::Timeout(_)) => (),
Err(e) => panic!("Expected Timeout error, got: {:?}", e),
Ok(_) => panic!("Expected Timeout error, got Ok"),
}
}
pub struct TimedAssertion {
start: Instant,
min_duration: Option<Duration>,
max_duration: Option<Duration>,
}
impl TimedAssertion {
pub fn new() -> Self {
Self {
start: Instant::now(),
min_duration: None,
max_duration: None,
}
}
pub fn expect_at_least(mut self, duration: Duration) -> Self {
self.min_duration = Some(duration);
self
}
pub fn expect_at_most(mut self, duration: Duration) -> Self {
self.max_duration = Some(duration);
self
}
pub fn assert_elapsed(self) {
let elapsed = self.start.elapsed();
if let Some(min) = self.min_duration {
assert!(
elapsed >= min,
"Expected at least {:?}, but only {:?} elapsed",
min,
elapsed
);
}
if let Some(max) = self.max_duration {
assert!(
elapsed <= max,
"Expected at most {:?}, but {:?} elapsed",
max,
elapsed
);
}
}
}
impl Default for TimedAssertion {
fn default() -> Self {
Self::new()
}
}
pub async fn assert_completes_within<F, T>(future: F, timeout: Duration) -> T
where
F: std::future::Future<Output = T>,
{
tokio::time::timeout(timeout, future)
.await
.expect(&format!("Operation did not complete within {:?}", timeout))
}
pub async fn assert_times_out<F, T>(future: F, timeout: Duration)
where
F: std::future::Future<Output = T>,
{
match tokio::time::timeout(timeout, future).await {
Ok(_) => panic!("Expected operation to time out, but it completed"),
Err(_) => (),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_assert_error_contains() {
let result: Result<(), Error> = Err(Error::Origin("connection refused".into()));
assert_error_contains(result, "connection");
}
#[test]
#[should_panic(expected = "Expected error containing")]
fn test_assert_error_contains_failure() {
let result: Result<(), Error> = Err(Error::Origin("something else".into()));
assert_error_contains(result, "connection");
}
#[test]
fn test_assert_io_error() {
let result: Result<(), Error> = Err(Error::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"test",
)));
assert_io_error(result);
}
#[test]
fn test_timed_assertion_at_least() {
let timer = TimedAssertion::new().expect_at_least(Duration::from_millis(10));
std::thread::sleep(Duration::from_millis(15));
timer.assert_elapsed();
}
#[test]
fn test_timed_assertion_at_most() {
let timer = TimedAssertion::new().expect_at_most(Duration::from_millis(100));
timer.assert_elapsed();
}
#[tokio::test]
async fn test_assert_completes_within() {
let result = assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
assert_eq!(result, 42);
}
#[tokio::test]
async fn test_assert_times_out() {
assert_times_out(
async {
tokio::time::sleep(Duration::from_secs(10)).await;
},
Duration::from_millis(10),
)
.await;
}
}
crates/musicfs-test-utils/src/faulty_cas.rs
-250
View File
@@ -1,250 +0,0 @@
use bytes::Bytes;
use musicfs_cas::{CasConfig, CasError, CasStore, DedupStats};
use musicfs_core::ChunkHash;
use std::io::{self, ErrorKind};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
pub struct FaultyCasStore {
inner: Arc<CasStore>,
inject_enospc: AtomicBool,
inject_eio_on_read: AtomicBool,
inject_eio_on_write: AtomicBool,
inject_corruption: AtomicBool,
fail_after_n_puts: AtomicUsize,
put_count: AtomicUsize,
}
impl FaultyCasStore {
pub fn new(inner: Arc<CasStore>) -> Self {
Self {
inner,
inject_enospc: AtomicBool::new(false),
inject_eio_on_read: AtomicBool::new(false),
inject_eio_on_write: AtomicBool::new(false),
inject_corruption: AtomicBool::new(false),
fail_after_n_puts: AtomicUsize::new(usize::MAX),
put_count: AtomicUsize::new(0),
}
}
pub async fn open(config: CasConfig) -> Result<Self, CasError> {
let store = CasStore::open(config).await?;
Ok(Self::new(Arc::new(store)))
}
pub fn set_inject_enospc(&self, enabled: bool) {
self.inject_enospc.store(enabled, Ordering::SeqCst);
}
pub fn set_inject_eio_on_read(&self, enabled: bool) {
self.inject_eio_on_read.store(enabled, Ordering::SeqCst);
}
pub fn set_inject_eio_on_write(&self, enabled: bool) {
self.inject_eio_on_write.store(enabled, Ordering::SeqCst);
}
pub fn set_inject_corruption(&self, enabled: bool) {
self.inject_corruption.store(enabled, Ordering::SeqCst);
}
pub fn set_fail_after_n_puts(&self, n: usize) {
self.fail_after_n_puts.store(n, Ordering::SeqCst);
self.put_count.store(0, Ordering::SeqCst);
}
pub fn reset_faults(&self) {
self.inject_enospc.store(false, Ordering::SeqCst);
self.inject_eio_on_read.store(false, Ordering::SeqCst);
self.inject_eio_on_write.store(false, Ordering::SeqCst);
self.inject_corruption.store(false, Ordering::SeqCst);
self.fail_after_n_puts.store(usize::MAX, Ordering::SeqCst);
self.put_count.store(0, Ordering::SeqCst);
}
pub fn put_count(&self) -> usize {
self.put_count.load(Ordering::SeqCst)
}
pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
let count = self.put_count.fetch_add(1, Ordering::SeqCst);
if self.inject_enospc.load(Ordering::SeqCst) {
return Err(CasError::Io(io::Error::new(
ErrorKind::Other,
"No space left on device (ENOSPC injected)",
)));
}
if self.inject_eio_on_write.load(Ordering::SeqCst) {
return Err(CasError::Io(io::Error::new(
ErrorKind::Other,
"Input/output error (EIO injected)",
)));
}
let threshold = self.fail_after_n_puts.load(Ordering::SeqCst);
if count >= threshold {
return Err(CasError::Io(io::Error::new(
ErrorKind::Other,
"Injected failure after N puts",
)));
}
self.inner.put(data).await
}
pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
if self.inject_eio_on_read.load(Ordering::SeqCst) {
return Err(CasError::Io(io::Error::new(
ErrorKind::Other,
"Input/output error (EIO injected)",
)));
}
let data = self.inner.get(hash).await?;
if self.inject_corruption.load(Ordering::SeqCst) {
let mut corrupted = data.to_vec();
if !corrupted.is_empty() {
corrupted[0] = corrupted[0].wrapping_add(1);
}
return Err(CasError::IntegrityError {
expected: hash.as_hex(),
actual: ChunkHash::from_bytes(&corrupted).as_hex(),
});
}
Ok(data)
}
pub fn exists(&self, hash: &ChunkHash) -> bool {
self.inner.exists(hash)
}
pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
if self.inject_eio_on_write.load(Ordering::SeqCst) {
return Err(CasError::Io(io::Error::new(
ErrorKind::Other,
"Input/output error (EIO injected)",
)));
}
self.inner.delete(hash).await
}
pub fn current_size(&self) -> u64 {
self.inner.current_size()
}
pub fn max_size(&self) -> u64 {
self.inner.max_size()
}
pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
self.inner.list_chunks()
}
pub fn dedup_stats(&self) -> DedupStats {
self.inner.dedup_stats()
}
pub fn inner(&self) -> &Arc<CasStore> {
&self.inner
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
async fn test_store() -> (FaultyCasStore, TempDir) {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 1024 * 1024,
shard_levels: 2,
};
let store = FaultyCasStore::open(config).await.unwrap();
(store, dir)
}
#[tokio::test]
async fn test_healthy_passthrough() {
let (store, _dir) = test_store().await;
let data = b"test data";
let hash = store.put(data).await.unwrap();
let retrieved = store.get(&hash).await.unwrap();
assert_eq!(&retrieved[..], data);
}
#[tokio::test]
async fn test_inject_enospc() {
let (store, _dir) = test_store().await;
store.set_inject_enospc(true);
let result = store.put(b"test").await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, CasError::Io(_)));
store.set_inject_enospc(false);
assert!(store.put(b"test").await.is_ok());
}
#[tokio::test]
async fn test_inject_eio_on_read() {
let (store, _dir) = test_store().await;
let hash = store.put(b"test").await.unwrap();
store.set_inject_eio_on_read(true);
let result = store.get(&hash).await;
assert!(result.is_err());
store.set_inject_eio_on_read(false);
assert!(store.get(&hash).await.is_ok());
}
#[tokio::test]
async fn test_inject_corruption() {
let (store, _dir) = test_store().await;
let hash = store.put(b"test data").await.unwrap();
store.set_inject_corruption(true);
let result = store.get(&hash).await;
assert!(matches!(result, Err(CasError::IntegrityError { .. })));
}
#[tokio::test]
async fn test_fail_after_n_puts() {
let (store, _dir) = test_store().await;
store.set_fail_after_n_puts(2);
assert!(store.put(b"data1").await.is_ok());
assert!(store.put(b"data2").await.is_ok());
assert!(store.put(b"data3").await.is_err());
assert!(store.put(b"data4").await.is_err());
assert_eq!(store.put_count(), 4);
}
#[tokio::test]
async fn test_reset_faults() {
let (store, _dir) = test_store().await;
store.set_inject_enospc(true);
store.set_inject_eio_on_read(true);
store.set_fail_after_n_puts(1);
store.reset_faults();
assert!(store.put(b"test").await.is_ok());
let hash = store.put(b"test2").await.unwrap();
assert!(store.get(&hash).await.is_ok());
}
}
crates/musicfs-test-utils/src/faulty_origin.rs
-328
View File
@@ -1,328 +0,0 @@
use async_trait::async_trait;
use musicfs_core::{DirEntry, Error, FileStat, HealthStatus, OriginId, OriginType, Result};
use musicfs_origins::{Origin, WatchCallback, WatchHandle};
use parking_lot::RwLock;
use std::io::{self, ErrorKind};
use std::path::Path;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::io::AsyncRead;
#[derive(Debug, Clone)]
pub enum FailMode {
Healthy,
FailEveryNth(usize),
FailAfterN(usize),
TimeoutMs(u64),
PartialRead { max_bytes: usize },
ReturnError(ErrorKind),
}
impl Default for FailMode {
fn default() -> Self {
FailMode::Healthy
}
}
pub struct FaultyOrigin {
inner: Arc<dyn Origin>,
fail_mode: Arc<RwLock<FailMode>>,
call_count: AtomicUsize,
}
impl FaultyOrigin {
pub fn new(inner: Arc<dyn Origin>, mode: FailMode) -> Self {
Self {
inner,
fail_mode: Arc::new(RwLock::new(mode)),
call_count: AtomicUsize::new(0),
}
}
pub fn wrap(inner: impl Origin + 'static) -> Self {
Self::new(Arc::new(inner), FailMode::Healthy)
}
pub fn set_mode(&self, mode: FailMode) {
*self.fail_mode.write() = mode;
}
pub fn call_count(&self) -> usize {
self.call_count.load(Ordering::SeqCst)
}
pub fn reset_count(&self) {
self.call_count.store(0, Ordering::SeqCst);
}
fn increment_and_check(&self) -> Option<Error> {
let count = self.call_count.fetch_add(1, Ordering::SeqCst) + 1;
let mode = self.fail_mode.read();
match *mode {
FailMode::Healthy => None,
FailMode::FailEveryNth(n) if n > 0 && count % n == 0 => {
Some(Error::Origin("Injected failure (every Nth)".into()))
}
FailMode::FailEveryNth(_) => None,
FailMode::FailAfterN(n) if count > n => {
Some(Error::Origin("Injected failure (after N)".into()))
}
FailMode::FailAfterN(_) => None,
FailMode::TimeoutMs(_) => None,
FailMode::PartialRead { .. } => None,
FailMode::ReturnError(kind) => {
Some(Error::Io(io::Error::new(kind, "Injected I/O error")))
}
}
}
async fn maybe_timeout(&self) -> Option<Error> {
let mode = self.fail_mode.read().clone();
if let FailMode::TimeoutMs(ms) = mode {
tokio::time::sleep(Duration::from_millis(ms)).await;
Some(Error::Timeout("Injected timeout".into()))
} else {
None
}
}
fn truncate_if_partial(&self, mut data: Vec<u8>) -> Vec<u8> {
let mode = self.fail_mode.read();
if let FailMode::PartialRead { max_bytes } = *mode {
data.truncate(max_bytes);
}
data
}
}
#[async_trait]
impl Origin for FaultyOrigin {
fn id(&self) -> &OriginId {
self.inner.id()
}
fn origin_type(&self) -> OriginType {
self.inner.origin_type()
}
fn display_name(&self) -> &str {
self.inner.display_name()
}
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
if let Some(err) = self.increment_and_check() {
return Err(err);
}
if let Some(err) = self.maybe_timeout().await {
return Err(err);
}
self.inner.readdir(path).await
}
async fn stat(&self, path: &Path) -> Result<FileStat> {
if let Some(err) = self.increment_and_check() {
return Err(err);
}
if let Some(err) = self.maybe_timeout().await {
return Err(err);
}
self.inner.stat(path).await
}
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
if let Some(err) = self.increment_and_check() {
return Err(err);
}
if let Some(err) = self.maybe_timeout().await {
return Err(err);
}
let data = self.inner.read(path, offset, size).await?;
Ok(self.truncate_if_partial(data))
}
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
if let Some(err) = self.increment_and_check() {
return Err(err);
}
if let Some(err) = self.maybe_timeout().await {
return Err(err);
}
let data = self.inner.read_full(path).await?;
Ok(self.truncate_if_partial(data))
}
async fn exists(&self, path: &Path) -> Result<bool> {
if let Some(err) = self.increment_and_check() {
return Err(err);
}
if let Some(err) = self.maybe_timeout().await {
return Err(err);
}
self.inner.exists(path).await
}
async fn health(&self) -> HealthStatus {
let mode = self.fail_mode.read().clone();
match mode {
FailMode::Healthy => self.inner.health().await,
FailMode::ReturnError(_) => HealthStatus::Unhealthy,
FailMode::TimeoutMs(ms) => {
tokio::time::sleep(Duration::from_millis(ms)).await;
HealthStatus::Unhealthy
}
FailMode::FailAfterN(n) if self.call_count.load(Ordering::SeqCst) >= n => {
HealthStatus::Unhealthy
}
_ => self.inner.health().await,
}
}
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
if let Some(err) = self.increment_and_check() {
return Err(err);
}
if let Some(err) = self.maybe_timeout().await {
return Err(err);
}
self.inner.open_read(path).await
}
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
self.inner.watch(path, callback).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::SystemTime;
struct MockOrigin {
id: OriginId,
}
impl MockOrigin {
fn new(id: &str) -> Self {
Self {
id: OriginId::from(id),
}
}
}
#[async_trait]
impl Origin for MockOrigin {
fn id(&self) -> &OriginId {
&self.id
}
fn origin_type(&self) -> OriginType {
OriginType::Local
}
fn display_name(&self) -> &str {
"mock"
}
async fn readdir(&self, _path: &Path) -> Result<Vec<DirEntry>> {
Ok(vec![])
}
async fn stat(&self, _path: &Path) -> Result<FileStat> {
Ok(FileStat {
size: 1000,
mtime: SystemTime::now(),
is_dir: false,
})
}
async fn read(&self, _path: &Path, _offset: u64, size: u32) -> Result<Vec<u8>> {
Ok(vec![0u8; size as usize])
}
async fn read_full(&self, _path: &Path) -> Result<Vec<u8>> {
Ok(vec![0u8; 100])
}
async fn exists(&self, _path: &Path) -> Result<bool> {
Ok(true)
}
async fn health(&self) -> HealthStatus {
HealthStatus::Healthy
}
async fn open_read(&self, _path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
Err(Error::Origin("Not implemented".into()))
}
async fn watch(&self, _path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
Err(Error::Origin("Not implemented".into()))
}
}
#[tokio::test]
async fn test_healthy_passthrough() {
let inner = Arc::new(MockOrigin::new("test"));
let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
let result = faulty.stat(Path::new("/test")).await;
assert!(result.is_ok());
assert_eq!(faulty.call_count(), 1);
}
#[tokio::test]
async fn test_fail_every_nth() {
let inner = Arc::new(MockOrigin::new("test"));
let faulty = FaultyOrigin::new(inner, FailMode::FailEveryNth(2));
assert!(faulty.stat(Path::new("/test")).await.is_ok());
assert!(faulty.stat(Path::new("/test")).await.is_err());
assert!(faulty.stat(Path::new("/test")).await.is_ok());
assert!(faulty.stat(Path::new("/test")).await.is_err());
assert_eq!(faulty.call_count(), 4);
}
#[tokio::test]
async fn test_fail_after_n() {
let inner = Arc::new(MockOrigin::new("test"));
let faulty = FaultyOrigin::new(inner, FailMode::FailAfterN(2));
assert!(faulty.stat(Path::new("/test")).await.is_ok());
assert!(faulty.stat(Path::new("/test")).await.is_ok());
assert!(faulty.stat(Path::new("/test")).await.is_err());
assert!(faulty.stat(Path::new("/test")).await.is_err());
}
#[tokio::test]
async fn test_partial_read() {
let inner = Arc::new(MockOrigin::new("test"));
let faulty = FaultyOrigin::new(inner, FailMode::PartialRead { max_bytes: 10 });
let data = faulty.read(Path::new("/test"), 0, 100).await.unwrap();
assert_eq!(data.len(), 10);
}
#[tokio::test]
async fn test_mode_change_mid_test() {
let inner = Arc::new(MockOrigin::new("test"));
let faulty = FaultyOrigin::new(inner, FailMode::ReturnError(ErrorKind::ConnectionRefused));
assert!(faulty.stat(Path::new("/test")).await.is_err());
faulty.set_mode(FailMode::Healthy);
assert!(faulty.stat(Path::new("/test")).await.is_ok());
}
#[tokio::test]
async fn test_health_reflects_mode() {
let inner = Arc::new(MockOrigin::new("test"));
let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
assert_eq!(faulty.health().await, HealthStatus::Healthy);
faulty.set_mode(FailMode::ReturnError(ErrorKind::ConnectionRefused));
assert_eq!(faulty.health().await, HealthStatus::Unhealthy);
}
}
crates/musicfs-test-utils/src/fixtures.rs
-254
View File
@@ -1,254 +0,0 @@
use musicfs_cache::TreeBuilder;
use musicfs_cas::{CasConfig, CasStore};
use musicfs_core::{AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath};
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use std::time::SystemTime;
use tempfile::TempDir;
pub fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(vpath),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from(vpath),
},
size,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
}
}
pub fn make_file_meta_with_origin(id: i64, vpath: &str, size: u64, origin_id: &str) -> FileMeta {
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(vpath),
real_path: RealPath {
origin_id: OriginId::from(origin_id),
path: PathBuf::from(vpath),
},
size,
mtime: SystemTime::now(),
content_hash: None,
audio: None,
}
}
pub fn make_audio_meta(artist: &str, album: &str, title: &str) -> AudioMeta {
AudioMeta {
title: Some(title.to_string()),
artist: Some(artist.to_string()),
album: Some(album.to_string()),
album_artist: None,
genre: None,
year: None,
track: None,
disc: None,
duration_ms: Some(180_000),
bitrate: Some(320),
sample_rate: Some(44100),
format: AudioFormat::Flac,
..Default::default()
}
}
pub fn make_audio_file(
id: i64,
vpath: &str,
size: u64,
artist: &str,
album: &str,
title: &str,
) -> FileMeta {
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(vpath),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from(vpath),
},
size,
mtime: SystemTime::now(),
content_hash: None,
audio: Some(make_audio_meta(artist, album, title)),
}
}
pub fn make_audio_file_full(
id: i64,
vpath: &str,
size: u64,
artist: &str,
album: &str,
title: &str,
track: u32,
year: u32,
) -> FileMeta {
let mut audio = make_audio_meta(artist, album, title);
audio.track = Some(track);
audio.year = Some(year);
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(vpath),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from(vpath),
},
size,
mtime: SystemTime::now(),
content_hash: None,
audio: Some(audio),
}
}
pub struct TestCasStore {
pub store: Arc<CasStore>,
pub dir: TempDir,
}
pub async fn setup_test_cas() -> TestCasStore {
let dir = TempDir::new().expect("Failed to create temp dir for CAS");
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 100 * 1024 * 1024,
shard_levels: 2,
};
let store = CasStore::open(config)
.await
.expect("Failed to open CAS store");
TestCasStore {
store: Arc::new(store),
dir,
}
}
pub async fn setup_test_cas_with_size(max_size: u64) -> TestCasStore {
let dir = TempDir::new().expect("Failed to create temp dir for CAS");
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size,
shard_levels: 2,
};
let store = CasStore::open(config)
.await
.expect("Failed to open CAS store");
TestCasStore {
store: Arc::new(store),
dir,
}
}
pub fn setup_test_tree(files: &[FileMeta]) -> Arc<RwLock<musicfs_cache::VirtualTree>> {
let mut builder = TreeBuilder::new();
for file in files {
builder.add_file(file);
}
Arc::new(RwLock::new(builder.build()))
}
pub fn create_test_file(dir: &Path, relative_path: &str, content: &[u8]) -> PathBuf {
let full_path = dir.join(relative_path);
if let Some(parent) = full_path.parent() {
std::fs::create_dir_all(parent).expect("Failed to create parent directories");
}
std::fs::write(&full_path, content).expect("Failed to write test file");
full_path
}
pub fn create_test_dir_structure(base: &Path, structure: &[&str]) {
for path in structure {
let full_path = base.join(path);
if path.ends_with('/') {
std::fs::create_dir_all(&full_path).expect("Failed to create directory");
} else {
if let Some(parent) = full_path.parent() {
std::fs::create_dir_all(parent).expect("Failed to create parent");
}
std::fs::write(&full_path, format!("content of {}", path))
.expect("Failed to write file");
}
}
}
pub struct TestOriginDir {
pub dir: TempDir,
}
impl TestOriginDir {
pub fn new() -> Self {
Self {
dir: TempDir::new().expect("Failed to create origin temp dir"),
}
}
pub fn add_file(&self, path: &str, content: &[u8]) -> PathBuf {
create_test_file(self.dir.path(), path, content)
}
pub fn add_audio_file(&self, path: &str) -> PathBuf {
let fake_audio = b"FAKE_FLAC_HEADER_FOR_TESTING_ONLY";
self.add_file(path, fake_audio)
}
pub fn path(&self) -> &Path {
self.dir.path()
}
}
impl Default for TestOriginDir {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_make_file_meta() {
let meta = make_file_meta(1, "/Artist/Album/Track.flac", 1000);
assert_eq!(meta.id.0, 1);
assert_eq!(meta.virtual_path.as_str(), "/Artist/Album/Track.flac");
assert_eq!(meta.size, 1000);
assert!(meta.audio.is_none());
}
#[test]
fn test_make_audio_file() {
let meta = make_audio_file(1, "/path.flac", 5000, "Artist", "Album", "Title");
assert!(meta.audio.is_some());
let audio = meta.audio.unwrap();
assert_eq!(audio.artist, Some("Artist".to_string()));
assert_eq!(audio.album, Some("Album".to_string()));
assert_eq!(audio.title, Some("Title".to_string()));
}
#[tokio::test]
async fn test_setup_test_cas() {
let test_cas = setup_test_cas().await;
let hash = test_cas.store.put(b"test data").await.unwrap();
assert!(test_cas.store.exists(&hash));
}
#[test]
fn test_setup_test_tree() {
let files = vec![
make_file_meta(1, "/A/B/1.flac", 100),
make_file_meta(2, "/A/B/2.flac", 200),
];
let tree = setup_test_tree(&files);
let guard = tree.read().unwrap();
assert!(guard.file_count() > 0);
}
#[test]
fn test_origin_dir() {
let origin = TestOriginDir::new();
let path = origin.add_file("artist/album/track.flac", b"content");
assert!(path.exists());
}
}
crates/musicfs-test-utils/src/lib.rs
-9
View File
@@ -1,9 +0,0 @@
pub mod assertions;
pub mod faulty_cas;
pub mod faulty_origin;
pub mod fixtures;
pub use assertions::*;
pub use faulty_cas::FaultyCasStore;
pub use faulty_origin::{FailMode, FaultyOrigin};
pub use fixtures::*;
crates/musicfs-test-utils/tests/docker_network.rs
-141
View File
@@ -1,141 +0,0 @@
#![cfg(feature = "docker-tests")]
use musicfs_core::{OriginId, OriginType};
use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
use noxious_client::{Client, StreamDirection, Toxic, ToxicKind};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use tempfile::TempDir;
const TOXIPROXY_API: &str = "http://localhost:8474";
const TOXIPROXY_LISTEN: &str = "localhost:18080";
const UPSTREAM_ADDR: &str = "minio:9000";
async fn require_toxiproxy() {
let available = match reqwest::get(format!("{}/version", TOXIPROXY_API)).await {
Ok(resp) => resp.status().is_success(),
Err(_) => false,
};
assert!(
available,
"Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d",
TOXIPROXY_API
);
}
#[tokio::test]
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
async fn test_toxiproxy_latency_injection() {
require_toxiproxy().await;
let client = Client::new(TOXIPROXY_API);
let proxy = client
.create_proxy("minio_latency", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
.await
.expect("Failed to create proxy");
let toxic = Toxic {
name: "latency_downstream".to_string(),
kind: ToxicKind::Latency {
latency: 500,
jitter: 100,
},
direction: StreamDirection::Downstream,
toxicity: 1.0,
};
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
let start = std::time::Instant::now();
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(400),
"Latency should be injected, got {:?}",
elapsed
);
proxy.delete().await.expect("Failed to cleanup proxy");
}
#[tokio::test]
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
async fn test_toxiproxy_timeout_simulates_network_partition() {
require_toxiproxy().await;
let client = Client::new(TOXIPROXY_API);
let proxy = client
.create_proxy("minio_partition", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
.await
.expect("Failed to create proxy");
let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
assert!(result.is_ok(), "Should reach MinIO through proxy initially");
let toxic = Toxic {
name: "timeout".to_string(),
kind: ToxicKind::Timeout { timeout: 0 },
direction: StreamDirection::Downstream,
toxicity: 1.0,
};
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
let result = tokio::time::timeout(
Duration::from_secs(2),
reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)),
)
.await;
assert!(
result.is_err() || result.unwrap().is_err(),
"Should timeout during partition"
);
proxy
.remove_toxic("timeout")
.await
.expect("Failed to remove toxic");
tokio::time::sleep(Duration::from_millis(100)).await;
let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
assert!(result.is_ok(), "Should reach MinIO after partition heals");
proxy.delete().await.expect("Failed to cleanup proxy");
}
#[tokio::test]
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
async fn test_toxiproxy_slow_close_throttles_responses() {
require_toxiproxy().await;
let client = Client::new(TOXIPROXY_API);
let proxy = client
.create_proxy("minio_slow", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
.await
.expect("Failed to create proxy");
let toxic = Toxic {
name: "slow_close".to_string(),
kind: ToxicKind::SlowClose { delay: 1000 },
direction: StreamDirection::Downstream,
toxicity: 1.0,
};
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
let start = std::time::Instant::now();
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(800),
"Slow close should delay response, got {:?}",
elapsed
);
proxy.delete().await.expect("Failed to cleanup proxy");
}
crates/musicfs-test-utils/tests/resilience.rs
-838
View File
@@ -1,838 +0,0 @@
use musicfs_cache::{Database, VirtualTree, ROOT_INODE};
use musicfs_cas::{CasConfig, CasStore};
use musicfs_core::supervisor::{TaskStatus, TaskSupervisor};
use musicfs_core::{
AudioMeta, FileId, FileMeta, HealthStatus, OriginId, OriginType, RealPath, VirtualPath,
};
use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
use musicfs_search::SearchIndex;
use musicfs_test_utils::{FailMode, FaultyOrigin};
use std::collections::HashMap;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant, UNIX_EPOCH};
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
fn setup_test_file(dir: &TempDir, name: &str, content: &[u8]) -> PathBuf {
let path = dir.path().join(name);
std::fs::write(&path, content).unwrap();
path
}
async fn setup_cas(dir: &Path) -> CasStore {
CasStore::open(CasConfig {
chunks_dir: dir.join("chunks"),
max_size: 100 * 1024 * 1024,
shard_levels: 2,
})
.await
.unwrap()
}
fn create_faulty_origin(id: &str, dir: &TempDir, mode: FailMode) -> Arc<FaultyOrigin> {
let inner = Arc::new(LocalOrigin::new(
OriginId::from(id),
dir.path().to_path_buf(),
));
Arc::new(FaultyOrigin::new(inner, mode))
}
fn make_file_meta(id: i64, path: &str, size: u64) -> FileMeta {
let name = Path::new(path)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("unknown")
.to_string();
FileMeta {
id: FileId(id),
virtual_path: VirtualPath::new(path),
real_path: RealPath {
origin_id: OriginId::from("test"),
path: PathBuf::from(path),
},
size,
mtime: UNIX_EPOCH,
content_hash: None,
audio: Some(AudioMeta {
title: Some(name),
..Default::default()
}),
}
}
#[tokio::test]
async fn test_sqlite_integrity_check_detects_corruption() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("test.db");
{
let db = Database::open(&db_path).unwrap();
db.upsert_file(
&OriginId::from("test"),
Path::new("/test.flac"),
&VirtualPath::new("/Test.flac"),
&AudioMeta::default(),
UNIX_EPOCH,
1000,
)
.unwrap();
}
let mut data = std::fs::read(&db_path).unwrap();
let mid = data.len() / 2;
data[mid..mid + 100].fill(0xFF);
std::fs::write(&db_path, &data).unwrap();
let result = Database::open_with_integrity_check(&db_path);
assert!(result.is_err());
}
#[tokio::test]
async fn test_tantivy_corruption_triggers_rebuild() {
let dir = TempDir::new().unwrap();
let index_path = dir.path().join("search_idx");
{
let index = SearchIndex::open(&index_path).unwrap();
index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap();
}
std::fs::write(index_path.join("meta.json"), b"corrupted").unwrap();
let index = SearchIndex::open_with_recovery(&index_path).unwrap();
let results = index.search("a", 10).unwrap();
assert_eq!(results.len(), 0);
}
#[tokio::test]
async fn test_sled_corruption_triggers_repair() {
let dir = TempDir::new().unwrap();
let chunks_dir = dir.path().join("chunks");
let config = CasConfig {
chunks_dir: chunks_dir.clone(),
max_size: 10_000_000,
shard_levels: 2,
};
{
let store = CasStore::open(config.clone()).await.unwrap();
store.put(b"test data").await.unwrap();
}
let sled_dir = chunks_dir.join("index.sled");
if sled_dir.exists() {
for entry in std::fs::read_dir(&sled_dir).unwrap() {
let entry = entry.unwrap();
if entry.metadata().unwrap().is_file() {
std::fs::write(entry.path(), b"corrupted").unwrap();
}
}
}
let result = CasStore::open(config).await;
assert!(result.is_ok(), "sled should recover from corruption");
}
#[tokio::test]
async fn test_cas_put_handles_enospc() {
let dir = TempDir::new().unwrap();
let store = CasStore::open(CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 100,
shard_levels: 2,
})
.await
.unwrap();
let large_data = vec![0u8; 1000];
let result = store.put(&large_data).await;
assert!(
result.is_err(),
"Issue 2.8: CasStore should pre-check space and reject oversized write"
);
}
/// Demonstrates the PROBLEM with std::sync::RwLock: after a writer panic,
/// the lock is poisoned and all subsequent access fails with PoisonError.
/// This is why we use parking_lot::RwLock instead (see test_parking_lot_rwlock_survives_panic).
#[test]
fn test_poisoned_tree_lock_returns_eio_not_panic() {
use std::sync::{Arc, RwLock};
use std::thread;
let lock = Arc::new(RwLock::new(42));
let lock_clone = lock.clone();
let handle = thread::spawn(move || {
let _guard = lock_clone.write().unwrap();
panic!("writer panic");
});
let _ = handle.join();
let result = lock.read();
// std::sync::RwLock poisons after writer panic - this is the problem we fix with parking_lot
assert!(result.is_err(), "Issue 2.9: std::sync::RwLock should poison after writer panic (this demonstrates the problem)");
}
#[test]
fn test_parking_lot_rwlock_survives_panic() {
use parking_lot::RwLock;
use std::sync::Arc;
use std::thread;
let tree = Arc::new(RwLock::new(VirtualTree::new()));
let tree_clone = tree.clone();
let handle = thread::spawn(move || {
let _guard = tree_clone.write();
panic!("writer panic");
});
let _ = handle.join();
assert!(
tree.read().get(ROOT_INODE).is_some(),
"parking_lot RwLock should survive writer panic"
);
}
#[tokio::test]
async fn test_failover_on_primary_death() {
let primary_dir = TempDir::new().unwrap();
let backup_dir = TempDir::new().unwrap();
setup_test_file(&primary_dir, "test.txt", b"primary");
setup_test_file(&backup_dir, "test.txt", b"backup");
let primary = create_faulty_origin(
"primary",
&primary_dir,
FailMode::ReturnError(ErrorKind::ConnectionRefused),
);
let backup = create_faulty_origin("backup", &backup_dir, FailMode::Healthy);
let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 1);
let monitor =
Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
let registry = Arc::new(OriginRegistry::new(monitor.clone()));
registry.register(primary.clone(), 1);
registry.register(backup.clone(), 2);
monitor.check_now(&OriginId::from("primary")).await;
monitor.check_now(&OriginId::from("backup")).await;
assert!(registry.health().is_unhealthy(&OriginId::from("primary")));
assert!(registry.health().is_healthy(&OriginId::from("backup")));
let path = RealPath {
origin_id: OriginId::from("backup"),
path: PathBuf::from("/test.txt"),
};
let candidates = registry.route_all(&path);
assert_eq!(candidates.len(), 1);
assert_eq!(candidates[0].id(), &OriginId::from("backup"));
}
#[tokio::test]
async fn test_origin_recovery_resumes_routing() {
let dir = TempDir::new().unwrap();
setup_test_file(&dir, "test.txt", b"content");
let faulty = create_faulty_origin(
"recovering",
&dir,
FailMode::ReturnError(ErrorKind::ConnectionRefused),
);
let mut thresholds = HashMap::new();
thresholds.insert(OriginType::Local, 1);
let monitor =
Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
monitor.add_origin(faulty.clone());
monitor.check_now(&OriginId::from("recovering")).await;
assert_eq!(
monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.status,
HealthStatus::Unhealthy
);
faulty.set_mode(FailMode::Healthy);
monitor.check_now(&OriginId::from("recovering")).await;
assert_eq!(
monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.status,
HealthStatus::Healthy
);
assert_eq!(
monitor
.get_state(&OriginId::from("recovering"))
.unwrap()
.consecutive_failures,
0
);
}
#[tokio::test]
async fn test_local_origin_health_check_has_timeout() {
let dir = TempDir::new().unwrap();
setup_test_file(&dir, "test.txt", b"content");
let slow = create_faulty_origin("slow", &dir, FailMode::TimeoutMs(5_000));
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
monitor.add_origin(slow.clone());
let start = Instant::now();
monitor.check_now(&OriginId::from("slow")).await;
let elapsed = start.elapsed();
assert!(
elapsed < Duration::from_secs(2),
"Issue 4.2.1: Health check should timeout in <2s, took {:?}",
elapsed
);
let state = monitor.get_state(&OriginId::from("slow")).unwrap();
assert_eq!(state.status, HealthStatus::Unhealthy);
}
#[tokio::test]
async fn test_health_checks_run_in_parallel() {
let slow1_dir = TempDir::new().unwrap();
let slow2_dir = TempDir::new().unwrap();
let slow3_dir = TempDir::new().unwrap();
let slow1 = create_faulty_origin("slow1", &slow1_dir, FailMode::TimeoutMs(200));
let slow2 = create_faulty_origin("slow2", &slow2_dir, FailMode::TimeoutMs(200));
let slow3 = create_faulty_origin("slow3", &slow3_dir, FailMode::TimeoutMs(200));
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
monitor.add_origin(slow1);
monitor.add_origin(slow2);
monitor.add_origin(slow3);
let start = Instant::now();
monitor.check_all().await;
let elapsed = start.elapsed();
assert!(
elapsed < Duration::from_millis(350),
"Issue 4.2.2: check_all() should run in parallel (sequential would take ~600ms), took {:?}",
elapsed
);
}
#[test]
fn test_tantivy_survives_uncommitted_crash() {
let dir = TempDir::new().unwrap();
let index_path = dir.path().join("search_idx");
{
let index = SearchIndex::open(&index_path).unwrap();
index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap();
index
.index_file(&make_file_meta(2, "/b.flac", 1000))
.unwrap();
}
let index = SearchIndex::open(&index_path).unwrap();
let results = index.search("a", 10).unwrap();
assert_eq!(results.len(), 1);
}
#[tokio::test]
#[cfg(feature = "resource-limits")]
async fn test_fd_exhaustion_handling() {
use rlimit::{getrlimit, setrlimit, Resource};
let (orig_soft, orig_hard) = getrlimit(Resource::NOFILE).unwrap();
setrlimit(Resource::NOFILE, 64, 64).unwrap();
let dir = TempDir::new().unwrap();
let result = CasStore::open(CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 1_000_000,
shard_levels: 2,
})
.await;
match result {
Ok(_store) => {}
Err(e) => {
let msg = format!("{}", e);
assert!(!msg.contains("panic"), "Should not panic on fd exhaustion");
}
}
setrlimit(Resource::NOFILE, orig_soft, orig_hard).unwrap();
}
#[tokio::test]
#[cfg(not(feature = "resource-limits"))]
async fn test_fd_exhaustion_handling() {
eprintln!("Skipping test_fd_exhaustion_handling: resource-limits feature not enabled");
}
#[tokio::test]
async fn test_corrupt_chunk_auto_refetched() {
use musicfs_cas::{ContentFetcher, FileReader};
use musicfs_origins::LocalOrigin;
let dir = TempDir::new().unwrap();
let origin_dir = TempDir::new().unwrap();
let test_content = b"original audio data for chunk test";
setup_test_file(&origin_dir, "test.flac", test_content);
let store = Arc::new(setup_cas(dir.path()).await);
let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin);
let file_meta = FileMeta {
id: FileId(1),
virtual_path: VirtualPath::new("/test.flac"),
real_path: RealPath {
origin_id: OriginId::from("local"),
path: PathBuf::from("/test.flac"),
},
size: test_content.len() as u64,
mtime: UNIX_EPOCH,
content_hash: None,
audio: None,
};
fetcher.register_file(file_meta);
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
let chunk_hash = manifest.chunks[0].hash;
let hex = chunk_hash.as_hex();
let chunk_path = dir
.path()
.join("chunks")
.join(&hex[0..2])
.join(&hex[2..4])
.join(&hex);
let mut corrupted = std::fs::read(&chunk_path).unwrap();
corrupted[0] = corrupted[0].wrapping_add(1);
std::fs::write(&chunk_path, &corrupted).unwrap();
let reader = FileReader::with_fetcher(store, fetcher);
reader.register_manifest(manifest);
let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
assert!(
result.is_ok(),
"Issue 6.4: Corrupted chunk should be auto-refetched from origin"
);
assert_eq!(
&result.unwrap()[..],
test_content,
"Data should match original after re-fetch"
);
}
#[tokio::test]
async fn test_missing_chunk_triggers_origin_fetch() {
use musicfs_cas::{ContentFetcher, FileReader};
use musicfs_origins::LocalOrigin;
let dir = TempDir::new().unwrap();
let origin_dir = TempDir::new().unwrap();
let test_content = b"test data for missing chunk";
setup_test_file(&origin_dir, "test.flac", test_content);
let store = Arc::new(setup_cas(dir.path()).await);
let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin);
let file_meta = FileMeta {
id: FileId(1),
virtual_path: VirtualPath::new("/test.flac"),
real_path: RealPath {
origin_id: OriginId::from("local"),
path: PathBuf::from("/test.flac"),
},
size: test_content.len() as u64,
mtime: UNIX_EPOCH,
content_hash: None,
audio: None,
};
fetcher.register_file(file_meta);
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
let chunk_hash = manifest.chunks[0].hash;
let hex = chunk_hash.as_hex();
let chunk_path = dir
.path()
.join("chunks")
.join(&hex[0..2])
.join(&hex[2..4])
.join(&hex);
std::fs::remove_file(&chunk_path).unwrap();
let reader = FileReader::with_fetcher(store, fetcher);
reader.register_manifest(manifest);
let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
assert!(
result.is_ok(),
"Issue 6.4: Missing chunk should be re-fetched from origin"
);
assert_eq!(
&result.unwrap()[..],
test_content,
"Data should match original after re-fetch"
);
}
#[tokio::test]
async fn test_passthrough_mode_when_cache_disk_dead() {
use musicfs_cas::ContentFetcher;
use musicfs_origins::LocalOrigin;
let dir = TempDir::new().unwrap();
let origin_dir = TempDir::new().unwrap();
let test_content = b"passthrough test data";
setup_test_file(&origin_dir, "test.flac", test_content);
let store = Arc::new(
CasStore::open(CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 10,
shard_levels: 2,
})
.await
.unwrap(),
);
let origin = Arc::new(LocalOrigin::new(
OriginId::from("local"),
origin_dir.path().to_path_buf(),
));
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
fetcher.register_origin(origin);
let file_meta = FileMeta {
id: FileId(1),
virtual_path: VirtualPath::new("/test.flac"),
real_path: RealPath {
origin_id: OriginId::from("local"),
path: PathBuf::from("/test.flac"),
},
size: test_content.len() as u64,
mtime: UNIX_EPOCH,
content_hash: None,
audio: None,
};
fetcher.register_file(file_meta);
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
assert!(
!manifest.chunks.is_empty(),
"Issue 6.6: Fetch should complete even when CAS write fails (passthrough mode)"
);
}
#[tokio::test]
async fn test_cas_size_tracking_is_correct() {
let dir = TempDir::new().unwrap();
let config = CasConfig {
chunks_dir: dir.path().join("chunks"),
max_size: 10_000_000,
shard_levels: 2,
};
let store = CasStore::open(config).await.unwrap();
let data = vec![0u8; 1000];
store.put(&data).await.unwrap();
assert!(
store.current_size() >= 1000,
"Issue C6: current_size should track chunk data (recursive), got {}",
store.current_size()
);
}
#[test]
fn test_pid_file_prevents_concurrent_mount() {
use std::fs::File;
use std::os::unix::io::AsRawFd;
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("musicfs.lock");
fn try_lock(path: &Path) -> Result<File, std::io::Error> {
let file = File::create(path)?;
let fd = file.as_raw_fd();
let ret = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
if ret != 0 {
return Err(std::io::Error::last_os_error());
}
Ok(file)
}
let lock1 = try_lock(&lock_path);
assert!(lock1.is_ok(), "Issue C9: First lock should succeed");
let lock2 = try_lock(&lock_path);
assert!(
lock2.is_err(),
"Issue C9: Second lock should fail (already held)"
);
drop(lock1);
let lock3 = try_lock(&lock_path);
assert!(
lock3.is_ok(),
"Issue C9: Third lock should succeed after first released"
);
}
#[test]
fn test_panic_hook_logs_to_tracing() {
use std::panic;
musicfs_core::install_panic_hook();
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
panic!("test panic message");
}));
assert!(result.is_err(), "Panic should have been caught");
}
#[test]
fn test_stale_mount_check_function_exists() {
let path = std::path::Path::new("/nonexistent/musicfs/mount");
assert!(
!path.exists(),
"Test path should not exist for this test to be meaningful"
);
}
#[test]
fn test_systemd_service_has_execstoppost() {
let service_path = std::path::Path::new("../../dist/musicfs.service");
if !service_path.exists() {
panic!(
"Issue 3.7: dist/musicfs.service does not exist at {:?}",
service_path
);
}
let content = std::fs::read_to_string(service_path).unwrap();
assert!(
content.contains("ExecStopPost") && content.contains("fusermount"),
"Issue 3.7: Service file should have ExecStopPost with fusermount for cleanup"
);
}
#[test]
fn test_sd_notify_ready_sent() {
use std::os::unix::net::UnixDatagram;
use tempfile::TempDir;
let dir = TempDir::new().unwrap();
let socket_path = dir.path().join("notify.sock");
let socket = UnixDatagram::bind(&socket_path).unwrap();
socket
.set_read_timeout(Some(Duration::from_secs(1)))
.unwrap();
std::env::set_var("NOTIFY_SOCKET", &socket_path);
let result = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]);
assert!(
result.is_ok(),
"sd_notify should succeed when NOTIFY_SOCKET is set"
);
let mut buf = [0u8; 256];
let len = socket.recv(&mut buf).unwrap();
let msg = std::str::from_utf8(&buf[..len]).unwrap();
assert!(
msg.contains("READY=1"),
"sd_notify should send READY=1, got: {}",
msg
);
std::env::remove_var("NOTIFY_SOCKET");
}
#[tokio::test]
async fn test_shutdown_cancels_background_tasks() {
let token = CancellationToken::new();
let stopped = Arc::new(AtomicBool::new(false));
let stopped_clone = stopped.clone();
let token_clone = token.clone();
tokio::spawn(async move {
token_clone.cancelled().await;
stopped_clone.store(true, Ordering::SeqCst);
});
assert!(!stopped.load(Ordering::SeqCst));
token.cancel();
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(stopped.load(Ordering::SeqCst));
}
#[tokio::test]
async fn test_shutdown_flushes_tantivy() {
let dir = TempDir::new().unwrap();
let idx_path = dir.path().join("idx");
{
let index = SearchIndex::open(&idx_path).unwrap();
index
.index_file(&make_file_meta(1, "/a.flac", 1000))
.unwrap();
index.commit().unwrap();
}
let index2 = SearchIndex::open(&idx_path).unwrap();
assert_eq!(index2.search("a", 10).unwrap().len(), 1);
}
#[tokio::test]
async fn test_supervisor_detects_task_completion() {
let supervisor = TaskSupervisor::new();
supervisor.spawn_supervised("fast", async {});
tokio::time::sleep(Duration::from_millis(50)).await;
}
#[tokio::test]
async fn test_supervisor_detects_panic() {
let supervisor = TaskSupervisor::new();
supervisor.spawn_supervised("panicker", async {
panic!("boom");
});
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(matches!(
supervisor.task_status("panicker"),
TaskStatus::Failed { .. }
));
}
#[tokio::test]
async fn test_supervisor_restarts_critical_task() {
let count = Arc::new(AtomicU32::new(0));
let c = count.clone();
let supervisor = TaskSupervisor::new();
supervisor.spawn_critical("restartable", move || {
let c = c.clone();
async move {
let n = c.fetch_add(1, Ordering::SeqCst);
if n == 0 {
panic!("first run fails");
}
loop {
tokio::time::sleep(Duration::from_secs(60)).await;
}
}
});
tokio::time::sleep(Duration::from_secs(2)).await;
assert_eq!(count.load(Ordering::SeqCst), 2);
assert!(matches!(
supervisor.task_status("restartable"),
TaskStatus::Running
));
}
#[tokio::test]
async fn test_sigterm_triggers_shutdown() {
use std::process::{Command, Stdio};
use std::time::Duration;
use tokio::time::timeout;
let musicfs_bin = std::env::var("CARGO_BIN_EXE_musicfs").ok();
if musicfs_bin.is_none() {
eprintln!(
"Skipping test_sigterm_triggers_shutdown: musicfs binary not available in test context"
);
return;
}
let bin_path = musicfs_bin.unwrap();
let temp_dir = tempfile::TempDir::new().unwrap();
let mountpoint = temp_dir.path().join("mount");
let origin = temp_dir.path().join("origin");
std::fs::create_dir_all(&mountpoint).unwrap();
std::fs::create_dir_all(&origin).unwrap();
let mut child = Command::new(&bin_path)
.args([
"mount",
"--origin",
origin.to_str().unwrap(),
mountpoint.to_str().unwrap(),
])
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn();
if child.is_err() {
eprintln!("Skipping test_sigterm_triggers_shutdown: failed to spawn musicfs");
return;
}
let mut child = child.unwrap();
tokio::time::sleep(Duration::from_millis(500)).await;
unsafe {
libc::kill(child.id() as i32, libc::SIGTERM);
}
let exit_result = timeout(Duration::from_secs(10), async {
loop {
match child.try_wait() {
Ok(Some(status)) => return status,
Ok(None) => tokio::time::sleep(Duration::from_millis(100)).await,
Err(_) => break,
}
}
child.wait().unwrap()
})
.await;
assert!(
exit_result.is_ok(),
"Issue 2.1: Process should exit within 10s after SIGTERM"
);
}