Move the files around
This commit is contained in:
Alexander
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "musicfs-cache"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-metadata = { path = "../musicfs-metadata" }
|
||||
rusqlite = { workspace = true, features = ["bundled"] }
|
||||
sled.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
image.workspace = true
|
||||
parking_lot.workspace = true
|
||||
chrono.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -0,0 +1,213 @@
|
||||
use image::ImageFormat;
|
||||
use musicfs_cas::CasStore;
|
||||
use musicfs_core::ChunkHash;
|
||||
use musicfs_metadata::artwork::{ArtSize, ArtType, Artwork};
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
const MAX_ARTWORK_INPUT_SIZE: usize = 10 * 1024 * 1024;
|
||||
|
||||
pub struct ArtworkCache {
|
||||
store: Arc<CasStore>,
|
||||
db_path: std::path::PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct CachedArtwork {
|
||||
pub file_id: i64,
|
||||
pub art_type: String,
|
||||
pub chunk_hash: ChunkHash,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
impl ArtworkCache {
|
||||
pub fn new(store: Arc<CasStore>, db_path: &Path) -> Result<Self, ArtworkError> {
|
||||
let db = rusqlite::Connection::open(db_path)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS artwork (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL,
|
||||
art_type TEXT NOT NULL,
|
||||
chunk_hash TEXT NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
UNIQUE(file_id, art_type)
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
info!(path = ?db_path, "Artwork cache opened");
|
||||
Ok(Self {
|
||||
store,
|
||||
db_path: db_path.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn store(&self, file_id: i64, artwork: &Artwork) -> Result<ChunkHash, ArtworkError> {
|
||||
trace!(
|
||||
file_id = file_id,
|
||||
size_bytes = artwork.data.len(),
|
||||
"Storing artwork"
|
||||
);
|
||||
if artwork.data.len() > MAX_ARTWORK_INPUT_SIZE {
|
||||
warn!(
|
||||
file_id = file_id,
|
||||
size = artwork.data.len(),
|
||||
max = MAX_ARTWORK_INPUT_SIZE,
|
||||
"Artwork too large"
|
||||
);
|
||||
return Err(ArtworkError::ImageTooLarge(artwork.data.len()));
|
||||
}
|
||||
|
||||
let hash = self.store.put(&artwork.data).await?;
|
||||
|
||||
let art_type_str = match artwork.art_type {
|
||||
ArtType::Front => "front",
|
||||
ArtType::Back => "back",
|
||||
ArtType::Other => "other",
|
||||
};
|
||||
|
||||
let db_path = self.db_path.clone();
|
||||
let art_type_clone = art_type_str.to_string();
|
||||
let hash_hex = hash.to_hex();
|
||||
let width = artwork.width;
|
||||
let height = artwork.height;
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let db = rusqlite::Connection::open(&db_path)?;
|
||||
db.execute(
|
||||
"INSERT OR REPLACE INTO artwork
|
||||
(file_id, art_type, chunk_hash, width, height)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
rusqlite::params![file_id, art_type_clone, hash_hex, width, height],
|
||||
)?;
|
||||
Ok::<_, ArtworkError>(())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))??;
|
||||
|
||||
debug!("Cached artwork for file {}", file_id);
|
||||
Ok(hash)
|
||||
}
|
||||
|
||||
pub async fn get(
|
||||
&self,
|
||||
file_id: i64,
|
||||
art_type: &str,
|
||||
size: ArtSize,
|
||||
) -> Result<Option<Vec<u8>>, ArtworkError> {
|
||||
trace!(file_id = file_id, art_type = %art_type, "Getting artwork");
|
||||
let db_path = self.db_path.clone();
|
||||
let art_type_clone = art_type.to_string();
|
||||
|
||||
let hash_hex: Option<String> = tokio::task::spawn_blocking(move || {
|
||||
let db = rusqlite::Connection::open(&db_path)?;
|
||||
db.query_row(
|
||||
"SELECT chunk_hash FROM artwork WHERE file_id = ?1 AND art_type = ?2",
|
||||
rusqlite::params![file_id, art_type_clone],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.ok()
|
||||
.ok_or(ArtworkError::NotFound)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))?
|
||||
.ok();
|
||||
|
||||
match hash_hex {
|
||||
Some(hex) => {
|
||||
trace!(file_id = file_id, "Artwork cache hit");
|
||||
let hash = ChunkHash::from_hex(&hex).ok_or(ArtworkError::InvalidHash)?;
|
||||
let data = self.store.get(&hash).await?;
|
||||
|
||||
match size {
|
||||
ArtSize::Full => Ok(Some(data.to_vec())),
|
||||
ArtSize::Thumbnail | ArtSize::Medium => {
|
||||
let resized = self.resize_on_demand(&data, size)?;
|
||||
Ok(Some(resized))
|
||||
}
|
||||
}
|
||||
}
|
||||
None => {
|
||||
trace!(file_id = file_id, "Artwork cache miss");
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn has(&self, file_id: i64, art_type: &str) -> Result<bool, ArtworkError> {
|
||||
let db_path = self.db_path.clone();
|
||||
let art_type_clone = art_type.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let db = rusqlite::Connection::open(&db_path)?;
|
||||
let count: i64 = db.query_row(
|
||||
"SELECT COUNT(*) FROM artwork WHERE file_id = ?1 AND art_type = ?2",
|
||||
rusqlite::params![file_id, art_type_clone],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok(count > 0)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))?
|
||||
}
|
||||
|
||||
fn resize_on_demand(&self, data: &[u8], size: ArtSize) -> Result<Vec<u8>, ArtworkError> {
|
||||
let max_dim = size.max_dimension().unwrap_or(300);
|
||||
let img = image::load_from_memory(data).map_err(|_| ArtworkError::InvalidImage)?;
|
||||
|
||||
if img.width() <= max_dim && img.height() <= max_dim {
|
||||
return Ok(data.to_vec());
|
||||
}
|
||||
|
||||
let resized = img.thumbnail(max_dim, max_dim);
|
||||
let mut output = Vec::new();
|
||||
let mut cursor = Cursor::new(&mut output);
|
||||
resized
|
||||
.write_to(&mut cursor, ImageFormat::Jpeg)
|
||||
.map_err(|_| ArtworkError::ResizeFailed)?;
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ArtworkError {
|
||||
#[error("database error: {0}")]
|
||||
Database(#[from] rusqlite::Error),
|
||||
|
||||
#[error("CAS error: {0}")]
|
||||
Cas(#[from] musicfs_cas::CasError),
|
||||
|
||||
#[error("invalid hash")]
|
||||
InvalidHash,
|
||||
|
||||
#[error("artwork not found")]
|
||||
NotFound,
|
||||
|
||||
#[error("image too large: {0} bytes (max 10MB)")]
|
||||
ImageTooLarge(usize),
|
||||
|
||||
#[error("invalid image data")]
|
||||
InvalidImage,
|
||||
|
||||
#[error("resize failed")]
|
||||
ResizeFailed,
|
||||
|
||||
#[error("spawn_blocking error: {0}")]
|
||||
SpawnBlocking(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_max_artwork_size() {
|
||||
assert_eq!(MAX_ARTWORK_INPUT_SIZE, 10 * 1024 * 1024);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
use musicfs_core::{
|
||||
AudioFormat, AudioMeta, ContentHash, Error, FileId, FileMeta, OriginId, RealPath, Result,
|
||||
VirtualPath,
|
||||
};
|
||||
use rusqlite::{params, Connection, OptionalExtension};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const SCHEMA: &str = include_str!("schema.sql");
|
||||
|
||||
pub struct Database {
|
||||
conn: Arc<Mutex<Connection>>,
|
||||
}
|
||||
|
||||
impl Database {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
debug!(?path, "Opening database");
|
||||
|
||||
let conn =
|
||||
Connection::open(path).map_err(|e| Error::Database(format!("open failed: {}", e)))?;
|
||||
|
||||
conn.execute_batch(SCHEMA)
|
||||
.map_err(|e| Error::Database(format!("schema init failed: {}", e)))?;
|
||||
|
||||
let db = Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
};
|
||||
let count = db.file_count().unwrap_or(0);
|
||||
info!(path = ?path, file_count = count, "Database opened");
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
pub fn open_with_integrity_check(path: &Path) -> Result<Self> {
|
||||
debug!(?path, "Opening database with integrity check");
|
||||
|
||||
let conn =
|
||||
Connection::open(path).map_err(|e| Error::Database(format!("open failed: {}", e)))?;
|
||||
|
||||
let integrity: String = conn
|
||||
.query_row("PRAGMA integrity_check(1)", [], |row| row.get(0))
|
||||
.map_err(|e| Error::Database(format!("integrity check failed: {}", e)))?;
|
||||
|
||||
if integrity != "ok" {
|
||||
warn!(path = ?path, result = %integrity, "Database integrity check failed");
|
||||
return Err(Error::DatabaseCorrupted(format!(
|
||||
"integrity check failed: {}",
|
||||
integrity
|
||||
)));
|
||||
}
|
||||
|
||||
conn.execute_batch(SCHEMA)
|
||||
.map_err(|e| Error::Database(format!("schema init failed: {}", e)))?;
|
||||
|
||||
let db = Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
};
|
||||
let count = db.file_count().unwrap_or(0);
|
||||
info!(path = ?path, file_count = count, "Database opened (integrity verified)");
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
pub fn open_memory() -> Result<Self> {
|
||||
let conn = Connection::open_in_memory()
|
||||
.map_err(|e| Error::Database(format!("open_in_memory failed: {}", e)))?;
|
||||
|
||||
conn.execute_batch(SCHEMA)
|
||||
.map_err(|e| Error::Database(format!("schema init failed: {}", e)))?;
|
||||
|
||||
Ok(Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn upsert_file(
|
||||
&self,
|
||||
origin_id: &OriginId,
|
||||
real_path: &Path,
|
||||
virtual_path: &VirtualPath,
|
||||
audio_meta: &AudioMeta,
|
||||
origin_mtime: SystemTime,
|
||||
origin_size: u64,
|
||||
) -> Result<FileId> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
|
||||
let mtime_secs = origin_mtime
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs() as i64;
|
||||
|
||||
conn.execute(
|
||||
r#"
|
||||
INSERT INTO files (
|
||||
origin_id, real_path, virtual_path,
|
||||
title, artist, album, album_artist, genre,
|
||||
year, track, disc,
|
||||
duration_ms, bitrate, sample_rate, format,
|
||||
origin_mtime, origin_size
|
||||
) VALUES (
|
||||
?1, ?2, ?3,
|
||||
?4, ?5, ?6, ?7, ?8,
|
||||
?9, ?10, ?11,
|
||||
?12, ?13, ?14, ?15,
|
||||
?16, ?17
|
||||
)
|
||||
ON CONFLICT(origin_id, real_path) DO UPDATE SET
|
||||
virtual_path = excluded.virtual_path,
|
||||
title = excluded.title,
|
||||
artist = excluded.artist,
|
||||
album = excluded.album,
|
||||
album_artist = excluded.album_artist,
|
||||
genre = excluded.genre,
|
||||
year = excluded.year,
|
||||
track = excluded.track,
|
||||
disc = excluded.disc,
|
||||
duration_ms = excluded.duration_ms,
|
||||
bitrate = excluded.bitrate,
|
||||
sample_rate = excluded.sample_rate,
|
||||
format = excluded.format,
|
||||
origin_mtime = excluded.origin_mtime,
|
||||
origin_size = excluded.origin_size,
|
||||
last_sync = strftime('%s', 'now')
|
||||
"#,
|
||||
params![
|
||||
&origin_id.0,
|
||||
real_path.to_string_lossy(),
|
||||
virtual_path.as_str(),
|
||||
&audio_meta.title,
|
||||
&audio_meta.artist,
|
||||
&audio_meta.album,
|
||||
&audio_meta.album_artist,
|
||||
&audio_meta.genre,
|
||||
&audio_meta.year,
|
||||
&audio_meta.track,
|
||||
&audio_meta.disc,
|
||||
&audio_meta.duration_ms.map(|d| d as i64),
|
||||
&audio_meta.bitrate,
|
||||
&audio_meta.sample_rate,
|
||||
format!("{:?}", audio_meta.format),
|
||||
mtime_secs,
|
||||
origin_size as i64,
|
||||
],
|
||||
)
|
||||
.map_err(|e| Error::Database(format!("upsert failed: {}", e)))?;
|
||||
|
||||
let id = conn.last_insert_rowid();
|
||||
debug!(id, vpath = virtual_path.as_str(), "Upserted file");
|
||||
|
||||
Ok(FileId(id))
|
||||
}
|
||||
|
||||
pub fn get_file_by_virtual_path(&self, path: &VirtualPath) -> Result<Option<FileMeta>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
|
||||
conn.query_row(
|
||||
r#"
|
||||
SELECT id, origin_id, real_path, virtual_path,
|
||||
title, artist, album, album_artist, genre,
|
||||
year, track, disc,
|
||||
duration_ms, bitrate, sample_rate, format,
|
||||
origin_mtime, origin_size, content_hash
|
||||
FROM files
|
||||
WHERE virtual_path = ?1
|
||||
"#,
|
||||
params![path.as_str()],
|
||||
|row| {
|
||||
let format_str: Option<String> = row.get(15)?;
|
||||
let format = format_str
|
||||
.as_deref()
|
||||
.map(parse_audio_format)
|
||||
.unwrap_or(AudioFormat::Unknown);
|
||||
|
||||
let content_hash: Option<String> = row.get(18)?;
|
||||
|
||||
Ok(FileMeta {
|
||||
id: FileId(row.get(0)?),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId(row.get(1)?),
|
||||
path: PathBuf::from(row.get::<_, String>(2)?),
|
||||
},
|
||||
virtual_path: VirtualPath::new(row.get::<_, String>(3)?),
|
||||
audio: Some(AudioMeta {
|
||||
title: row.get(4)?,
|
||||
artist: row.get(5)?,
|
||||
album: row.get(6)?,
|
||||
album_artist: row.get(7)?,
|
||||
genre: row.get(8)?,
|
||||
year: row.get(9)?,
|
||||
track: row.get(10)?,
|
||||
disc: row.get(11)?,
|
||||
duration_ms: row.get::<_, Option<i64>>(12)?.map(|d| d as u64),
|
||||
bitrate: row.get(13)?,
|
||||
sample_rate: row.get(14)?,
|
||||
format,
|
||||
}),
|
||||
size: row.get::<_, i64>(17)? as u64,
|
||||
mtime: UNIX_EPOCH + Duration::from_secs(row.get::<_, i64>(16)? as u64),
|
||||
content_hash: content_hash.and_then(|s| parse_content_hash(&s)),
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| Error::Database(format!("query failed: {}", e)))
|
||||
}
|
||||
|
||||
pub fn get_file_by_id(&self, id: FileId) -> Result<Option<FileMeta>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
|
||||
let vpath: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT virtual_path FROM files WHERE id = ?1",
|
||||
params![id.0],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| Error::Database(format!("query failed: {}", e)))?;
|
||||
|
||||
drop(conn);
|
||||
|
||||
match vpath {
|
||||
Some(p) => self.get_file_by_virtual_path(&VirtualPath::new(p)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn list_files_by_origin(&self, origin_id: &OriginId) -> Result<Vec<VirtualPath>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT virtual_path FROM files WHERE origin_id = ?1")
|
||||
.map_err(|e| Error::Database(format!("prepare failed: {}", e)))?;
|
||||
|
||||
let paths: Vec<VirtualPath> = stmt
|
||||
.query_map(params![&origin_id.0], |row| {
|
||||
Ok(VirtualPath::new(row.get::<_, String>(0)?))
|
||||
})
|
||||
.map_err(|e| Error::Database(format!("query failed: {}", e)))?
|
||||
.filter_map(|r| r.ok())
|
||||
.collect();
|
||||
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
pub fn delete_file(&self, id: FileId) -> Result<()> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute("DELETE FROM files WHERE id = ?1", params![id.0])
|
||||
.map_err(|e| Error::Database(format!("delete failed: {}", e)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn file_count(&self) -> Result<u64> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.query_row("SELECT COUNT(*) FROM files", [], |row| row.get::<_, i64>(0))
|
||||
.map(|c| c as u64)
|
||||
.map_err(|e| Error::Database(format!("count failed: {}", e)))
|
||||
}
|
||||
|
||||
pub fn update_content_hash(&self, id: FileId, hash: &ContentHash) -> Result<()> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"UPDATE files SET content_hash = ?1 WHERE id = ?2",
|
||||
params![hash.to_hex(), id.0],
|
||||
)
|
||||
.map_err(|e| Error::Database(format!("update hash failed: {}", e)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_mtime_by_real_path(
|
||||
&self,
|
||||
origin_id: &OriginId,
|
||||
real_path: &Path,
|
||||
) -> Result<Option<SystemTime>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
|
||||
conn.query_row(
|
||||
"SELECT origin_mtime FROM files WHERE origin_id = ?1 AND real_path = ?2",
|
||||
params![&origin_id.0, real_path.to_string_lossy()],
|
||||
|row| {
|
||||
let mtime_secs: i64 = row.get(0)?;
|
||||
Ok(UNIX_EPOCH + Duration::from_secs(mtime_secs as u64))
|
||||
},
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| Error::Database(format!("query mtime failed: {}", e)))
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_audio_format(s: &str) -> AudioFormat {
|
||||
match s {
|
||||
"Flac" => AudioFormat::Flac,
|
||||
"Mp3" => AudioFormat::Mp3,
|
||||
"Aac" => AudioFormat::Aac,
|
||||
"Opus" => AudioFormat::Opus,
|
||||
"Vorbis" => AudioFormat::Vorbis,
|
||||
"Wav" => AudioFormat::Wav,
|
||||
"Alac" => AudioFormat::Alac,
|
||||
_ => AudioFormat::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_content_hash(hex: &str) -> Option<ContentHash> {
|
||||
if hex.len() != 16 {
|
||||
return None;
|
||||
}
|
||||
let mut bytes = [0u8; 8];
|
||||
for (i, chunk) in hex.as_bytes().chunks(2).enumerate() {
|
||||
if i >= 8 {
|
||||
break;
|
||||
}
|
||||
let s = std::str::from_utf8(chunk).ok()?;
|
||||
bytes[i] = u8::from_str_radix(s, 16).ok()?;
|
||||
}
|
||||
Some(ContentHash(bytes))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_database_creation() {
|
||||
let db = Database::open_memory().unwrap();
|
||||
assert_eq!(db.file_count().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_upsert_and_retrieve() {
|
||||
let db = Database::open_memory().unwrap();
|
||||
|
||||
let origin_id = OriginId::from("local");
|
||||
let real_path = Path::new("/music/test.flac");
|
||||
let virtual_path = VirtualPath::new("/Artist/Album/01 - Track.flac");
|
||||
let audio_meta = AudioMeta {
|
||||
title: Some("Track".to_string()),
|
||||
artist: Some("Artist".to_string()),
|
||||
album: Some("Album".to_string()),
|
||||
track: Some(1),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let id = db
|
||||
.upsert_file(
|
||||
&origin_id,
|
||||
real_path,
|
||||
&virtual_path,
|
||||
&audio_meta,
|
||||
UNIX_EPOCH,
|
||||
1000,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let retrieved = db.get_file_by_virtual_path(&virtual_path).unwrap().unwrap();
|
||||
assert_eq!(retrieved.id, id);
|
||||
assert_eq!(
|
||||
retrieved.audio.as_ref().unwrap().title,
|
||||
Some("Track".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_upsert_updates_existing() {
|
||||
let db = Database::open_memory().unwrap();
|
||||
|
||||
let origin_id = OriginId::from("local");
|
||||
let real_path = Path::new("/music/test.flac");
|
||||
let virtual_path = VirtualPath::new("/Artist/Album/01 - Track.flac");
|
||||
|
||||
let meta1 = AudioMeta {
|
||||
title: Some("Original".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
db.upsert_file(
|
||||
&origin_id,
|
||||
real_path,
|
||||
&virtual_path,
|
||||
&meta1,
|
||||
UNIX_EPOCH,
|
||||
1000,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let meta2 = AudioMeta {
|
||||
title: Some("Updated".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
db.upsert_file(
|
||||
&origin_id,
|
||||
real_path,
|
||||
&virtual_path,
|
||||
&meta2,
|
||||
UNIX_EPOCH,
|
||||
1000,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(db.file_count().unwrap(), 1);
|
||||
|
||||
let retrieved = db.get_file_by_virtual_path(&virtual_path).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
retrieved.audio.as_ref().unwrap().title,
|
||||
Some("Updated".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_metadata_persistence() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db_path = dir.path().join("test.db");
|
||||
|
||||
{
|
||||
let db = Database::open(&db_path).unwrap();
|
||||
db.upsert_file(
|
||||
&OriginId::from("local"),
|
||||
Path::new("/test.flac"),
|
||||
&VirtualPath::new("/Test.flac"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let db = Database::open(&db_path).unwrap();
|
||||
assert_eq!(db.file_count().unwrap(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_file() {
|
||||
let db = Database::open_memory().unwrap();
|
||||
|
||||
let id = db
|
||||
.upsert_file(
|
||||
&OriginId::from("local"),
|
||||
Path::new("/test.flac"),
|
||||
&VirtualPath::new("/Test.flac"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(db.file_count().unwrap(), 1);
|
||||
db.delete_file(id).unwrap();
|
||||
assert_eq!(db.file_count().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_files_by_origin() {
|
||||
let db = Database::open_memory().unwrap();
|
||||
let origin = OriginId::from("local");
|
||||
|
||||
db.upsert_file(
|
||||
&origin,
|
||||
Path::new("/a.flac"),
|
||||
&VirtualPath::new("/A.flac"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
db.upsert_file(
|
||||
&origin,
|
||||
Path::new("/b.flac"),
|
||||
&VirtualPath::new("/B.flac"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let paths = db.list_files_by_origin(&origin).unwrap();
|
||||
assert_eq!(paths.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_content_hash_update() {
|
||||
let db = Database::open_memory().unwrap();
|
||||
|
||||
let id = db
|
||||
.upsert_file(
|
||||
&OriginId::from("local"),
|
||||
Path::new("/test.flac"),
|
||||
&VirtualPath::new("/Test.flac"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let hash = ContentHash::from_bytes(b"test data");
|
||||
db.update_content_hash(id, &hash).unwrap();
|
||||
|
||||
let retrieved = db
|
||||
.get_file_by_virtual_path(&VirtualPath::new("/Test.flac"))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(retrieved.content_hash.is_some());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
use musicfs_cas::CasStore;
|
||||
use musicfs_core::ChunkHash;
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Instant;
|
||||
use tracing::info;
|
||||
|
||||
pub trait EvictionPolicy: Send + Sync {
|
||||
fn record_access(&self, hash: ChunkHash);
|
||||
fn select_victims(&self, count: usize) -> Vec<ChunkHash>;
|
||||
fn remove(&self, hash: &ChunkHash);
|
||||
}
|
||||
|
||||
pub struct LruEviction {
|
||||
access_times: RwLock<BTreeMap<Instant, ChunkHash>>,
|
||||
hash_to_time: RwLock<std::collections::HashMap<ChunkHash, Instant>>,
|
||||
}
|
||||
|
||||
impl LruEviction {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
access_times: RwLock::new(BTreeMap::new()),
|
||||
hash_to_time: RwLock::new(std::collections::HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn evict_to_target(
|
||||
&self,
|
||||
store: &CasStore,
|
||||
target_size: u64,
|
||||
) -> Result<u64, EvictionError> {
|
||||
let mut bytes_freed = 0u64;
|
||||
|
||||
while store.current_size() > target_size {
|
||||
let victims = self.select_victims(10);
|
||||
|
||||
if victims.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
for hash in victims {
|
||||
if let Ok(data) = store.get(&hash).await {
|
||||
bytes_freed += data.len() as u64;
|
||||
store.delete(&hash).await?;
|
||||
self.remove(&hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if bytes_freed > 0 {
|
||||
info!("Evicted {} bytes from cache", bytes_freed);
|
||||
}
|
||||
|
||||
Ok(bytes_freed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LruEviction {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl EvictionPolicy for LruEviction {
|
||||
fn record_access(&self, hash: ChunkHash) {
|
||||
let now = Instant::now();
|
||||
let mut times = self.access_times.write();
|
||||
let mut h2t = self.hash_to_time.write();
|
||||
|
||||
if let Some(old_time) = h2t.remove(&hash) {
|
||||
times.remove(&old_time);
|
||||
}
|
||||
|
||||
times.insert(now, hash);
|
||||
h2t.insert(hash, now);
|
||||
}
|
||||
|
||||
fn select_victims(&self, count: usize) -> Vec<ChunkHash> {
|
||||
let times = self.access_times.read();
|
||||
times.values().take(count).copied().collect()
|
||||
}
|
||||
|
||||
fn remove(&self, hash: &ChunkHash) {
|
||||
let mut times = self.access_times.write();
|
||||
let mut h2t = self.hash_to_time.write();
|
||||
|
||||
if let Some(time) = h2t.remove(hash) {
|
||||
times.remove(&time);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum EvictionError {
|
||||
#[error("CAS error: {0}")]
|
||||
Cas(#[from] musicfs_cas::CasError),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_lru_access_order() {
|
||||
let lru = LruEviction::new();
|
||||
|
||||
let h1 = ChunkHash::from_bytes(b"chunk1");
|
||||
let h2 = ChunkHash::from_bytes(b"chunk2");
|
||||
let h3 = ChunkHash::from_bytes(b"chunk3");
|
||||
|
||||
lru.record_access(h1);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
lru.record_access(h2);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
lru.record_access(h3);
|
||||
|
||||
let victims = lru.select_victims(2);
|
||||
assert_eq!(victims.len(), 2);
|
||||
assert_eq!(victims[0], h1);
|
||||
assert_eq!(victims[1], h2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lru_reaccess_updates_order() {
|
||||
let lru = LruEviction::new();
|
||||
|
||||
let h1 = ChunkHash::from_bytes(b"chunk1");
|
||||
let h2 = ChunkHash::from_bytes(b"chunk2");
|
||||
|
||||
lru.record_access(h1);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
lru.record_access(h2);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
lru.record_access(h1);
|
||||
|
||||
let victims = lru.select_victims(1);
|
||||
assert_eq!(victims[0], h2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lru_remove() {
|
||||
let lru = LruEviction::new();
|
||||
|
||||
let h1 = ChunkHash::from_bytes(b"chunk1");
|
||||
let h2 = ChunkHash::from_bytes(b"chunk2");
|
||||
|
||||
lru.record_access(h1);
|
||||
lru.record_access(h2);
|
||||
lru.remove(&h1);
|
||||
|
||||
let victims = lru.select_victims(10);
|
||||
assert_eq!(victims.len(), 1);
|
||||
assert_eq!(victims[0], h2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
mod artwork;
|
||||
mod db;
|
||||
mod eviction;
|
||||
mod metadata;
|
||||
mod patterns;
|
||||
mod prefetch;
|
||||
mod tree;
|
||||
|
||||
pub use artwork::{ArtworkCache, ArtworkError, CachedArtwork};
|
||||
pub use db::Database;
|
||||
pub use eviction::{EvictionError, EvictionPolicy, LruEviction};
|
||||
pub use metadata::MetadataCache;
|
||||
pub use patterns::{AccessContext, AccessPattern, PatternError, PatternStore};
|
||||
pub use prefetch::{PrefetchConfig, PrefetchEngine, PrefetchHandle};
|
||||
pub use tree::{
|
||||
DirNode, FileNode, Inode, RefreshPolicy, TreeBuilder, VirtualNode, VirtualTree, ROOT_INODE,
|
||||
};
|
||||
@@ -0,0 +1,138 @@
|
||||
use crate::db::Database;
|
||||
use musicfs_core::{AudioMeta, FileMeta, OriginId, Result, VirtualPath};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tracing::trace;
|
||||
|
||||
pub struct MetadataCache {
|
||||
db: Arc<Database>,
|
||||
}
|
||||
|
||||
impl MetadataCache {
|
||||
pub fn new(db: Arc<Database>) -> Self {
|
||||
Self { db }
|
||||
}
|
||||
|
||||
pub fn store(
|
||||
&self,
|
||||
origin_id: &OriginId,
|
||||
real_path: &Path,
|
||||
virtual_path: &VirtualPath,
|
||||
audio_meta: &AudioMeta,
|
||||
origin_mtime: SystemTime,
|
||||
origin_size: u64,
|
||||
) -> Result<()> {
|
||||
self.db.upsert_file(
|
||||
origin_id,
|
||||
real_path,
|
||||
virtual_path,
|
||||
audio_meta,
|
||||
origin_mtime,
|
||||
origin_size,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn lookup(&self, path: &VirtualPath) -> Result<Option<FileMeta>> {
|
||||
let result = self.db.get_file_by_virtual_path(path)?;
|
||||
let hit = result.is_some();
|
||||
trace!(path = path.as_str(), hit, "metadata cache lookup");
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn is_fresh(
|
||||
&self,
|
||||
origin_id: &OriginId,
|
||||
real_path: &Path,
|
||||
current_mtime: SystemTime,
|
||||
) -> Result<bool> {
|
||||
if let Some(cached_mtime) = self.db.get_mtime_by_real_path(origin_id, real_path)? {
|
||||
let current_secs = current_mtime
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs();
|
||||
let cached_secs = cached_mtime
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs();
|
||||
let hit = current_secs == cached_secs;
|
||||
trace!(path = ?real_path, hit, "metadata freshness check");
|
||||
Ok(hit)
|
||||
} else {
|
||||
trace!(path = ?real_path, hit = false, "metadata freshness check");
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn invalidate(&self, path: &VirtualPath) -> Result<()> {
|
||||
if let Some(meta) = self.db.get_file_by_virtual_path(path)? {
|
||||
self.db.delete_file(meta.id)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_core::AudioFormat;
|
||||
|
||||
#[test]
|
||||
fn test_metadata_cache_store_and_lookup() {
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
let cache = MetadataCache::new(db);
|
||||
|
||||
let origin_id = OriginId::from("local");
|
||||
let real_path = Path::new("/music/song.flac");
|
||||
let virtual_path = VirtualPath::new("/Artist/Album/Song.flac");
|
||||
let meta = AudioMeta {
|
||||
title: Some("Song".to_string()),
|
||||
artist: Some("Artist".to_string()),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
cache
|
||||
.store(
|
||||
&origin_id,
|
||||
real_path,
|
||||
&virtual_path,
|
||||
&meta,
|
||||
UNIX_EPOCH,
|
||||
5000,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let retrieved = cache.lookup(&virtual_path).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
retrieved.audio.as_ref().unwrap().title,
|
||||
Some("Song".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_metadata_cache_invalidate() {
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
let cache = MetadataCache::new(db);
|
||||
|
||||
let virtual_path = VirtualPath::new("/Test.flac");
|
||||
|
||||
cache
|
||||
.store(
|
||||
&OriginId::from("local"),
|
||||
Path::new("/test.flac"),
|
||||
&virtual_path,
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(cache.lookup(&virtual_path).unwrap().is_some());
|
||||
|
||||
cache.invalidate(&virtual_path).unwrap();
|
||||
|
||||
assert!(cache.lookup(&virtual_path).unwrap().is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
use musicfs_core::FileId;
|
||||
use parking_lot::{Mutex, RwLock};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tracing::{debug, info, trace};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccessPattern {
|
||||
pub file_id: FileId,
|
||||
pub timestamp: SystemTime,
|
||||
pub context: AccessContext,
|
||||
pub hour_of_day: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AccessContext {
|
||||
pub album_id: Option<i64>,
|
||||
pub track_number: Option<u32>,
|
||||
pub artist: Option<String>,
|
||||
}
|
||||
|
||||
pub struct PatternStore {
|
||||
db: Mutex<rusqlite::Connection>,
|
||||
sequence_counts: RwLock<HashMap<(FileId, FileId), u32>>,
|
||||
time_patterns: RwLock<HashMap<u8, Vec<FileId>>>,
|
||||
max_history: usize,
|
||||
}
|
||||
|
||||
impl PatternStore {
|
||||
pub fn new(db_path: &Path, max_history: usize) -> Result<Self, PatternError> {
|
||||
let db = rusqlite::Connection::open(db_path)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS access_log (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL,
|
||||
access_time INTEGER NOT NULL,
|
||||
hour_of_day INTEGER NOT NULL
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_access_log_file ON access_log(file_id)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_access_log_time ON access_log(access_time)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS sequence_counts (
|
||||
from_file_id INTEGER NOT NULL,
|
||||
to_file_id INTEGER NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY (from_file_id, to_file_id)
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
let sequence_counts = {
|
||||
let mut map = HashMap::new();
|
||||
let mut stmt =
|
||||
db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
(FileId(row.get::<_, i64>(0)?), FileId(row.get::<_, i64>(1)?)),
|
||||
row.get::<_, u32>(2)?,
|
||||
))
|
||||
})?;
|
||||
for row in rows {
|
||||
let (key, count) = row?;
|
||||
map.insert(key, count);
|
||||
}
|
||||
map
|
||||
};
|
||||
|
||||
let store = Self {
|
||||
db: Mutex::new(db),
|
||||
sequence_counts: RwLock::new(sequence_counts),
|
||||
time_patterns: RwLock::new(HashMap::new()),
|
||||
max_history,
|
||||
};
|
||||
let sequence_count = store.sequence_counts.read().len();
|
||||
info!(path = ?db_path, sequence_count = sequence_count, max_history = max_history, "Pattern store opened");
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub fn record(&self, file_id: FileId, _context: AccessContext) -> Result<(), PatternError> {
|
||||
trace!(file_id = file_id.0, "Recording access pattern");
|
||||
let now = SystemTime::now();
|
||||
let timestamp = now.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64;
|
||||
let hour = (timestamp / 3600 % 24) as u8;
|
||||
|
||||
let db = self.db.lock();
|
||||
|
||||
db.execute(
|
||||
"INSERT INTO access_log (file_id, access_time, hour_of_day) VALUES (?1, ?2, ?3)",
|
||||
rusqlite::params![file_id.0, timestamp, hour],
|
||||
)?;
|
||||
|
||||
{
|
||||
let mut time_patterns = self.time_patterns.write();
|
||||
time_patterns.entry(hour).or_default().push(file_id);
|
||||
}
|
||||
|
||||
let prev_file_id: Option<i64> = db
|
||||
.query_row(
|
||||
"SELECT file_id FROM access_log WHERE id = (SELECT MAX(id) - 1 FROM access_log)",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.ok();
|
||||
|
||||
if let Some(prev_id) = prev_file_id {
|
||||
let prev = FileId(prev_id);
|
||||
|
||||
{
|
||||
let mut sequences = self.sequence_counts.write();
|
||||
*sequences.entry((prev, file_id)).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
db.execute(
|
||||
"INSERT INTO sequence_counts (from_file_id, to_file_id, count)
|
||||
VALUES (?1, ?2, 1)
|
||||
ON CONFLICT(from_file_id, to_file_id) DO UPDATE SET count = count + 1",
|
||||
rusqlite::params![prev_id, file_id.0],
|
||||
)?;
|
||||
}
|
||||
|
||||
let cutoff = timestamp - (self.max_history as i64 * 86400);
|
||||
db.execute("DELETE FROM access_log WHERE access_time < ?1", [cutoff])?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn predict_next(&self, current: FileId, limit: usize) -> Vec<FileId> {
|
||||
let sequences = self.sequence_counts.read();
|
||||
|
||||
let mut predictions: Vec<_> = sequences
|
||||
.iter()
|
||||
.filter(|((from, _), count)| *from == current && **count >= 2)
|
||||
.map(|((_, to), count)| (*to, *count))
|
||||
.collect();
|
||||
|
||||
predictions.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
let result: Vec<FileId> = predictions
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|(id, _)| id)
|
||||
.collect();
|
||||
debug!(
|
||||
file_id = current.0,
|
||||
predictions = result.len(),
|
||||
"Predicted next files"
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
pub fn predict_for_time(&self, hour: u8, limit: usize) -> Vec<FileId> {
|
||||
let time_patterns = self.time_patterns.read();
|
||||
|
||||
time_patterns
|
||||
.get(&hour)
|
||||
.map(|files| files.iter().rev().take(limit).copied().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn recently_played(&self, days: u32) -> Result<Vec<FileId>, PatternError> {
|
||||
let cutoff = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs() as i64
|
||||
- (days as i64 * 86400);
|
||||
|
||||
let db = self.db.lock();
|
||||
let mut stmt = db.prepare(
|
||||
"SELECT DISTINCT file_id FROM access_log WHERE access_time >= ?1 ORDER BY access_time DESC",
|
||||
)?;
|
||||
|
||||
let files: Vec<FileId> = stmt
|
||||
.query_map([cutoff], |row| Ok(FileId(row.get(0)?)))?
|
||||
.filter_map(|r| r.ok())
|
||||
.collect();
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
pub fn most_played(&self, limit: u32) -> Result<Vec<FileId>, PatternError> {
|
||||
let db = self.db.lock();
|
||||
let mut stmt = db.prepare(
|
||||
"SELECT file_id, COUNT(*) as play_count FROM access_log
|
||||
GROUP BY file_id ORDER BY play_count DESC LIMIT ?1",
|
||||
)?;
|
||||
|
||||
let files: Vec<FileId> = stmt
|
||||
.query_map([limit], |row| Ok(FileId(row.get(0)?)))?
|
||||
.filter_map(|r| r.ok())
|
||||
.collect();
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum PatternError {
|
||||
#[error("database error: {0}")]
|
||||
Database(#[from] rusqlite::Error),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_pattern_prediction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("patterns.db");
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
let ctx = AccessContext::default();
|
||||
|
||||
for _ in 0..5 {
|
||||
store.record(FileId(1), ctx.clone()).unwrap();
|
||||
store.record(FileId(2), ctx.clone()).unwrap();
|
||||
store.record(FileId(3), ctx.clone()).unwrap();
|
||||
}
|
||||
|
||||
let predictions = store.predict_next(FileId(1), 3);
|
||||
assert!(!predictions.is_empty());
|
||||
assert_eq!(predictions[0], FileId(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pattern_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("patterns.db");
|
||||
let ctx = AccessContext::default();
|
||||
|
||||
{
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
for _ in 0..3 {
|
||||
store.record(FileId(1), ctx.clone()).unwrap();
|
||||
store.record(FileId(2), ctx.clone()).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
let predictions = store.predict_next(FileId(1), 3);
|
||||
assert!(!predictions.is_empty());
|
||||
assert_eq!(predictions[0], FileId(2));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_recently_played() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("patterns.db");
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
let ctx = AccessContext::default();
|
||||
|
||||
store.record(FileId(100), ctx.clone()).unwrap();
|
||||
store.record(FileId(200), ctx.clone()).unwrap();
|
||||
|
||||
let recent = store.recently_played(7).unwrap();
|
||||
assert!(recent.contains(&FileId(100)));
|
||||
assert!(recent.contains(&FileId(200)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_most_played() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("patterns.db");
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
let ctx = AccessContext::default();
|
||||
|
||||
for _ in 0..5 {
|
||||
store.record(FileId(1), ctx.clone()).unwrap();
|
||||
}
|
||||
for _ in 0..2 {
|
||||
store.record(FileId(2), ctx.clone()).unwrap();
|
||||
}
|
||||
|
||||
let most = store.most_played(10).unwrap();
|
||||
assert_eq!(most[0], FileId(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
use crate::patterns::{AccessContext, PatternStore};
|
||||
use musicfs_cas::ContentFetcher;
|
||||
use musicfs_core::{Event, EventBus, FileId};
|
||||
use parking_lot::Mutex as ParkingMutex;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const DEFAULT_PREFETCH_LOOKAHEAD: usize = 3;
|
||||
const DEFAULT_MAX_CONCURRENT: usize = 2;
|
||||
const DEFAULT_COOLDOWN_MS: u64 = 100;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PrefetchConfig {
|
||||
pub lookahead: usize,
|
||||
pub max_concurrent: usize,
|
||||
pub cooldown: Duration,
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for PrefetchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
lookahead: DEFAULT_PREFETCH_LOOKAHEAD,
|
||||
max_concurrent: DEFAULT_MAX_CONCURRENT,
|
||||
cooldown: Duration::from_millis(DEFAULT_COOLDOWN_MS),
|
||||
enabled: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PrefetchEngine {
|
||||
config: PrefetchConfig,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
in_flight: Arc<ParkingMutex<HashSet<FileId>>>,
|
||||
semaphore: Arc<Semaphore>,
|
||||
running: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
pub struct PrefetchHandle {
|
||||
handle: JoinHandle<()>,
|
||||
running: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl PrefetchHandle {
|
||||
pub async fn stop(self) {
|
||||
self.running.store(false, Ordering::SeqCst);
|
||||
let _ = self.handle.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl PrefetchEngine {
|
||||
pub fn new(
|
||||
config: PrefetchConfig,
|
||||
_pattern_store: Arc<PatternStore>,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
) -> Self {
|
||||
let semaphore = Arc::new(Semaphore::new(config.max_concurrent));
|
||||
|
||||
Self {
|
||||
config,
|
||||
fetcher,
|
||||
in_flight: Arc::new(ParkingMutex::new(HashSet::new())),
|
||||
semaphore,
|
||||
running: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(
|
||||
self: Arc<Self>,
|
||||
event_bus: Arc<EventBus>,
|
||||
pattern_store: Arc<PatternStore>,
|
||||
) -> PrefetchHandle {
|
||||
self.running.store(true, Ordering::SeqCst);
|
||||
let running = self.running.clone();
|
||||
|
||||
let config = self.config.clone();
|
||||
let fetcher = self.fetcher.clone();
|
||||
let in_flight = self.in_flight.clone();
|
||||
let semaphore = self.semaphore.clone();
|
||||
let running_inner = running.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut rx = event_bus.subscribe();
|
||||
|
||||
while running_inner.load(Ordering::SeqCst) {
|
||||
match tokio::time::timeout(Duration::from_secs(1), rx.recv()).await {
|
||||
Ok(Ok(event)) => {
|
||||
if let Event::FileAccessed { file_id, .. } = event {
|
||||
if config.enabled {
|
||||
let ctx = AccessContext::default();
|
||||
if let Err(e) = pattern_store.record(file_id, ctx) {
|
||||
warn!("Failed to record access pattern: {}", e);
|
||||
continue;
|
||||
}
|
||||
|
||||
let predictions =
|
||||
pattern_store.predict_next(file_id, config.lookahead);
|
||||
|
||||
for predicted_id in predictions {
|
||||
prefetch_file(predicted_id, &fetcher, &in_flight, &semaphore)
|
||||
.await;
|
||||
}
|
||||
|
||||
tokio::time::sleep(config.cooldown).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(_)) => break,
|
||||
Err(_) => continue,
|
||||
}
|
||||
}
|
||||
|
||||
info!("Prefetch engine stopped");
|
||||
});
|
||||
|
||||
PrefetchHandle { handle, running }
|
||||
}
|
||||
|
||||
pub fn is_running(&self) -> bool {
|
||||
self.running.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn in_flight_count(&self) -> usize {
|
||||
self.in_flight.lock().len()
|
||||
}
|
||||
|
||||
pub fn update_config(&mut self, config: PrefetchConfig) {
|
||||
self.config = config;
|
||||
}
|
||||
}
|
||||
|
||||
async fn prefetch_file(
|
||||
file_id: FileId,
|
||||
fetcher: &Arc<ContentFetcher>,
|
||||
in_flight: &Arc<ParkingMutex<HashSet<FileId>>>,
|
||||
semaphore: &Arc<Semaphore>,
|
||||
) {
|
||||
{
|
||||
let mut guard = in_flight.lock();
|
||||
if guard.contains(&file_id) {
|
||||
debug!("Skipping prefetch for {:?} - already in flight", file_id);
|
||||
return;
|
||||
}
|
||||
guard.insert(file_id);
|
||||
}
|
||||
|
||||
let permit = match semaphore.clone().try_acquire_owned() {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
debug!("Skipping prefetch for {:?} - concurrency limit", file_id);
|
||||
in_flight.lock().remove(&file_id);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let fetcher = fetcher.clone();
|
||||
let in_flight = in_flight.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
debug!("Prefetching file {:?}", file_id);
|
||||
|
||||
match fetcher.ensure_cached(file_id).await {
|
||||
Ok(manifest) => {
|
||||
info!(
|
||||
"Prefetched {:?}: {} chunks, {} bytes",
|
||||
file_id,
|
||||
manifest.chunks.len(),
|
||||
manifest.total_size
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Prefetch failed for {:?}: {}", file_id, e);
|
||||
}
|
||||
}
|
||||
|
||||
in_flight.lock().remove(&file_id);
|
||||
drop(permit);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_prefetch_config_defaults() {
|
||||
let config = PrefetchConfig::default();
|
||||
assert_eq!(config.lookahead, 3);
|
||||
assert_eq!(config.max_concurrent, 2);
|
||||
assert!(config.enabled);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA foreign_keys = ON;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
id INTEGER PRIMARY KEY,
|
||||
origin_id TEXT NOT NULL,
|
||||
real_path TEXT NOT NULL,
|
||||
virtual_path TEXT NOT NULL,
|
||||
|
||||
title TEXT,
|
||||
artist TEXT,
|
||||
album TEXT,
|
||||
album_artist TEXT,
|
||||
genre TEXT,
|
||||
year INTEGER,
|
||||
track INTEGER,
|
||||
disc INTEGER,
|
||||
duration_ms INTEGER,
|
||||
bitrate INTEGER,
|
||||
sample_rate INTEGER,
|
||||
format TEXT,
|
||||
|
||||
origin_mtime INTEGER NOT NULL,
|
||||
origin_size INTEGER NOT NULL,
|
||||
content_hash TEXT,
|
||||
chunk_manifest BLOB,
|
||||
last_sync INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
|
||||
|
||||
UNIQUE(origin_id, real_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artwork (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
||||
art_type TEXT NOT NULL,
|
||||
chunk_hash TEXT NOT NULL,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
mime_type TEXT,
|
||||
UNIQUE(file_id, art_type)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS collections (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
query_json TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
|
||||
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_files_virtual ON files(virtual_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_artist_album ON files(artist, album);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_content_hash ON files(content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_real ON files(origin_id, real_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_origin ON files(origin_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_last_sync ON files(last_sync);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_file ON artwork(file_id);
|
||||
@@ -0,0 +1,448 @@
|
||||
use musicfs_core::{FileId, FileMeta, VirtualPath};
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::{debug, trace};
|
||||
|
||||
pub type Inode = u64;
|
||||
pub const ROOT_INODE: Inode = 1;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum VirtualNode {
|
||||
Directory(DirNode),
|
||||
File(FileNode),
|
||||
}
|
||||
|
||||
impl VirtualNode {
|
||||
pub fn inode(&self) -> Inode {
|
||||
match self {
|
||||
VirtualNode::Directory(d) => d.inode,
|
||||
VirtualNode::File(f) => f.inode,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &OsStr {
|
||||
match self {
|
||||
VirtualNode::Directory(d) => &d.name,
|
||||
VirtualNode::File(f) => &f.name,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_dir(&self) -> bool {
|
||||
matches!(self, VirtualNode::Directory(_))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DirNode {
|
||||
pub inode: Inode,
|
||||
pub parent: Inode,
|
||||
pub name: OsString,
|
||||
pub children: BTreeMap<OsString, Inode>,
|
||||
pub mtime: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct FileNode {
|
||||
pub inode: Inode,
|
||||
pub name: OsString,
|
||||
pub file_id: FileId,
|
||||
pub size: u64,
|
||||
pub mtime: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RefreshPolicy {
|
||||
pub ttl: Duration,
|
||||
pub refresh_on_access: bool,
|
||||
pub background_interval: Option<Duration>,
|
||||
}
|
||||
|
||||
impl Default for RefreshPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
ttl: Duration::from_secs(300),
|
||||
refresh_on_access: false,
|
||||
background_interval: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct VirtualTree {
|
||||
nodes: HashMap<Inode, VirtualNode>,
|
||||
path_to_inode: HashMap<VirtualPath, Inode>,
|
||||
next_inode: AtomicU64,
|
||||
last_refresh: RwLock<SystemTime>,
|
||||
refresh_policy: RefreshPolicy,
|
||||
}
|
||||
|
||||
impl VirtualTree {
|
||||
pub fn new() -> Self {
|
||||
Self::with_policy(RefreshPolicy::default())
|
||||
}
|
||||
|
||||
pub fn with_policy(policy: RefreshPolicy) -> Self {
|
||||
let mut tree = Self {
|
||||
nodes: HashMap::new(),
|
||||
path_to_inode: HashMap::new(),
|
||||
next_inode: AtomicU64::new(ROOT_INODE + 1),
|
||||
last_refresh: RwLock::new(SystemTime::now()),
|
||||
refresh_policy: policy,
|
||||
};
|
||||
|
||||
tree.nodes.insert(
|
||||
ROOT_INODE,
|
||||
VirtualNode::Directory(DirNode {
|
||||
inode: ROOT_INODE,
|
||||
parent: ROOT_INODE,
|
||||
name: OsString::from(""),
|
||||
children: BTreeMap::new(),
|
||||
mtime: SystemTime::now(),
|
||||
}),
|
||||
);
|
||||
tree.path_to_inode.insert(VirtualPath::new("/"), ROOT_INODE);
|
||||
|
||||
tree
|
||||
}
|
||||
|
||||
fn alloc_inode(&self) -> Inode {
|
||||
self.next_inode.fetch_add(1, Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn get(&self, inode: Inode) -> Option<&VirtualNode> {
|
||||
self.nodes.get(&inode)
|
||||
}
|
||||
|
||||
pub fn get_by_path(&self, path: &VirtualPath) -> Option<&VirtualNode> {
|
||||
self.path_to_inode
|
||||
.get(path)
|
||||
.and_then(|ino| self.nodes.get(ino))
|
||||
}
|
||||
|
||||
pub fn lookup(&self, parent_inode: Inode, name: &OsStr) -> Option<Inode> {
|
||||
if let Some(VirtualNode::Directory(dir)) = self.nodes.get(&parent_inode) {
|
||||
let result = dir.children.get(name).copied();
|
||||
let hit = result.is_some();
|
||||
trace!(inode = parent_inode, name = ?name, hit, "tree lookup");
|
||||
result
|
||||
} else {
|
||||
trace!(inode = parent_inode, name = ?name, hit = false, "tree lookup");
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn readdir(&self, inode: Inode) -> Option<Vec<(OsString, Inode, bool)>> {
|
||||
if let Some(VirtualNode::Directory(dir)) = self.nodes.get(&inode) {
|
||||
Some(
|
||||
dir.children
|
||||
.iter()
|
||||
.map(|(name, &ino)| {
|
||||
let is_dir = self.nodes.get(&ino).map(|n| n.is_dir()).unwrap_or(false);
|
||||
(name.clone(), ino, is_dir)
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_parent(&self, inode: Inode) -> Option<Inode> {
|
||||
match self.nodes.get(&inode) {
|
||||
Some(VirtualNode::Directory(dir)) => Some(dir.parent),
|
||||
Some(VirtualNode::File(_)) => self.find_parent_by_path_lookup(inode),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn find_parent_by_path_lookup(&self, inode: Inode) -> Option<Inode> {
|
||||
for (path, &ino) in &self.path_to_inode {
|
||||
if ino == inode {
|
||||
return std::path::Path::new(path.as_str()).parent().and_then(|p| {
|
||||
self.path_to_inode
|
||||
.get(&VirtualPath::new(p.to_string_lossy().into_owned()))
|
||||
.copied()
|
||||
});
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn insert_file(&mut self, meta: &FileMeta) -> Inode {
|
||||
let path = &meta.virtual_path;
|
||||
|
||||
let parent_inode = self.ensure_parents(path);
|
||||
|
||||
let inode = self.alloc_inode();
|
||||
let name = std::path::Path::new(path.as_str())
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_os_string();
|
||||
|
||||
let file_node = FileNode {
|
||||
inode,
|
||||
name: name.clone(),
|
||||
file_id: meta.id,
|
||||
size: meta.size,
|
||||
mtime: meta.mtime,
|
||||
};
|
||||
|
||||
self.nodes.insert(inode, VirtualNode::File(file_node));
|
||||
self.path_to_inode.insert(path.clone(), inode);
|
||||
|
||||
if let Some(VirtualNode::Directory(dir)) = self.nodes.get_mut(&parent_inode) {
|
||||
dir.children.insert(name, inode);
|
||||
}
|
||||
|
||||
debug!(inode, path = path.as_str(), file_id = ?meta.id, "add file to tree");
|
||||
inode
|
||||
}
|
||||
|
||||
fn ensure_parents(&mut self, path: &VirtualPath) -> Inode {
|
||||
let path_str = path.as_str();
|
||||
let components: Vec<&str> = path_str
|
||||
.trim_start_matches('/')
|
||||
.split('/')
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect();
|
||||
|
||||
if components.len() <= 1 {
|
||||
return ROOT_INODE;
|
||||
}
|
||||
|
||||
let mut current_inode = ROOT_INODE;
|
||||
let mut current_path = String::from("/");
|
||||
|
||||
for component in &components[..components.len() - 1] {
|
||||
current_path.push_str(component);
|
||||
|
||||
let vpath = VirtualPath::new(¤t_path);
|
||||
|
||||
if let Some(&existing) = self.path_to_inode.get(&vpath) {
|
||||
current_inode = existing;
|
||||
} else {
|
||||
let new_inode = self.alloc_inode();
|
||||
let name = OsString::from(*component);
|
||||
|
||||
let dir_node = DirNode {
|
||||
inode: new_inode,
|
||||
parent: current_inode,
|
||||
name: name.clone(),
|
||||
children: BTreeMap::new(),
|
||||
mtime: SystemTime::now(),
|
||||
};
|
||||
|
||||
self.nodes
|
||||
.insert(new_inode, VirtualNode::Directory(dir_node));
|
||||
self.path_to_inode.insert(vpath, new_inode);
|
||||
|
||||
if let Some(VirtualNode::Directory(parent)) = self.nodes.get_mut(¤t_inode) {
|
||||
parent.children.insert(name, new_inode);
|
||||
}
|
||||
|
||||
current_inode = new_inode;
|
||||
}
|
||||
|
||||
current_path.push('/');
|
||||
}
|
||||
|
||||
current_inode
|
||||
}
|
||||
|
||||
pub fn remove_file(&mut self, path: &VirtualPath) -> Option<FileId> {
|
||||
let inode = self.path_to_inode.remove(path)?;
|
||||
|
||||
if let Some(VirtualNode::File(file)) = self.nodes.remove(&inode) {
|
||||
let parent_path = std::path::Path::new(path.as_str())
|
||||
.parent()
|
||||
.map(|p| VirtualPath::new(p.to_string_lossy().into_owned()))
|
||||
.unwrap_or_else(|| VirtualPath::new("/"));
|
||||
|
||||
if let Some(&parent_inode) = self.path_to_inode.get(&parent_path) {
|
||||
if let Some(VirtualNode::Directory(dir)) = self.nodes.get_mut(&parent_inode) {
|
||||
dir.children.remove(&file.name);
|
||||
}
|
||||
}
|
||||
|
||||
debug!(inode, path = path.as_str(), file_id = ?file.file_id, "remove file from tree");
|
||||
Some(file.file_id)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn file_count(&self) -> usize {
|
||||
self.nodes
|
||||
.values()
|
||||
.filter(|n| matches!(n, VirtualNode::File(_)))
|
||||
.count()
|
||||
}
|
||||
|
||||
pub fn dir_count(&self) -> usize {
|
||||
self.nodes
|
||||
.values()
|
||||
.filter(|n| matches!(n, VirtualNode::Directory(_)))
|
||||
.count()
|
||||
}
|
||||
|
||||
pub fn needs_refresh(&self) -> bool {
|
||||
let last = *self.last_refresh.read();
|
||||
last.elapsed().unwrap_or(Duration::MAX) > self.refresh_policy.ttl
|
||||
}
|
||||
|
||||
pub fn force_refresh(&mut self) {
|
||||
self.nodes.retain(|&ino, _| ino == ROOT_INODE);
|
||||
self.path_to_inode.retain(|p, _| p.as_str() == "/");
|
||||
|
||||
if let Some(VirtualNode::Directory(root)) = self.nodes.get_mut(&ROOT_INODE) {
|
||||
root.children.clear();
|
||||
}
|
||||
|
||||
*self.last_refresh.write() = SystemTime::now();
|
||||
}
|
||||
|
||||
pub fn mark_refreshed(&self) {
|
||||
*self.last_refresh.write() = SystemTime::now();
|
||||
}
|
||||
|
||||
pub fn refresh_policy(&self) -> &RefreshPolicy {
|
||||
&self.refresh_policy
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for VirtualTree {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TreeBuilder {
|
||||
tree: VirtualTree,
|
||||
}
|
||||
|
||||
impl TreeBuilder {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tree: VirtualTree::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_policy(policy: RefreshPolicy) -> Self {
|
||||
Self {
|
||||
tree: VirtualTree::with_policy(policy),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_file(&mut self, meta: &FileMeta) {
|
||||
self.tree.insert_file(meta);
|
||||
}
|
||||
|
||||
pub fn build(self) -> VirtualTree {
|
||||
self.tree
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TreeBuilder {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_core::{OriginId, RealPath};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn make_file_meta(id: i64, vpath: &str) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from("/test"),
|
||||
},
|
||||
size: 1000,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tree_creation() {
|
||||
let tree = VirtualTree::new();
|
||||
assert!(tree.get(ROOT_INODE).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_insert_file() {
|
||||
let mut tree = VirtualTree::new();
|
||||
let meta = make_file_meta(1, "/Artist/Album/Track.flac");
|
||||
tree.insert_file(&meta);
|
||||
|
||||
assert!(tree.get_by_path(&VirtualPath::new("/Artist")).is_some());
|
||||
assert!(tree
|
||||
.get_by_path(&VirtualPath::new("/Artist/Album"))
|
||||
.is_some());
|
||||
assert!(tree
|
||||
.get_by_path(&VirtualPath::new("/Artist/Album/Track.flac"))
|
||||
.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_readdir() {
|
||||
let mut tree = VirtualTree::new();
|
||||
tree.insert_file(&make_file_meta(1, "/Artist/Album/Track1.flac"));
|
||||
tree.insert_file(&make_file_meta(2, "/Artist/Album/Track2.flac"));
|
||||
|
||||
let root_children = tree.readdir(ROOT_INODE).unwrap();
|
||||
assert_eq!(root_children.len(), 1);
|
||||
assert_eq!(root_children[0].0, "Artist");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lookup() {
|
||||
let mut tree = VirtualTree::new();
|
||||
tree.insert_file(&make_file_meta(1, "/Artist/Album/Track.flac"));
|
||||
|
||||
let artist_inode = tree.lookup(ROOT_INODE, OsStr::new("Artist")).unwrap();
|
||||
assert!(tree.lookup(artist_inode, OsStr::new("Album")).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_and_dir_count() {
|
||||
let mut tree = VirtualTree::new();
|
||||
tree.insert_file(&make_file_meta(1, "/A/B/Track1.flac"));
|
||||
tree.insert_file(&make_file_meta(2, "/A/B/Track2.flac"));
|
||||
tree.insert_file(&make_file_meta(3, "/A/C/Track3.flac"));
|
||||
|
||||
assert_eq!(tree.file_count(), 3);
|
||||
assert_eq!(tree.dir_count(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_file() {
|
||||
let mut tree = VirtualTree::new();
|
||||
let path = VirtualPath::new("/Artist/Album/Track.flac");
|
||||
tree.insert_file(&make_file_meta(1, path.as_str()));
|
||||
|
||||
assert!(tree.get_by_path(&path).is_some());
|
||||
|
||||
let removed_id = tree.remove_file(&path);
|
||||
assert_eq!(removed_id, Some(FileId(1)));
|
||||
assert!(tree.get_by_path(&path).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tree_builder() {
|
||||
let mut builder = TreeBuilder::new();
|
||||
builder.add_file(&make_file_meta(1, "/A/Track1.flac"));
|
||||
builder.add_file(&make_file_meta(2, "/A/Track2.flac"));
|
||||
|
||||
let tree = builder.build();
|
||||
assert_eq!(tree.file_count(), 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "musicfs-cas"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
failpoints = ["fail/failpoints"]
|
||||
|
||||
[dependencies]
|
||||
fail = { workspace = true, optional = true }
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-origins = { path = "../musicfs-origins" }
|
||||
musicfs-sync = { path = "../musicfs-sync" }
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
serde.workspace = true
|
||||
sled.workspace = true
|
||||
xxhash-rust.workspace = true
|
||||
bytes.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
hex.workspace = true
|
||||
dirs.workspace = true
|
||||
thiserror.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
musicfs-cache = { path = "../musicfs-cache" }
|
||||
@@ -0,0 +1,45 @@
|
||||
use musicfs_core::ChunkHash;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkLocation {
|
||||
pub path: PathBuf,
|
||||
pub size: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkRef {
|
||||
pub hash: ChunkHash,
|
||||
pub offset: u64,
|
||||
pub size: u32,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_chunk_hash_from_bytes() {
|
||||
let data = b"hello world";
|
||||
let hash = ChunkHash::from_bytes(data);
|
||||
assert_eq!(hash.as_hex().len(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunk_hash_deterministic() {
|
||||
let data = b"test data";
|
||||
let hash1 = ChunkHash::from_bytes(data);
|
||||
let hash2 = ChunkHash::from_bytes(data);
|
||||
assert_eq!(hash1, hash2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunk_hash_hex_roundtrip() {
|
||||
let data = b"roundtrip test";
|
||||
let hash = ChunkHash::from_bytes(data);
|
||||
let hex = hash.as_hex();
|
||||
let restored = ChunkHash::from_hex(&hex).unwrap();
|
||||
assert_eq!(hash, restored);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
use crate::{CasStore, ChunkManifest, ChunkRef};
|
||||
use musicfs_core::{Event, EventBus, FileId, FileMeta, OriginId};
|
||||
use musicfs_origins::Origin;
|
||||
use musicfs_sync::CdcChunker;
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
pub struct ContentFetcher {
|
||||
store: Arc<CasStore>,
|
||||
origins: RwLock<HashMap<OriginId, Arc<dyn Origin>>>,
|
||||
file_meta: RwLock<HashMap<FileId, FileMeta>>,
|
||||
event_bus: Option<Arc<EventBus>>,
|
||||
chunker: CdcChunker,
|
||||
}
|
||||
|
||||
impl ContentFetcher {
|
||||
pub fn new(store: Arc<CasStore>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
origins: RwLock::new(HashMap::new()),
|
||||
file_meta: RwLock::new(HashMap::new()),
|
||||
event_bus: None,
|
||||
chunker: CdcChunker::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_event_bus(store: Arc<CasStore>, event_bus: Arc<EventBus>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
origins: RwLock::new(HashMap::new()),
|
||||
file_meta: RwLock::new(HashMap::new()),
|
||||
event_bus: Some(event_bus),
|
||||
chunker: CdcChunker::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_origin(&self, origin: Arc<dyn Origin>) {
|
||||
let id = origin.id().clone();
|
||||
self.origins.write().insert(id, origin);
|
||||
}
|
||||
|
||||
pub fn register_file(&self, meta: FileMeta) {
|
||||
self.file_meta.write().insert(meta.id, meta);
|
||||
}
|
||||
|
||||
pub fn register_files(&self, files: impl IntoIterator<Item = FileMeta>) {
|
||||
let mut map = self.file_meta.write();
|
||||
for meta in files {
|
||||
map.insert(meta.id, meta);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn fetch_file(&self, file_id: FileId) -> Result<ChunkManifest, FetchError> {
|
||||
let meta = {
|
||||
let files = self.file_meta.read();
|
||||
files
|
||||
.get(&file_id)
|
||||
.cloned()
|
||||
.ok_or(FetchError::FileNotFound(file_id))?
|
||||
};
|
||||
|
||||
let origin = {
|
||||
let origins = self.origins.read();
|
||||
origins
|
||||
.get(&meta.real_path.origin_id)
|
||||
.cloned()
|
||||
.ok_or_else(|| FetchError::OriginNotFound(meta.real_path.origin_id.clone()))?
|
||||
};
|
||||
|
||||
info!("Fetching file {:?} from origin {}", file_id, origin.id());
|
||||
|
||||
let data = origin
|
||||
.read_full(&meta.real_path.path)
|
||||
.await
|
||||
.map_err(|e| FetchError::OriginRead(e.to_string()))?;
|
||||
|
||||
let mtime = meta
|
||||
.mtime
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0);
|
||||
|
||||
let chunks = self.chunker.chunk_refs(&data);
|
||||
info!("Chunked {:?} into {} chunks", file_id, chunks.len());
|
||||
|
||||
let mut chunk_refs = Vec::with_capacity(chunks.len());
|
||||
for chunk in chunks {
|
||||
if !self.store.exists(&chunk.hash) {
|
||||
if let Err(e) = self.store.put(chunk.data).await {
|
||||
warn!(hash = %chunk.hash, error = %e, "CAS write failed, continuing in passthrough mode");
|
||||
}
|
||||
}
|
||||
|
||||
chunk_refs.push(ChunkRef {
|
||||
hash: chunk.hash,
|
||||
offset: chunk.offset,
|
||||
size: chunk.length,
|
||||
});
|
||||
}
|
||||
|
||||
let manifest = ChunkManifest {
|
||||
file_id,
|
||||
total_size: meta.size,
|
||||
mtime,
|
||||
chunks: chunk_refs,
|
||||
};
|
||||
|
||||
debug!(
|
||||
"Created manifest for {:?}: {} bytes, {} chunks",
|
||||
file_id,
|
||||
meta.size,
|
||||
manifest.chunks.len()
|
||||
);
|
||||
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
pub async fn ensure_cached(&self, file_id: FileId) -> Result<ChunkManifest, FetchError> {
|
||||
self.fetch_file(file_id).await
|
||||
}
|
||||
|
||||
pub fn get_file_meta(&self, file_id: FileId) -> Option<FileMeta> {
|
||||
self.file_meta.read().get(&file_id).cloned()
|
||||
}
|
||||
|
||||
pub fn emit_access_event(&self, meta: &FileMeta, offset: u64, size: u32) {
|
||||
if let Some(bus) = &self.event_bus {
|
||||
bus.publish(Event::FileAccessed {
|
||||
file_id: meta.id,
|
||||
path: meta.virtual_path.clone(),
|
||||
origin_id: meta.real_path.origin_id.clone(),
|
||||
offset,
|
||||
size,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FetchError {
|
||||
#[error("File not found: {0:?}")]
|
||||
FileNotFound(FileId),
|
||||
|
||||
#[error("Origin not found: {0}")]
|
||||
OriginNotFound(OriginId),
|
||||
|
||||
#[error("Origin read error: {0}")]
|
||||
OriginRead(String),
|
||||
|
||||
#[error("Store error: {0}")]
|
||||
Store(#[from] crate::CasError),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::CasConfig;
|
||||
use musicfs_core::{RealPath, VirtualPath};
|
||||
use musicfs_origins::LocalOrigin;
|
||||
use std::path::PathBuf;
|
||||
use std::time::SystemTime;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_file() {
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
|
||||
std::fs::write(origin_dir.path().join("test.flac"), b"fake audio data").unwrap();
|
||||
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
let fetcher = ContentFetcher::new(store.clone());
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new("local", origin_dir.path()));
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/Artist/Album/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("local"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: 15,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(meta);
|
||||
|
||||
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
|
||||
assert_eq!(manifest.total_size, 15);
|
||||
assert_eq!(manifest.chunks.len(), 1);
|
||||
|
||||
let data = store.get(&manifest.chunks[0].hash).await.unwrap();
|
||||
assert_eq!(&data[..], b"fake audio data");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_file_not_found() {
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
let fetcher = ContentFetcher::new(store);
|
||||
|
||||
let result = fetcher.fetch_file(FileId(999)).await;
|
||||
assert!(matches!(result, Err(FetchError::FileNotFound(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_emits_event() {
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
std::fs::write(origin_dir.path().join("test.flac"), b"audio").unwrap();
|
||||
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
let event_bus = Arc::new(EventBus::default());
|
||||
let mut rx = event_bus.subscribe();
|
||||
|
||||
let fetcher = ContentFetcher::with_event_bus(store, event_bus);
|
||||
let origin = Arc::new(LocalOrigin::new("local", origin_dir.path()));
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/Artist/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("local"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: 5,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(meta.clone());
|
||||
|
||||
fetcher.emit_access_event(&meta, 0, 5);
|
||||
|
||||
let event = rx.try_recv().unwrap();
|
||||
assert!(matches!(event, Event::FileAccessed { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_origin_not_found() {
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
let fetcher = ContentFetcher::new(store);
|
||||
|
||||
let meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("nonexistent"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: 100,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(meta);
|
||||
|
||||
let result = fetcher.fetch_file(FileId(1)).await;
|
||||
assert!(matches!(result, Err(FetchError::OriginNotFound(_))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
mod chunks;
|
||||
mod fetcher;
|
||||
mod reader;
|
||||
mod store;
|
||||
|
||||
pub use chunks::{ChunkLocation, ChunkRef};
|
||||
pub use fetcher::{ContentFetcher, FetchError};
|
||||
pub use reader::{ChunkManifest, FileReader, ReaderError};
|
||||
pub use store::{CasConfig, CasError, CasStore, DedupStats};
|
||||
@@ -0,0 +1,332 @@
|
||||
use crate::chunks::ChunkRef;
|
||||
use crate::fetcher::{ContentFetcher, FetchError};
|
||||
use crate::store::{CasError, CasStore};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use musicfs_core::FileId;
|
||||
use parking_lot::RwLock;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkManifest {
|
||||
pub file_id: FileId,
|
||||
pub total_size: u64,
|
||||
pub mtime: i64,
|
||||
pub chunks: Vec<ChunkRef>,
|
||||
}
|
||||
|
||||
impl ChunkManifest {
|
||||
pub fn chunks_to_bytes(&self) -> Vec<u8> {
|
||||
rmp_serde::to_vec(&self.chunks).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn chunks_from_bytes(data: &[u8]) -> Option<Vec<ChunkRef>> {
|
||||
rmp_serde::from_slice(data).ok()
|
||||
}
|
||||
|
||||
pub fn from_db(
|
||||
file_id: FileId,
|
||||
total_size: u64,
|
||||
mtime: i64,
|
||||
chunk_blob: &[u8],
|
||||
) -> Option<Self> {
|
||||
let chunks = Self::chunks_from_bytes(chunk_blob)?;
|
||||
Some(Self {
|
||||
file_id,
|
||||
total_size,
|
||||
mtime,
|
||||
chunks,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FileReader {
|
||||
store: Arc<CasStore>,
|
||||
fetcher: Option<Arc<ContentFetcher>>,
|
||||
manifests: RwLock<HashMap<FileId, ChunkManifest>>,
|
||||
}
|
||||
|
||||
impl FileReader {
|
||||
pub fn new(store: Arc<CasStore>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
fetcher: None,
|
||||
manifests: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_fetcher(store: Arc<CasStore>, fetcher: Arc<ContentFetcher>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
fetcher: Some(fetcher),
|
||||
manifests: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_manifest(&self, manifest: ChunkManifest) {
|
||||
let mut manifests = self.manifests.write();
|
||||
manifests.insert(manifest.file_id, manifest);
|
||||
}
|
||||
|
||||
async fn get_or_fetch_manifest(&self, file_id: FileId) -> Result<ChunkManifest, ReaderError> {
|
||||
{
|
||||
let manifests = self.manifests.read();
|
||||
if let Some(m) = manifests.get(&file_id) {
|
||||
trace!(file_id = ?file_id, "manifest cache hit");
|
||||
return Ok(m.clone());
|
||||
}
|
||||
}
|
||||
|
||||
trace!(file_id = ?file_id, "manifest cache miss");
|
||||
let Some(fetcher) = &self.fetcher else {
|
||||
return Err(ReaderError::ManifestNotFound(file_id));
|
||||
};
|
||||
|
||||
let manifest = fetcher.ensure_cached(file_id).await?;
|
||||
self.manifests.write().insert(file_id, manifest.clone());
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
pub async fn read(
|
||||
&self,
|
||||
file_id: FileId,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> Result<Bytes, ReaderError> {
|
||||
let manifest = self.get_or_fetch_manifest(file_id).await?;
|
||||
|
||||
if let Some(fetcher) = &self.fetcher {
|
||||
if let Some(meta) = fetcher.get_file_meta(file_id) {
|
||||
fetcher.emit_access_event(&meta, offset, size);
|
||||
}
|
||||
}
|
||||
|
||||
if offset >= manifest.total_size {
|
||||
return Ok(Bytes::new());
|
||||
}
|
||||
|
||||
let end = std::cmp::min(offset + size as u64, manifest.total_size);
|
||||
let mut result = BytesMut::with_capacity((end - offset) as usize);
|
||||
let mut chunks_read = 0u32;
|
||||
|
||||
for chunk_ref in &manifest.chunks {
|
||||
let chunk_start = chunk_ref.offset;
|
||||
let chunk_end = chunk_ref.offset + chunk_ref.size as u64;
|
||||
|
||||
if chunk_end <= offset || chunk_start >= end {
|
||||
continue;
|
||||
}
|
||||
|
||||
let chunk_data = match self.store.get(&chunk_ref.hash).await {
|
||||
Ok(data) => data,
|
||||
Err(CasError::IntegrityError { .. }) => {
|
||||
warn!(hash = %chunk_ref.hash, "Chunk corrupt, deleting and re-fetching");
|
||||
let _ = self.store.delete(&chunk_ref.hash).await;
|
||||
if let Some(fetcher) = &self.fetcher {
|
||||
let new_manifest = fetcher.fetch_file(file_id).await?;
|
||||
self.manifests.write().insert(file_id, new_manifest);
|
||||
self.store.get(&chunk_ref.hash).await?
|
||||
} else {
|
||||
return Err(ReaderError::Cas(CasError::NotFound(
|
||||
chunk_ref.hash.as_hex(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(CasError::NotFound(_)) => {
|
||||
warn!(hash = %chunk_ref.hash, "Chunk missing, attempting re-fetch");
|
||||
if let Some(fetcher) = &self.fetcher {
|
||||
let new_manifest = fetcher.fetch_file(file_id).await?;
|
||||
self.manifests.write().insert(file_id, new_manifest);
|
||||
self.store.get(&chunk_ref.hash).await?
|
||||
} else {
|
||||
return Err(ReaderError::Cas(CasError::NotFound(
|
||||
chunk_ref.hash.as_hex(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(ReaderError::Cas(e)),
|
||||
};
|
||||
|
||||
let read_start = if offset > chunk_start {
|
||||
(offset - chunk_start) as usize
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let read_end = if end < chunk_end {
|
||||
(end - chunk_start) as usize
|
||||
} else {
|
||||
chunk_ref.size as usize
|
||||
};
|
||||
|
||||
result.extend_from_slice(&chunk_data[read_start..read_end]);
|
||||
chunks_read += 1;
|
||||
}
|
||||
|
||||
let bytes_read = result.len() as u64;
|
||||
debug!(file_id = ?file_id, offset, size, chunks_read, bytes_read, "read completed");
|
||||
Ok(result.freeze())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ReaderError {
|
||||
#[error("Manifest not found for file {0:?}")]
|
||||
ManifestNotFound(FileId),
|
||||
|
||||
#[error("Fetch error: {0}")]
|
||||
Fetch(#[from] FetchError),
|
||||
|
||||
#[error("CAS error: {0}")]
|
||||
Cas(#[from] crate::store::CasError),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::CasConfig;
|
||||
use musicfs_core::ChunkHash;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_reader_simple() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let data = b"Hello, World!";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let reader = FileReader::new(store);
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 0,
|
||||
size: data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let result = reader.read(FileId(1), 0, data.len() as u32).await.unwrap();
|
||||
assert_eq!(&result[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_reader_partial() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let data = b"ABCDEFGHIJ";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let reader = FileReader::new(store);
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 0,
|
||||
size: data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let result = reader.read(FileId(1), 3, 4).await.unwrap();
|
||||
assert_eq!(&result[..], b"DEFG");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_reader_multi_chunk() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let chunk1 = b"AAAA";
|
||||
let chunk2 = b"BBBB";
|
||||
let hash1 = store.put(chunk1).await.unwrap();
|
||||
let hash2 = store.put(chunk2).await.unwrap();
|
||||
|
||||
let reader = FileReader::new(store);
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: 8,
|
||||
mtime: 0,
|
||||
chunks: vec![
|
||||
ChunkRef {
|
||||
hash: hash1,
|
||||
offset: 0,
|
||||
size: 4,
|
||||
},
|
||||
ChunkRef {
|
||||
hash: hash2,
|
||||
offset: 4,
|
||||
size: 4,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
let result = reader.read(FileId(1), 2, 4).await.unwrap();
|
||||
assert_eq!(&result[..], b"AABB");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_reader_eof() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let data = b"short";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let reader = FileReader::new(store);
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 0,
|
||||
size: data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let result = reader.read(FileId(1), 100, 10).await.unwrap();
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunk_manifest_serialization() {
|
||||
let manifest = ChunkManifest {
|
||||
file_id: FileId(42),
|
||||
total_size: 1024,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash: ChunkHash::from_bytes(b"test"),
|
||||
offset: 0,
|
||||
size: 1024,
|
||||
}],
|
||||
};
|
||||
|
||||
let bytes = manifest.chunks_to_bytes();
|
||||
let restored = ChunkManifest::chunks_from_bytes(&bytes).unwrap();
|
||||
assert_eq!(restored.len(), 1);
|
||||
assert_eq!(restored[0].size, 1024);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
use crate::chunks::ChunkLocation;
|
||||
use bytes::Bytes;
|
||||
use musicfs_core::ChunkHash;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::fs;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
use fail::fail_point;
|
||||
|
||||
const DEFAULT_MAX_SIZE_10GB: u64 = 10 * 1024 * 1024 * 1024;
|
||||
const DEFAULT_SHARD_LEVELS_256_SUBDIRS: u8 = 2;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CasConfig {
|
||||
pub chunks_dir: PathBuf,
|
||||
pub max_size: u64,
|
||||
pub shard_levels: u8,
|
||||
}
|
||||
|
||||
impl Default for CasConfig {
|
||||
fn default() -> Self {
|
||||
let cache_dir = dirs::cache_dir()
|
||||
.unwrap_or_else(|| PathBuf::from(".cache"))
|
||||
.join("musicfs")
|
||||
.join("chunks");
|
||||
|
||||
Self {
|
||||
chunks_dir: cache_dir,
|
||||
max_size: DEFAULT_MAX_SIZE_10GB,
|
||||
shard_levels: DEFAULT_SHARD_LEVELS_256_SUBDIRS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CasStore {
|
||||
config: CasConfig,
|
||||
index: sled::Db,
|
||||
current_size: AtomicU64,
|
||||
}
|
||||
|
||||
impl CasStore {
|
||||
pub async fn open(config: CasConfig) -> Result<Self, CasError> {
|
||||
fs::create_dir_all(&config.chunks_dir).await?;
|
||||
|
||||
let index_path = config.chunks_dir.join("index.sled");
|
||||
let index = match sled::open(&index_path) {
|
||||
Ok(db) => db,
|
||||
Err(e) => {
|
||||
warn!(error = %e, path = ?index_path, "sled index corrupted, attempting recovery");
|
||||
|
||||
match sled::Config::new().path(&index_path).open() {
|
||||
Ok(db) => {
|
||||
info!("sled index repaired successfully");
|
||||
db
|
||||
}
|
||||
Err(repair_err) => {
|
||||
warn!(error = %repair_err, "sled repair failed, recreating index");
|
||||
if index_path.exists() {
|
||||
std::fs::remove_dir_all(&index_path).map_err(CasError::Io)?;
|
||||
}
|
||||
sled::open(&index_path)?
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let current_size = Self::calculate_size(&config.chunks_dir).await;
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
index,
|
||||
current_size: AtomicU64::new(current_size),
|
||||
})
|
||||
}
|
||||
|
||||
async fn calculate_size(dir: &Path) -> u64 {
|
||||
Self::calculate_size_recursive(dir).await
|
||||
}
|
||||
|
||||
fn calculate_size_recursive(
|
||||
dir: &Path,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = u64> + Send + '_>> {
|
||||
Box::pin(async move {
|
||||
let mut size = 0u64;
|
||||
if let Ok(mut entries) = fs::read_dir(dir).await {
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
if let Ok(meta) = entry.metadata().await {
|
||||
if meta.is_file() {
|
||||
size += meta.len();
|
||||
} else if meta.is_dir() {
|
||||
// Skip sled index directory
|
||||
let name = entry.file_name();
|
||||
if name != "index.sled" {
|
||||
size += Self::calculate_size_recursive(&entry.path()).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
size
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
|
||||
let hash = ChunkHash::from_bytes(data);
|
||||
let path = self.chunk_path(&hash);
|
||||
|
||||
if path.exists() {
|
||||
trace!(hash = %hash, size_bytes = data.len(), "dedup hit");
|
||||
return Ok(hash);
|
||||
}
|
||||
|
||||
if self.config.max_size > 0 {
|
||||
let new_size = self.current_size.load(Ordering::SeqCst) + data.len() as u64;
|
||||
if new_size > self.config.max_size {
|
||||
warn!(
|
||||
current_size = self.current_size.load(Ordering::SeqCst),
|
||||
chunk_size = data.len(),
|
||||
max_size = self.config.max_size,
|
||||
"CAS store full, rejecting write"
|
||||
);
|
||||
return Err(CasError::StoreFull {
|
||||
current: self.current_size.load(Ordering::SeqCst),
|
||||
max: self.config.max_size,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
fail_point!("cas-put-before-write", |_| {
|
||||
Err(CasError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Failpoint: cas-put-before-write",
|
||||
)))
|
||||
});
|
||||
|
||||
fs::write(&path, data).await?;
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
fail_point!("cas-put-after-write-before-index", |_| {
|
||||
Err(CasError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Failpoint: cas-put-after-write-before-index",
|
||||
)))
|
||||
});
|
||||
|
||||
let location = ChunkLocation {
|
||||
path: path.clone(),
|
||||
size: data.len() as u32,
|
||||
};
|
||||
self.index.insert(
|
||||
hash.0.as_slice(),
|
||||
rmp_serde::to_vec(&location).map_err(|e| CasError::Serialization(e.to_string()))?,
|
||||
)?;
|
||||
|
||||
self.current_size
|
||||
.fetch_add(data.len() as u64, Ordering::SeqCst);
|
||||
|
||||
debug!(hash = %hash, size_bytes = data.len(), "chunk stored");
|
||||
Ok(hash)
|
||||
}
|
||||
|
||||
pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
|
||||
let path = self.chunk_path(hash);
|
||||
|
||||
if !path.exists() {
|
||||
return Err(CasError::NotFound(hash.as_hex()));
|
||||
}
|
||||
|
||||
let data = fs::read(&path).await?;
|
||||
|
||||
if self.config.max_size > 0 {
|
||||
self.verify_integrity(hash, &data)?;
|
||||
}
|
||||
|
||||
debug!(hash = %hash, size_bytes = data.len(), "chunk retrieved");
|
||||
Ok(Bytes::from(data))
|
||||
}
|
||||
|
||||
pub fn exists(&self, hash: &ChunkHash) -> bool {
|
||||
self.chunk_path(hash).exists()
|
||||
}
|
||||
|
||||
fn verify_integrity(&self, expected: &ChunkHash, data: &[u8]) -> Result<(), CasError> {
|
||||
let actual = ChunkHash::from_bytes(data);
|
||||
if actual != *expected {
|
||||
warn!(
|
||||
"Chunk integrity failure: expected {}, got {}",
|
||||
expected, actual
|
||||
);
|
||||
return Err(CasError::IntegrityError {
|
||||
expected: expected.as_hex(),
|
||||
actual: actual.as_hex(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn chunk_path(&self, hash: &ChunkHash) -> PathBuf {
|
||||
let hex = hash.as_hex();
|
||||
let mut path = self.config.chunks_dir.clone();
|
||||
|
||||
for i in 0..self.config.shard_levels as usize {
|
||||
let start = i * 2;
|
||||
let end = start + 2;
|
||||
if end <= hex.len() {
|
||||
path = path.join(&hex[start..end]);
|
||||
}
|
||||
}
|
||||
|
||||
path.join(&hex)
|
||||
}
|
||||
|
||||
pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
|
||||
let path = self.chunk_path(hash);
|
||||
|
||||
if path.exists() {
|
||||
let meta = fs::metadata(&path).await?;
|
||||
fs::remove_file(&path).await?;
|
||||
self.index.remove(hash.0.as_slice())?;
|
||||
self.current_size.fetch_sub(meta.len(), Ordering::SeqCst);
|
||||
debug!(hash = %hash, size_bytes = meta.len(), "chunk deleted");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn current_size(&self) -> u64 {
|
||||
self.current_size.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn max_size(&self) -> u64 {
|
||||
self.config.max_size
|
||||
}
|
||||
|
||||
pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
|
||||
self.index.iter().filter_map(|r| {
|
||||
r.ok().and_then(|(k, _)| {
|
||||
if k.len() == 8 {
|
||||
let mut arr = [0u8; 8];
|
||||
arr.copy_from_slice(&k);
|
||||
Some(ChunkHash(arr))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn dedup_stats(&self) -> DedupStats {
|
||||
let chunks_stored = self.index.len() as u64;
|
||||
let size_bytes = self.current_size();
|
||||
|
||||
DedupStats {
|
||||
chunks_stored,
|
||||
chunks_unique: chunks_stored,
|
||||
size_bytes,
|
||||
size_limit_bytes: self.config.max_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DedupStats {
|
||||
pub chunks_stored: u64,
|
||||
pub chunks_unique: u64,
|
||||
pub size_bytes: u64,
|
||||
pub size_limit_bytes: u64,
|
||||
}
|
||||
|
||||
impl DedupStats {
|
||||
pub fn dedup_ratio(&self) -> f64 {
|
||||
if self.chunks_stored == 0 {
|
||||
0.0
|
||||
} else {
|
||||
1.0 - (self.chunks_unique as f64 / self.chunks_stored as f64)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CasError {
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("Sled error: {0}")]
|
||||
Sled(#[from] sled::Error),
|
||||
|
||||
#[error("Chunk not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
#[error("Integrity error: expected {expected}, got {actual}")]
|
||||
IntegrityError { expected: String, actual: String },
|
||||
|
||||
#[error("Serialization error: {0}")]
|
||||
Serialization(String),
|
||||
|
||||
#[error("Store full: {current} / {max} bytes")]
|
||||
StoreFull { current: u64, max: u64 },
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
async fn test_store() -> (CasStore, TempDir) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 1024 * 1024,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = CasStore::open(config).await.unwrap();
|
||||
(store, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_put_get() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"test chunk data";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let retrieved = store.get(&hash).await.unwrap();
|
||||
assert_eq!(&retrieved[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_dedup() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"duplicate data";
|
||||
let hash1 = store.put(data).await.unwrap();
|
||||
let hash2 = store.put(data).await.unwrap();
|
||||
|
||||
assert_eq!(hash1, hash2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_exists() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"existence test";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
assert!(store.exists(&hash));
|
||||
|
||||
let fake_hash = ChunkHash::from_bytes(b"nonexistent");
|
||||
assert!(!store.exists(&fake_hash));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_delete() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"delete me";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
assert!(store.exists(&hash));
|
||||
|
||||
store.delete(&hash).await.unwrap();
|
||||
|
||||
assert!(!store.exists(&hash));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_integrity() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"integrity test";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let retrieved = store.get(&hash).await.unwrap();
|
||||
assert_eq!(&retrieved[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_dedup_stats() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
store.put(b"chunk1").await.unwrap();
|
||||
store.put(b"chunk2").await.unwrap();
|
||||
store.put(b"chunk1").await.unwrap();
|
||||
|
||||
let stats = store.dedup_stats();
|
||||
assert_eq!(stats.chunks_stored, 2);
|
||||
assert_eq!(stats.chunks_unique, 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_cas::{CasConfig, CasStore, ChunkManifest, ChunkRef, ContentFetcher, FileReader};
|
||||
use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath};
|
||||
use musicfs_origins::LocalOrigin;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::SystemTime;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from("/test"),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_and_tree_integration() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let file_data = b"This is test audio file content for testing.";
|
||||
let chunk_hash = store.put(file_data).await.unwrap();
|
||||
|
||||
let mut builder = TreeBuilder::new();
|
||||
builder.add_file(&make_file_meta(
|
||||
1,
|
||||
"/Artist/Album/Track.flac",
|
||||
file_data.len() as u64,
|
||||
));
|
||||
let _tree = Arc::new(RwLock::new(builder.build()));
|
||||
|
||||
let reader = Arc::new(FileReader::new(store.clone()));
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: file_data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash: chunk_hash,
|
||||
offset: 0,
|
||||
size: file_data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let result = reader
|
||||
.read(FileId(1), 0, file_data.len() as u32)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(&result[..], file_data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let data = b"persistent data";
|
||||
let hash = {
|
||||
let store = CasStore::open(config.clone()).await.unwrap();
|
||||
store.put(data).await.unwrap()
|
||||
};
|
||||
|
||||
let store = CasStore::open(config).await.unwrap();
|
||||
let retrieved = store.get(&hash).await.unwrap();
|
||||
assert_eq!(&retrieved[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deduplication() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = CasStore::open(config).await.unwrap();
|
||||
|
||||
let data = b"duplicate this content";
|
||||
|
||||
let hash1 = store.put(data).await.unwrap();
|
||||
let size_after_first = store.current_size();
|
||||
|
||||
let hash2 = store.put(data).await.unwrap();
|
||||
let size_after_second = store.current_size();
|
||||
|
||||
assert_eq!(hash1, hash2);
|
||||
assert_eq!(size_after_first, size_after_second);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetcher_cache_miss_flow() {
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
|
||||
let test_content = b"This is audio content that will be fetched on cache miss";
|
||||
let test_file_path = origin_dir.path().join("test.flac");
|
||||
std::fs::write(&test_file_path, test_content).unwrap();
|
||||
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let origin_id = OriginId::from("test-origin");
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
origin_id.clone(),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
|
||||
let fetcher = ContentFetcher::new(store.clone());
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let file_id = FileId(42);
|
||||
let file_meta = FileMeta {
|
||||
id: file_id,
|
||||
virtual_path: VirtualPath::new("/Artist/Album/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id,
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: test_content.len() as u64,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(file_meta);
|
||||
|
||||
let manifest = fetcher.fetch_file(file_id).await.unwrap();
|
||||
|
||||
assert_eq!(manifest.file_id, file_id);
|
||||
assert_eq!(manifest.total_size, test_content.len() as u64);
|
||||
assert_eq!(manifest.chunks.len(), 1);
|
||||
|
||||
let chunk_data = store.get(&manifest.chunks[0].hash).await.unwrap();
|
||||
assert_eq!(&chunk_data[..], test_content);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_reader_with_fetcher_integration() {
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
|
||||
let test_content = b"Audio file content for reader integration test";
|
||||
let test_file_path = origin_dir.path().join("song.flac");
|
||||
std::fs::write(&test_file_path, test_content).unwrap();
|
||||
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let origin_id = OriginId::from("local");
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
origin_id.clone(),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
|
||||
let fetcher = ContentFetcher::new(store.clone());
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let file_id = FileId(100);
|
||||
let file_meta = FileMeta {
|
||||
id: file_id,
|
||||
virtual_path: VirtualPath::new("/Test/song.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id,
|
||||
path: PathBuf::from("/song.flac"),
|
||||
},
|
||||
size: test_content.len() as u64,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(file_meta);
|
||||
|
||||
let reader = FileReader::with_fetcher(store, Arc::new(fetcher));
|
||||
|
||||
let result = reader
|
||||
.read(file_id, 0, test_content.len() as u32)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(&result[..], test_content);
|
||||
|
||||
let result2 = reader.read(file_id, 0, 10).await.unwrap();
|
||||
assert_eq!(&result2[..], &test_content[..10]);
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
[package]
|
||||
name = "musicfs-cli"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "musicfs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
musicfs-core.path = "../musicfs-core"
|
||||
musicfs-origins.path = "../musicfs-origins"
|
||||
musicfs-cache.path = "../musicfs-cache"
|
||||
musicfs-cas.path = "../musicfs-cas"
|
||||
musicfs-fuse.path = "../musicfs-fuse"
|
||||
musicfs-metadata.path = "../musicfs-metadata"
|
||||
|
||||
clap.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
tracing-appender.workspace = true
|
||||
anyhow.workspace = true
|
||||
dirs.workspace = true
|
||||
parking_lot.workspace = true
|
||||
libc.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
tracing-journald.workspace = true
|
||||
sd-notify.workspace = true
|
||||
@@ -0,0 +1 @@
|
||||
#![allow(dead_code)]
|
||||
@@ -0,0 +1,529 @@
|
||||
use anyhow::{Context, Result};
|
||||
use clap::{Parser, Subcommand};
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_cas::{CasConfig, CasStore, ContentFetcher, FileReader};
|
||||
use musicfs_core::{FileId, FileMeta, LoggingConfig, OriginId, RealPath, VirtualPath};
|
||||
use musicfs_fuse::MusicFs;
|
||||
use musicfs_metadata::MetadataParser;
|
||||
use musicfs_origins::{LocalOrigin, Origin};
|
||||
use parking_lot::RwLock;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing_appender::non_blocking::WorkerGuard;
|
||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter, Layer};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "musicfs")]
|
||||
#[command(about = "Virtual FUSE filesystem for music libraries")]
|
||||
struct Cli {
|
||||
#[arg(short, long, default_value = "info", help = "Log level")]
|
||||
log_level: String,
|
||||
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
Mount {
|
||||
#[arg(short, long, help = "Config file path")]
|
||||
config: Option<PathBuf>,
|
||||
#[arg(help = "Mount point")]
|
||||
mountpoint: PathBuf,
|
||||
#[arg(short, long, help = "Source music directory")]
|
||||
origin: Option<PathBuf>,
|
||||
#[arg(short = 'd', long, help = "Cache directory")]
|
||||
cache_dir: Option<PathBuf>,
|
||||
},
|
||||
Status,
|
||||
Cache {
|
||||
#[command(subcommand)]
|
||||
command: CacheCommands,
|
||||
},
|
||||
Search {
|
||||
query: String,
|
||||
#[arg(short, long, default_value = "100")]
|
||||
limit: u32,
|
||||
},
|
||||
Origin {
|
||||
#[command(subcommand)]
|
||||
command: OriginCommands,
|
||||
},
|
||||
Events {
|
||||
#[arg(short, long, help = "Filter by event type")]
|
||||
r#type: Option<String>,
|
||||
},
|
||||
Shutdown {
|
||||
#[arg(short, long, default_value = "true")]
|
||||
graceful: bool,
|
||||
#[arg(short, long, default_value = "30")]
|
||||
timeout: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum CacheCommands {
|
||||
Stats,
|
||||
Clear {
|
||||
#[arg(help = "Origin to clear cache for")]
|
||||
origin: Option<String>,
|
||||
},
|
||||
Prefetch {
|
||||
#[arg(help = "Paths to prefetch")]
|
||||
paths: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum OriginCommands {
|
||||
List,
|
||||
Health { origin_id: String },
|
||||
Rescan { origin_id: String },
|
||||
}
|
||||
|
||||
struct LockFile {
|
||||
_file: File,
|
||||
}
|
||||
|
||||
fn try_acquire_lock(path: &Path) -> Result<LockFile> {
|
||||
let file = File::create(path).context("Failed to create lock file")?;
|
||||
let fd = file.as_raw_fd();
|
||||
|
||||
let ret = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
|
||||
if ret != 0 {
|
||||
let err = std::io::Error::last_os_error();
|
||||
if err.kind() == std::io::ErrorKind::WouldBlock {
|
||||
anyhow::bail!("MusicFS is already running (lock file: {:?})", path);
|
||||
}
|
||||
return Err(err).context("Failed to acquire lock");
|
||||
}
|
||||
|
||||
let mut f = &file;
|
||||
writeln!(f, "{}", std::process::id())?;
|
||||
|
||||
Ok(LockFile { _file: file })
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
musicfs_core::install_panic_hook();
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.command {
|
||||
Commands::Mount {
|
||||
config: _,
|
||||
mountpoint,
|
||||
origin,
|
||||
cache_dir,
|
||||
} => {
|
||||
let log_config = LoggingConfig {
|
||||
level: cli.log_level,
|
||||
..Default::default()
|
||||
};
|
||||
let _guard = init_logging(&log_config)?;
|
||||
run_mount(mountpoint, origin, cache_dir)
|
||||
}
|
||||
Commands::Status => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_status()
|
||||
}
|
||||
Commands::Cache { command } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_cache(command)
|
||||
}
|
||||
Commands::Search { query, limit } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_search(&query, limit)
|
||||
}
|
||||
Commands::Origin { command } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_origin(command)
|
||||
}
|
||||
Commands::Events { r#type } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_events(r#type)
|
||||
}
|
||||
Commands::Shutdown { graceful, timeout } => {
|
||||
init_basic_logging(&cli.log_level);
|
||||
run_shutdown(graceful, timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_mount(
|
||||
mountpoint: PathBuf,
|
||||
origin_path: Option<PathBuf>,
|
||||
cache_dir: Option<PathBuf>,
|
||||
) -> Result<()> {
|
||||
let origin_path = origin_path.context("--origin is required for mount")?;
|
||||
|
||||
let cache_dir = cache_dir.unwrap_or_else(|| {
|
||||
dirs::cache_dir()
|
||||
.unwrap_or_else(|| PathBuf::from("/tmp"))
|
||||
.join("musicfs")
|
||||
});
|
||||
|
||||
let runtime = tokio::runtime::Runtime::new().context("Failed to create Tokio runtime")?;
|
||||
let handle = runtime.handle().clone();
|
||||
|
||||
let cache_dir_clone = cache_dir.clone();
|
||||
let (tree, reader) = runtime.block_on(async {
|
||||
info!(origin = ?origin_path, mountpoint = ?mountpoint, "Mount configuration");
|
||||
info!("Cache directory: {:?}", cache_dir_clone);
|
||||
|
||||
std::fs::create_dir_all(&cache_dir_clone).context("Failed to create cache directory")?;
|
||||
std::fs::create_dir_all(&mountpoint).context("Failed to create mountpoint")?;
|
||||
|
||||
let cas_config = CasConfig {
|
||||
chunks_dir: cache_dir_clone.join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(
|
||||
CasStore::open(cas_config)
|
||||
.await
|
||||
.context("Failed to open CAS store")?,
|
||||
);
|
||||
info!("CAS store initialized");
|
||||
|
||||
let origin_id = OriginId::from("local");
|
||||
let origin = Arc::new(LocalOrigin::new(origin_id.clone(), origin_path.clone()));
|
||||
info!("Origin registered: {}", origin.display_name());
|
||||
|
||||
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
info!("Scanning music files...");
|
||||
let files = scan_music_files(&origin_path, &origin_id).await?;
|
||||
info!("Found {} music files", files.len());
|
||||
|
||||
let mut builder = TreeBuilder::new();
|
||||
for file in &files {
|
||||
builder.add_file(file);
|
||||
fetcher.register_file(file.clone());
|
||||
}
|
||||
let tree = Arc::new(RwLock::new(builder.build()));
|
||||
info!("Virtual tree built");
|
||||
|
||||
let reader = Arc::new(FileReader::with_fetcher(store, fetcher));
|
||||
|
||||
Ok::<_, anyhow::Error>((tree, reader))
|
||||
})?;
|
||||
|
||||
check_stale_mount(&mountpoint)?;
|
||||
|
||||
let lock_path = cache_dir.join("musicfs.lock");
|
||||
let _lock = try_acquire_lock(&lock_path)
|
||||
.context("Failed to acquire lock — is another instance running?")?;
|
||||
info!(lock_path = ?lock_path, "Lock acquired");
|
||||
|
||||
let fs = MusicFs::with_reader(tree, reader, handle.clone());
|
||||
|
||||
info!("Mounting filesystem at {:?}", mountpoint);
|
||||
|
||||
let session = fs
|
||||
.spawn_mount(&mountpoint)
|
||||
.context("Failed to mount filesystem")?;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if let Err(e) = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) {
|
||||
debug!("sd_notify not available (not running under systemd): {}", e);
|
||||
}
|
||||
}
|
||||
info!("MusicFS ready, PID {}", std::process::id());
|
||||
|
||||
let shutdown_token = tokio_util::sync::CancellationToken::new();
|
||||
|
||||
runtime.block_on(async {
|
||||
let mut sigterm =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
|
||||
let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
|
||||
|
||||
tokio::select! {
|
||||
_ = sigterm.recv() => {
|
||||
info!("Received SIGTERM, shutting down");
|
||||
}
|
||||
_ = sigint.recv() => {
|
||||
info!("Received SIGINT, shutting down");
|
||||
}
|
||||
}
|
||||
|
||||
info!("Beginning ordered shutdown");
|
||||
shutdown_token.cancel();
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
|
||||
info!("Background tasks stopped");
|
||||
|
||||
Ok::<_, anyhow::Error>(())
|
||||
})?;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]);
|
||||
}
|
||||
info!("Unmounting filesystem");
|
||||
drop(session);
|
||||
info!("Shutdown complete");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_status() -> Result<()> {
|
||||
println!("Status: Not connected to daemon");
|
||||
println!("Hint: gRPC client integration pending");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_cache(command: CacheCommands) -> Result<()> {
|
||||
match command {
|
||||
CacheCommands::Stats => {
|
||||
println!("Cache stats: gRPC client integration pending");
|
||||
}
|
||||
CacheCommands::Clear { origin } => {
|
||||
println!("Clearing cache for: {}", origin.as_deref().unwrap_or("all"));
|
||||
println!("gRPC client integration pending");
|
||||
}
|
||||
CacheCommands::Prefetch { paths } => {
|
||||
println!("Prefetching {} paths", paths.len());
|
||||
println!("gRPC client integration pending");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_search(query: &str, limit: u32) -> Result<()> {
|
||||
println!("Searching for: {} (limit: {})", query, limit);
|
||||
println!("gRPC client integration pending");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_origin(command: OriginCommands) -> Result<()> {
|
||||
match command {
|
||||
OriginCommands::List => {
|
||||
println!("Origins: gRPC client integration pending");
|
||||
}
|
||||
OriginCommands::Health { origin_id } => {
|
||||
println!("Health for {}: gRPC client integration pending", origin_id);
|
||||
}
|
||||
OriginCommands::Rescan { origin_id } => {
|
||||
println!("Rescanning {}: gRPC client integration pending", origin_id);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_events(event_type: Option<String>) -> Result<()> {
|
||||
println!(
|
||||
"Subscribing to events: {}",
|
||||
event_type.as_deref().unwrap_or("all")
|
||||
);
|
||||
println!("gRPC client integration pending");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_shutdown(graceful: bool, timeout: u32) -> Result<()> {
|
||||
println!(
|
||||
"Shutdown requested (graceful: {}, timeout: {}s)",
|
||||
graceful, timeout
|
||||
);
|
||||
println!("gRPC client integration pending");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn init_logging(config: &LoggingConfig) -> Result<WorkerGuard> {
|
||||
std::fs::create_dir_all(&config.log_dir)?;
|
||||
|
||||
let file_appender = tracing_appender::rolling::daily(&config.log_dir, "musicfs.log");
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
|
||||
let file_layer = if config.json_output {
|
||||
fmt::layer()
|
||||
.json()
|
||||
.with_writer(non_blocking)
|
||||
.with_ansi(false)
|
||||
.boxed()
|
||||
} else {
|
||||
fmt::layer()
|
||||
.with_writer(non_blocking)
|
||||
.with_ansi(false)
|
||||
.boxed()
|
||||
};
|
||||
|
||||
let stderr_layer = fmt::layer().with_writer(std::io::stderr).compact();
|
||||
|
||||
let filter =
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&config.level));
|
||||
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(file_layer)
|
||||
.with(stderr_layer);
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let subscriber = {
|
||||
let journald_layer = if config.journald {
|
||||
tracing_journald::layer()
|
||||
.ok()
|
||||
.map(|l| l.with_syslog_identifier("musicfs".to_string()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
subscriber.with(journald_layer)
|
||||
};
|
||||
|
||||
subscriber.init();
|
||||
|
||||
info!(version = env!("CARGO_PKG_VERSION"), "MusicFS starting");
|
||||
Ok(guard)
|
||||
}
|
||||
|
||||
fn init_basic_logging(level: &str) {
|
||||
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level));
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(fmt::layer().compact())
|
||||
.with(filter)
|
||||
.init();
|
||||
}
|
||||
|
||||
async fn scan_music_files(dir: &Path, origin_id: &OriginId) -> Result<Vec<FileMeta>> {
|
||||
let parser = MetadataParser::new();
|
||||
let mut files = Vec::new();
|
||||
let mut file_id_counter = 1i64;
|
||||
|
||||
scan_dir_recursive(
|
||||
dir,
|
||||
dir,
|
||||
origin_id,
|
||||
&parser,
|
||||
&mut files,
|
||||
&mut file_id_counter,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
async fn scan_dir_recursive(
|
||||
base: &Path,
|
||||
dir: &Path,
|
||||
origin_id: &OriginId,
|
||||
parser: &MetadataParser,
|
||||
files: &mut Vec<FileMeta>,
|
||||
id_counter: &mut i64,
|
||||
) -> Result<()> {
|
||||
let mut entries = tokio::fs::read_dir(dir).await?;
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let metadata = entry.metadata().await?;
|
||||
|
||||
if metadata.is_dir() {
|
||||
Box::pin(scan_dir_recursive(
|
||||
base, &path, origin_id, parser, files, id_counter,
|
||||
))
|
||||
.await?;
|
||||
} else if is_audio_file(&path) {
|
||||
let relative_path = path.strip_prefix(base).unwrap_or(&path);
|
||||
|
||||
let audio_meta = match parser.parse_file(&path) {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(e) => {
|
||||
debug!("Failed to parse metadata for {:?}: {}", path, e);
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let virtual_path = build_virtual_path(&path, audio_meta.as_ref());
|
||||
|
||||
let file_meta = FileMeta {
|
||||
id: FileId(*id_counter),
|
||||
virtual_path,
|
||||
real_path: RealPath {
|
||||
origin_id: origin_id.clone(),
|
||||
path: PathBuf::from("/").join(relative_path),
|
||||
},
|
||||
size: metadata.len(),
|
||||
mtime: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
|
||||
content_hash: None,
|
||||
audio: audio_meta,
|
||||
};
|
||||
|
||||
debug!(
|
||||
"Found: {:?} -> {:?}",
|
||||
file_meta.real_path.path, file_meta.virtual_path
|
||||
);
|
||||
files.push(file_meta);
|
||||
*id_counter += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_audio_file(path: &Path) -> bool {
|
||||
matches!(
|
||||
path.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.to_lowercase())
|
||||
.as_deref(),
|
||||
Some("flac" | "mp3" | "ogg" | "wav" | "m4a" | "aac" | "opus")
|
||||
)
|
||||
}
|
||||
|
||||
fn build_virtual_path(path: &Path, audio: Option<&musicfs_core::AudioMeta>) -> VirtualPath {
|
||||
if let Some(meta) = audio {
|
||||
let artist = meta.artist.as_deref().unwrap_or("Unknown Artist");
|
||||
let album = meta.album.as_deref().unwrap_or("Unknown Album");
|
||||
let filename = path.file_name().and_then(|n| n.to_str()).unwrap_or("track");
|
||||
|
||||
VirtualPath::new(&format!(
|
||||
"/{}/{}/{}",
|
||||
sanitize(artist),
|
||||
sanitize(album),
|
||||
filename
|
||||
))
|
||||
} else {
|
||||
let filename = path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("unknown");
|
||||
VirtualPath::new(&format!("/Unknown Artist/Unknown Album/{}", filename))
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| match c {
|
||||
'/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_',
|
||||
_ => c,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn check_stale_mount(mountpoint: &Path) -> Result<()> {
|
||||
if let Ok(mounts) = std::fs::read_to_string("/proc/mounts") {
|
||||
for line in mounts.lines() {
|
||||
if line.contains(mountpoint.to_string_lossy().as_ref()) && line.contains("fuse") {
|
||||
warn!(
|
||||
"Stale FUSE mount detected at {:?}, attempting cleanup",
|
||||
mountpoint
|
||||
);
|
||||
let status = std::process::Command::new("fusermount")
|
||||
.args(["-uz", &mountpoint.to_string_lossy()])
|
||||
.status();
|
||||
match status {
|
||||
Ok(s) if s.success() => info!("Stale mount cleaned up"),
|
||||
Ok(s) => warn!("fusermount exited with: {}", s),
|
||||
Err(e) => warn!("Failed to run fusermount: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "musicfs-core"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
toml.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tracing.workspace = true
|
||||
xxhash-rust.workspace = true
|
||||
hex.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -0,0 +1,239 @@
|
||||
use crate::OriginId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Config {
|
||||
pub mount_point: PathBuf,
|
||||
pub cache_dir: PathBuf,
|
||||
pub origins: Vec<OriginConfig>,
|
||||
|
||||
#[serde(default)]
|
||||
pub cache: CacheConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub health: HealthConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub logging: LoggingConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OriginConfig {
|
||||
pub id: String,
|
||||
pub origin_type: OriginType,
|
||||
pub priority: u8,
|
||||
|
||||
#[serde(default = "default_enabled")]
|
||||
pub enabled: bool,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub settings: HashMap<String, toml::Value>,
|
||||
}
|
||||
|
||||
fn default_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum OriginType {
|
||||
Local,
|
||||
Nfs,
|
||||
Smb,
|
||||
S3,
|
||||
Sftp,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CacheConfig {
|
||||
#[serde(default = "default_metadata_cache_mb")]
|
||||
pub metadata_cache_mb: u64,
|
||||
|
||||
#[serde(default = "default_content_cache_gb")]
|
||||
pub content_cache_gb: u64,
|
||||
}
|
||||
|
||||
impl Default for CacheConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
metadata_cache_mb: default_metadata_cache_mb(),
|
||||
content_cache_gb: default_content_cache_gb(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_metadata_cache_mb() -> u64 {
|
||||
100
|
||||
}
|
||||
fn default_content_cache_gb() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HealthConfig {
|
||||
#[serde(default = "default_check_interval_secs")]
|
||||
pub check_interval_secs: u64,
|
||||
|
||||
#[serde(default = "default_timeout_ms")]
|
||||
pub timeout_ms: u64,
|
||||
|
||||
#[serde(default = "default_unhealthy_threshold")]
|
||||
pub unhealthy_threshold: u32,
|
||||
|
||||
#[serde(default)]
|
||||
pub per_origin_thresholds: HashMap<OriginType, u32>,
|
||||
}
|
||||
|
||||
impl Default for HealthConfig {
|
||||
fn default() -> Self {
|
||||
let mut per_origin = HashMap::new();
|
||||
per_origin.insert(OriginType::Local, 1);
|
||||
per_origin.insert(OriginType::Nfs, 3);
|
||||
per_origin.insert(OriginType::Smb, 3);
|
||||
per_origin.insert(OriginType::S3, 3);
|
||||
per_origin.insert(OriginType::Sftp, 3);
|
||||
|
||||
Self {
|
||||
check_interval_secs: default_check_interval_secs(),
|
||||
timeout_ms: default_timeout_ms(),
|
||||
unhealthy_threshold: default_unhealthy_threshold(),
|
||||
per_origin_thresholds: per_origin,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthConfig {
|
||||
pub fn threshold_for(&self, origin_type: OriginType) -> u32 {
|
||||
self.per_origin_thresholds
|
||||
.get(&origin_type)
|
||||
.copied()
|
||||
.unwrap_or(self.unhealthy_threshold)
|
||||
}
|
||||
}
|
||||
|
||||
fn default_check_interval_secs() -> u64 {
|
||||
30
|
||||
}
|
||||
fn default_timeout_ms() -> u64 {
|
||||
5000
|
||||
}
|
||||
fn default_unhealthy_threshold() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LoggingConfig {
|
||||
#[serde(default = "default_log_dir")]
|
||||
pub log_dir: PathBuf,
|
||||
|
||||
#[serde(default)]
|
||||
pub json_output: bool,
|
||||
|
||||
#[serde(default = "default_true")]
|
||||
pub journald: bool,
|
||||
|
||||
#[serde(default = "default_log_level")]
|
||||
pub level: String,
|
||||
|
||||
#[serde(default = "default_sample_rate")]
|
||||
pub trace_sample_rate: f32,
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
log_dir: default_log_dir(),
|
||||
json_output: false,
|
||||
journald: true,
|
||||
level: default_log_level(),
|
||||
trace_sample_rate: default_sample_rate(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_log_dir() -> PathBuf {
|
||||
PathBuf::from("/var/log/musicfs")
|
||||
}
|
||||
|
||||
fn default_log_level() -> String {
|
||||
"musicfs=info,warn".to_string()
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn default_sample_rate() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn from_file(path: &std::path::Path) -> Result<Self, ConfigError> {
|
||||
let content =
|
||||
std::fs::read_to_string(path).map_err(|e| ConfigError::Read(e.to_string()))?;
|
||||
toml::from_str(&content).map_err(|e| ConfigError::Parse(e.to_string()))
|
||||
}
|
||||
|
||||
pub fn origin_id(&self, id: &str) -> Option<OriginId> {
|
||||
self.origins
|
||||
.iter()
|
||||
.find(|o| o.id == id)
|
||||
.map(|_| OriginId::from(id))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("Failed to read config: {0}")]
|
||||
Read(String),
|
||||
|
||||
#[error("Failed to parse config: {0}")]
|
||||
Parse(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_config() {
|
||||
let toml = r#"
|
||||
mount_point = "/mnt/music"
|
||||
cache_dir = "/home/user/.cache/musicfs"
|
||||
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/mnt/nas/music"
|
||||
|
||||
[[origins]]
|
||||
id = "backup"
|
||||
origin_type = "s3"
|
||||
priority = 2
|
||||
bucket = "music-backup"
|
||||
region = "us-east-1"
|
||||
"#;
|
||||
|
||||
let config: Config = toml::from_str(toml).unwrap();
|
||||
assert_eq!(config.origins.len(), 2);
|
||||
assert_eq!(config.origins[0].priority, 1);
|
||||
assert_eq!(config.origins[1].origin_type, OriginType::S3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_health_thresholds() {
|
||||
let health = HealthConfig::default();
|
||||
assert_eq!(health.threshold_for(OriginType::Local), 1);
|
||||
assert_eq!(health.threshold_for(OriginType::Sftp), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_defaults() {
|
||||
let cache = CacheConfig::default();
|
||||
assert_eq!(cache.metadata_cache_mb, 100);
|
||||
assert_eq!(cache.content_cache_gb, 10);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use thiserror::Error;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CredentialStore {
|
||||
cache: HashMap<String, Credential>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for CredentialStore {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("CredentialStore")
|
||||
.field("cache_keys", &self.cache.keys().collect::<Vec<_>>())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum Credential {
|
||||
Basic {
|
||||
username: String,
|
||||
#[serde(skip_serializing)]
|
||||
password: String,
|
||||
},
|
||||
|
||||
AwsKey {
|
||||
access_key_id: String,
|
||||
#[serde(skip_serializing)]
|
||||
secret_access_key: String,
|
||||
session_token: Option<String>,
|
||||
region: String,
|
||||
},
|
||||
|
||||
SshKey {
|
||||
username: String,
|
||||
private_key_path: PathBuf,
|
||||
#[serde(skip_serializing)]
|
||||
passphrase: Option<String>,
|
||||
},
|
||||
|
||||
EnvVar {
|
||||
var_name: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Credential {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Basic { username, .. } => f
|
||||
.debug_struct("Basic")
|
||||
.field("username", username)
|
||||
.field("password", &"[REDACTED]")
|
||||
.finish(),
|
||||
Self::AwsKey {
|
||||
access_key_id,
|
||||
session_token,
|
||||
region,
|
||||
..
|
||||
} => {
|
||||
let key_preview = if access_key_id.len() > 4 {
|
||||
format!("{}...", &access_key_id[..4])
|
||||
} else {
|
||||
"****".to_string()
|
||||
};
|
||||
let token_display = if session_token.is_some() {
|
||||
"[REDACTED]"
|
||||
} else {
|
||||
"None"
|
||||
};
|
||||
f.debug_struct("AwsKey")
|
||||
.field("access_key_id", &key_preview)
|
||||
.field("secret_access_key", &"[REDACTED]")
|
||||
.field("session_token", &token_display)
|
||||
.field("region", region)
|
||||
.finish()
|
||||
}
|
||||
Self::SshKey {
|
||||
username,
|
||||
private_key_path,
|
||||
..
|
||||
} => f
|
||||
.debug_struct("SshKey")
|
||||
.field("username", username)
|
||||
.field("private_key_path", private_key_path)
|
||||
.field("passphrase", &"[REDACTED]")
|
||||
.finish(),
|
||||
Self::EnvVar { var_name } => f
|
||||
.debug_struct("EnvVar")
|
||||
.field("var_name", var_name)
|
||||
.finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CredentialStore {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
cache: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(
|
||||
&mut self,
|
||||
origin_id: &str,
|
||||
config: &CredentialConfig,
|
||||
) -> Result<Credential, CredentialError> {
|
||||
debug!(origin_id = %origin_id, "Loading credentials");
|
||||
|
||||
if let Some(cred) = self.cache.get(origin_id) {
|
||||
trace!(origin_id = %origin_id, "Credential cache hit");
|
||||
return Ok(cred.clone());
|
||||
}
|
||||
|
||||
let cred = match config {
|
||||
CredentialConfig::Environment { prefix } => {
|
||||
trace!(origin_id = %origin_id, prefix = %prefix, "Loading from environment");
|
||||
self.load_from_env(prefix)?
|
||||
}
|
||||
CredentialConfig::File { path } => {
|
||||
trace!(origin_id = %origin_id, path = ?path, "Loading from file");
|
||||
self.load_from_file(path)?
|
||||
}
|
||||
CredentialConfig::Inline(cred) => {
|
||||
trace!(origin_id = %origin_id, "Using inline credential");
|
||||
cred.clone()
|
||||
}
|
||||
};
|
||||
|
||||
let cred_type = match &cred {
|
||||
Credential::Basic { .. } => "Basic",
|
||||
Credential::AwsKey { .. } => "AwsKey",
|
||||
Credential::SshKey { .. } => "SshKey",
|
||||
Credential::EnvVar { .. } => "EnvVar",
|
||||
};
|
||||
info!(origin_id = %origin_id, cred_type = %cred_type, "Credential loaded");
|
||||
|
||||
self.cache.insert(origin_id.to_string(), cred.clone());
|
||||
Ok(cred)
|
||||
}
|
||||
|
||||
fn load_from_env(&self, prefix: &str) -> Result<Credential, CredentialError> {
|
||||
if let (Ok(key), Ok(secret)) = (
|
||||
std::env::var(format!("{}_ACCESS_KEY_ID", prefix)),
|
||||
std::env::var(format!("{}_SECRET_ACCESS_KEY", prefix)),
|
||||
) {
|
||||
return Ok(Credential::AwsKey {
|
||||
access_key_id: key,
|
||||
secret_access_key: secret,
|
||||
session_token: std::env::var(format!("{}_SESSION_TOKEN", prefix)).ok(),
|
||||
region: std::env::var(format!("{}_REGION", prefix))
|
||||
.unwrap_or_else(|_| "us-east-1".to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
if let (Ok(user), Ok(pass)) = (
|
||||
std::env::var(format!("{}_USERNAME", prefix)),
|
||||
std::env::var(format!("{}_PASSWORD", prefix)),
|
||||
) {
|
||||
return Ok(Credential::Basic {
|
||||
username: user,
|
||||
password: pass,
|
||||
});
|
||||
}
|
||||
|
||||
warn!(prefix = %prefix, "No credentials found in environment");
|
||||
Err(CredentialError::NotFound(format!(
|
||||
"No credentials found with prefix {}",
|
||||
prefix
|
||||
)))
|
||||
}
|
||||
|
||||
fn load_from_file(&self, path: &PathBuf) -> Result<Credential, CredentialError> {
|
||||
let content =
|
||||
std::fs::read_to_string(path).map_err(|e| CredentialError::FileRead(e.to_string()))?;
|
||||
|
||||
if path.extension().map(|e| e == "json").unwrap_or(false) {
|
||||
serde_json::from_str(&content).map_err(|e| CredentialError::Parse(e.to_string()))
|
||||
} else {
|
||||
toml::from_str(&content).map_err(|e| CredentialError::Parse(e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.cache.clear();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "source")]
|
||||
pub enum CredentialConfig {
|
||||
Environment { prefix: String },
|
||||
File { path: PathBuf },
|
||||
Inline(Credential),
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CredentialError {
|
||||
#[error("Credential not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
#[error("Failed to read credential file: {0}")]
|
||||
FileRead(String),
|
||||
|
||||
#[error("Failed to parse credential: {0}")]
|
||||
Parse(String),
|
||||
}
|
||||
|
||||
impl Default for CredentialStore {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_credential_debug_redacted() {
|
||||
let cred = Credential::Basic {
|
||||
username: "user".to_string(),
|
||||
password: "secret123".to_string(),
|
||||
};
|
||||
|
||||
let debug_output = format!("{:?}", cred);
|
||||
assert!(debug_output.contains("user"));
|
||||
assert!(!debug_output.contains("secret123"));
|
||||
assert!(debug_output.contains("[REDACTED]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_aws_credential_debug_redacted() {
|
||||
let cred = Credential::AwsKey {
|
||||
access_key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
|
||||
secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
|
||||
session_token: None,
|
||||
region: "us-east-1".to_string(),
|
||||
};
|
||||
|
||||
let debug_output = format!("{:?}", cred);
|
||||
assert!(debug_output.contains("AKIA..."));
|
||||
assert!(!debug_output.contains("wJalrXUtnFEMI"));
|
||||
assert!(debug_output.contains("[REDACTED]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_credential_store_debug() {
|
||||
let mut store = CredentialStore::new();
|
||||
store.cache.insert(
|
||||
"test".to_string(),
|
||||
Credential::Basic {
|
||||
username: "user".to_string(),
|
||||
password: "secret".to_string(),
|
||||
},
|
||||
);
|
||||
|
||||
let debug_output = format!("{:?}", store);
|
||||
assert!(debug_output.contains("test"));
|
||||
assert!(!debug_output.contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_from_env() {
|
||||
std::env::set_var("TEST_ORIGIN_USERNAME", "testuser");
|
||||
std::env::set_var("TEST_ORIGIN_PASSWORD", "testpass");
|
||||
|
||||
let store = CredentialStore::new();
|
||||
let cred = store.load_from_env("TEST_ORIGIN").unwrap();
|
||||
|
||||
match cred {
|
||||
Credential::Basic { username, password } => {
|
||||
assert_eq!(username, "testuser");
|
||||
assert_eq!(password, "testpass");
|
||||
}
|
||||
_ => panic!("Expected Basic credential"),
|
||||
}
|
||||
|
||||
std::env::remove_var("TEST_ORIGIN_USERNAME");
|
||||
std::env::remove_var("TEST_ORIGIN_PASSWORD");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum Error {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("Origin not found: {0}")]
|
||||
OriginNotFound(String),
|
||||
|
||||
#[error("File not found: {0}")]
|
||||
FileNotFound(String),
|
||||
|
||||
#[error("Path resolution failed: {0}")]
|
||||
PathResolution(String),
|
||||
|
||||
#[error("Cache error: {0}")]
|
||||
Cache(String),
|
||||
|
||||
#[error("Metadata extraction error: {0}")]
|
||||
Metadata(String),
|
||||
|
||||
#[error("Database error: {0}")]
|
||||
Database(String),
|
||||
|
||||
#[error("Database corrupted: {0}")]
|
||||
DatabaseCorrupted(String),
|
||||
|
||||
#[error("NFS stale file handle")]
|
||||
NfsStaleHandle,
|
||||
|
||||
#[error("Operation not permitted (read-only filesystem)")]
|
||||
ReadOnly,
|
||||
|
||||
#[error("No origin available to serve request")]
|
||||
NoOriginAvailable,
|
||||
|
||||
#[error("Maximum retries exceeded")]
|
||||
MaxRetriesExceeded,
|
||||
|
||||
#[error("Origin error: {0}")]
|
||||
Origin(String),
|
||||
|
||||
#[error("S3 error: {0}")]
|
||||
S3(String),
|
||||
|
||||
#[error("SFTP error: {0}")]
|
||||
Sftp(String),
|
||||
|
||||
#[error("Operation timed out: {0}")]
|
||||
Timeout(String),
|
||||
|
||||
#[error("Credential error: {0}")]
|
||||
Credential(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl Error {
|
||||
pub fn is_not_found(&self) -> bool {
|
||||
matches!(self, Error::FileNotFound(_))
|
||||
}
|
||||
|
||||
pub fn downcast_io(&self) -> Option<&std::io::Error> {
|
||||
match self {
|
||||
Error::Io(e) => Some(e),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
use crate::types::{FileId, OriginId, VirtualPath};
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, trace};
|
||||
|
||||
pub struct EventBus {
|
||||
sender: broadcast::Sender<Event>,
|
||||
}
|
||||
|
||||
impl EventBus {
|
||||
pub fn new(capacity: usize) -> Self {
|
||||
let (sender, _) = broadcast::channel(capacity);
|
||||
Self { sender }
|
||||
}
|
||||
|
||||
pub fn publish(&self, event: Event) {
|
||||
trace!(event = ?event, "Publishing event");
|
||||
let receiver_count = self.sender.receiver_count();
|
||||
if self.sender.send(event).is_err() && receiver_count > 0 {
|
||||
debug!(
|
||||
receiver_count = receiver_count,
|
||||
"Event dropped, no active receivers"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<Event> {
|
||||
self.sender.subscribe()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EventBus {
|
||||
fn default() -> Self {
|
||||
Self::new(1024)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Event {
|
||||
FileAdded {
|
||||
path: VirtualPath,
|
||||
origin_id: OriginId,
|
||||
},
|
||||
FileRemoved {
|
||||
path: VirtualPath,
|
||||
file_id: Option<FileId>,
|
||||
},
|
||||
FileModified {
|
||||
path: VirtualPath,
|
||||
},
|
||||
FileAccessed {
|
||||
file_id: FileId,
|
||||
path: VirtualPath,
|
||||
origin_id: OriginId,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
},
|
||||
OriginConnected {
|
||||
origin_id: OriginId,
|
||||
},
|
||||
OriginDisconnected {
|
||||
origin_id: OriginId,
|
||||
},
|
||||
SyncStarted {
|
||||
origin_id: OriginId,
|
||||
},
|
||||
SyncCompleted {
|
||||
origin_id: OriginId,
|
||||
files_changed: u64,
|
||||
},
|
||||
CacheEviction {
|
||||
bytes_freed: u64,
|
||||
},
|
||||
AllOriginsUnhealthy {
|
||||
candidate_count: usize,
|
||||
},
|
||||
OriginHealthChanged {
|
||||
origin_id: OriginId,
|
||||
healthy: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_event_bus() {
|
||||
let bus = EventBus::new(16);
|
||||
let mut rx = bus.subscribe();
|
||||
|
||||
bus.publish(Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
});
|
||||
|
||||
let event = rx.recv().await.unwrap();
|
||||
assert!(matches!(event, Event::SyncStarted { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_event_bus_multiple_subscribers() {
|
||||
let bus = EventBus::new(16);
|
||||
let mut rx1 = bus.subscribe();
|
||||
let mut rx2 = bus.subscribe();
|
||||
|
||||
bus.publish(Event::CacheEviction { bytes_freed: 1024 });
|
||||
|
||||
let e1 = rx1.recv().await.unwrap();
|
||||
let e2 = rx2.recv().await.unwrap();
|
||||
|
||||
assert!(matches!(e1, Event::CacheEviction { bytes_freed: 1024 }));
|
||||
assert!(matches!(e2, Event::CacheEviction { bytes_freed: 1024 }));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
pub mod config;
|
||||
pub mod credentials;
|
||||
pub mod error;
|
||||
pub mod events;
|
||||
pub mod metrics;
|
||||
pub mod resolver;
|
||||
pub mod supervisor;
|
||||
pub mod types;
|
||||
|
||||
pub use config::{
|
||||
CacheConfig, Config, ConfigError, HealthConfig, LoggingConfig, OriginConfig, OriginType,
|
||||
};
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub fn sanitize_path(path: &Path) -> String {
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
path.to_string_lossy().replace(&home, "~")
|
||||
} else {
|
||||
path.to_string_lossy().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Install a custom panic hook that logs panics via tracing before the default behavior.
|
||||
/// This ensures panics are captured in log files and journald.
|
||||
pub fn install_panic_hook() {
|
||||
let default_hook = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(move |info| {
|
||||
let thread = std::thread::current();
|
||||
let thread_name = thread.name().unwrap_or("<unnamed>");
|
||||
|
||||
let message = if let Some(s) = info.payload().downcast_ref::<&str>() {
|
||||
(*s).to_string()
|
||||
} else if let Some(s) = info.payload().downcast_ref::<String>() {
|
||||
s.clone()
|
||||
} else {
|
||||
"unknown panic".to_string()
|
||||
};
|
||||
|
||||
let location = info
|
||||
.location()
|
||||
.map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
|
||||
.unwrap_or_else(|| "unknown location".to_string());
|
||||
|
||||
tracing::error!(
|
||||
thread = thread_name,
|
||||
location = %location,
|
||||
"PANIC: {}",
|
||||
message
|
||||
);
|
||||
|
||||
default_hook(info);
|
||||
}));
|
||||
}
|
||||
pub use credentials::{Credential, CredentialConfig, CredentialError, CredentialStore};
|
||||
pub use error::{Error, Result};
|
||||
pub use events::{Event, EventBus};
|
||||
pub use metrics::{CacheMetrics, FuseOpsMetrics, Metrics, OriginsMetrics};
|
||||
pub use resolver::{PathResolver, PathTemplate};
|
||||
pub use types::*;
|
||||
@@ -0,0 +1,322 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Metrics {
|
||||
pub fuse_ops: FuseOpsMetrics,
|
||||
pub fuse_latency: FuseLatencyMetrics,
|
||||
pub cache: CacheMetrics,
|
||||
pub origins: OriginsMetrics,
|
||||
pub origin_health: OriginHealthMetrics,
|
||||
start_time: Option<Instant>,
|
||||
}
|
||||
|
||||
impl Metrics {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
start_time: Some(Instant::now()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn uptime_secs(&self) -> u64 {
|
||||
self.start_time.map(|t| t.elapsed().as_secs()).unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn to_prometheus(&self) -> String {
|
||||
let mut output = String::new();
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_fuse_ops_total Total FUSE operations\n\
|
||||
# TYPE musicfs_fuse_ops_total counter\n\
|
||||
musicfs_fuse_ops_total{{op=\"lookup\"}} {}\n\
|
||||
musicfs_fuse_ops_total{{op=\"getattr\"}} {}\n\
|
||||
musicfs_fuse_ops_total{{op=\"read\"}} {}\n\
|
||||
musicfs_fuse_ops_total{{op=\"readdir\"}} {}\n\
|
||||
musicfs_fuse_ops_total{{op=\"open\"}} {}\n",
|
||||
self.fuse_ops.lookup.load(Ordering::Relaxed),
|
||||
self.fuse_ops.getattr.load(Ordering::Relaxed),
|
||||
self.fuse_ops.read.load(Ordering::Relaxed),
|
||||
self.fuse_ops.readdir.load(Ordering::Relaxed),
|
||||
self.fuse_ops.open.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
for (op, histogram) in self.fuse_latency.histograms.read().iter() {
|
||||
let quantiles = histogram.quantiles();
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_fuse_latency_seconds FUSE operation latency\n\
|
||||
# TYPE musicfs_fuse_latency_seconds summary\n\
|
||||
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.5\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.95\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.99\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds_sum{{op=\"{}\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds_count{{op=\"{}\"}} {}\n",
|
||||
op,
|
||||
quantiles.p50,
|
||||
op,
|
||||
quantiles.p95,
|
||||
op,
|
||||
quantiles.p99,
|
||||
op,
|
||||
histogram.sum_secs(),
|
||||
op,
|
||||
histogram.count(),
|
||||
));
|
||||
}
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_cache_hits_total Cache hits\n\
|
||||
# TYPE musicfs_cache_hits_total counter\n\
|
||||
musicfs_cache_hits_total {}\n",
|
||||
self.cache.hits.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_cache_misses_total Cache misses\n\
|
||||
# TYPE musicfs_cache_misses_total counter\n\
|
||||
musicfs_cache_misses_total {}\n",
|
||||
self.cache.misses.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_cache_size_bytes Current cache size in bytes\n\
|
||||
# TYPE musicfs_cache_size_bytes gauge\n\
|
||||
musicfs_cache_size_bytes {}\n",
|
||||
self.cache.size_bytes.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_cache_chunks_total Number of cached chunks\n\
|
||||
# TYPE musicfs_cache_chunks_total gauge\n\
|
||||
musicfs_cache_chunks_total {}\n",
|
||||
self.cache.chunk_count.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
output.push_str(
|
||||
"# HELP musicfs_origin_health Origin health status (1=healthy, 0=unhealthy)\n\
|
||||
# TYPE musicfs_origin_health gauge\n",
|
||||
);
|
||||
for (origin_id, healthy) in self.origin_health.status.read().iter() {
|
||||
output.push_str(&format!(
|
||||
"musicfs_origin_health{{origin=\"{}\"}} {}\n",
|
||||
origin_id,
|
||||
if *healthy { 1 } else { 0 }
|
||||
));
|
||||
}
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_uptime_seconds Daemon uptime in seconds\n\
|
||||
# TYPE musicfs_uptime_seconds gauge\n\
|
||||
musicfs_uptime_seconds {}\n",
|
||||
self.uptime_secs(),
|
||||
));
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
pub fn hit_ratio(&self) -> f64 {
|
||||
let hits = self.cache.hits.load(Ordering::Relaxed) as f64;
|
||||
let misses = self.cache.misses.load(Ordering::Relaxed) as f64;
|
||||
let total = hits + misses;
|
||||
|
||||
if total == 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
hits / total
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct FuseOpsMetrics {
|
||||
pub lookup: AtomicU64,
|
||||
pub getattr: AtomicU64,
|
||||
pub read: AtomicU64,
|
||||
pub readdir: AtomicU64,
|
||||
pub open: AtomicU64,
|
||||
}
|
||||
|
||||
impl FuseOpsMetrics {
|
||||
pub fn record_lookup(&self) {
|
||||
self.lookup.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_getattr(&self) {
|
||||
self.getattr.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_read(&self) {
|
||||
self.read.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_readdir(&self) {
|
||||
self.readdir.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_open(&self) {
|
||||
self.open.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CacheMetrics {
|
||||
pub hits: AtomicU64,
|
||||
pub misses: AtomicU64,
|
||||
pub size_bytes: AtomicU64,
|
||||
pub chunk_count: AtomicU64,
|
||||
}
|
||||
|
||||
impl CacheMetrics {
|
||||
pub fn record_hit(&self) {
|
||||
self.hits.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_miss(&self) {
|
||||
self.misses.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn update_size(&self, size: u64) {
|
||||
self.size_bytes.store(size, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn update_chunk_count(&self, count: u64) {
|
||||
self.chunk_count.store(count, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OriginsMetrics {
|
||||
pub healthy_count: AtomicU64,
|
||||
pub total_count: AtomicU64,
|
||||
}
|
||||
|
||||
impl OriginsMetrics {
|
||||
pub fn update(&self, healthy: u64, total: u64) {
|
||||
self.healthy_count.store(healthy, Ordering::Relaxed);
|
||||
self.total_count.store(total, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct FuseLatencyMetrics {
|
||||
pub histograms: RwLock<HashMap<String, LatencyHistogram>>,
|
||||
}
|
||||
|
||||
impl FuseLatencyMetrics {
|
||||
pub fn record(&self, op: &str, latency_secs: f64) {
|
||||
let mut histograms = self.histograms.write();
|
||||
histograms
|
||||
.entry(op.to_string())
|
||||
.or_default()
|
||||
.record(latency_secs);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct LatencyHistogram {
|
||||
samples: Vec<f64>,
|
||||
sum: f64,
|
||||
}
|
||||
|
||||
impl LatencyHistogram {
|
||||
pub fn record(&mut self, latency_secs: f64) {
|
||||
self.samples.push(latency_secs);
|
||||
self.sum += latency_secs;
|
||||
|
||||
if self.samples.len() > 10000 {
|
||||
self.samples.drain(..5000);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn quantiles(&self) -> Quantiles {
|
||||
if self.samples.is_empty() {
|
||||
return Quantiles::default();
|
||||
}
|
||||
|
||||
let mut sorted = self.samples.clone();
|
||||
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
let len = sorted.len();
|
||||
Quantiles {
|
||||
p50: sorted[len / 2],
|
||||
p95: sorted[(len as f64 * 0.95) as usize],
|
||||
p99: sorted[(len as f64 * 0.99) as usize],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sum_secs(&self) -> f64 {
|
||||
self.sum
|
||||
}
|
||||
|
||||
pub fn count(&self) -> u64 {
|
||||
self.samples.len() as u64
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Quantiles {
|
||||
pub p50: f64,
|
||||
pub p95: f64,
|
||||
pub p99: f64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OriginHealthMetrics {
|
||||
pub status: RwLock<HashMap<String, bool>>,
|
||||
}
|
||||
|
||||
impl OriginHealthMetrics {
|
||||
pub fn set_health(&self, origin_id: &str, healthy: bool) {
|
||||
self.status.write().insert(origin_id.to_string(), healthy);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_metrics_new() {
|
||||
let metrics = Metrics::new();
|
||||
assert!(metrics.uptime_secs() < 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fuse_ops_recording() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.fuse_ops.record_lookup();
|
||||
metrics.fuse_ops.record_lookup();
|
||||
metrics.fuse_ops.record_read();
|
||||
|
||||
assert_eq!(metrics.fuse_ops.lookup.load(Ordering::Relaxed), 2);
|
||||
assert_eq!(metrics.fuse_ops.read.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit_ratio() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.cache.hits.store(8, Ordering::Relaxed);
|
||||
metrics.cache.misses.store(2, Ordering::Relaxed);
|
||||
|
||||
assert!((metrics.hit_ratio() - 0.8).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit_ratio_zero() {
|
||||
let metrics = Metrics::new();
|
||||
assert_eq!(metrics.hit_ratio(), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_prometheus_format() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.fuse_ops.record_lookup();
|
||||
metrics.cache.record_hit();
|
||||
|
||||
let output = metrics.to_prometheus();
|
||||
assert!(output.contains("musicfs_fuse_ops_total"));
|
||||
assert!(output.contains("musicfs_cache_hits_total"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
use crate::{AudioMeta, VirtualPath};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PathTemplate {
|
||||
pub pattern: String,
|
||||
pub fallback_artist: String,
|
||||
pub fallback_album: String,
|
||||
pub fallback_title: String,
|
||||
pub fallback_year: String,
|
||||
}
|
||||
|
||||
impl Default for PathTemplate {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pattern: "$artist/$album ($year) [$format_upper]/$track - $title.$format".to_string(),
|
||||
fallback_artist: "Unknown Artist".to_string(),
|
||||
fallback_album: "Unknown Album".to_string(),
|
||||
fallback_title: "Unknown Track".to_string(),
|
||||
fallback_year: "Unknown".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PathResolver {
|
||||
template: PathTemplate,
|
||||
}
|
||||
|
||||
impl PathResolver {
|
||||
pub fn new(template: PathTemplate) -> Self {
|
||||
Self { template }
|
||||
}
|
||||
|
||||
pub fn resolve(&self, meta: &AudioMeta, extension: &str) -> VirtualPath {
|
||||
let artist = meta
|
||||
.artist
|
||||
.as_deref()
|
||||
.unwrap_or(&self.template.fallback_artist);
|
||||
let album = meta
|
||||
.album
|
||||
.as_deref()
|
||||
.unwrap_or(&self.template.fallback_album);
|
||||
let title = meta
|
||||
.title
|
||||
.as_deref()
|
||||
.unwrap_or(&self.template.fallback_title);
|
||||
let year = meta
|
||||
.year
|
||||
.map(|y| y.to_string())
|
||||
.unwrap_or_else(|| self.template.fallback_year.clone());
|
||||
let track = meta.track.unwrap_or(0);
|
||||
let disc = meta.disc.unwrap_or(1);
|
||||
let genre = meta.genre.as_deref().unwrap_or("Unknown");
|
||||
let format = extension.to_lowercase();
|
||||
let format_upper = extension.to_uppercase();
|
||||
|
||||
let artist = sanitize_path_component(artist);
|
||||
let album = sanitize_path_component(album);
|
||||
let title = sanitize_path_component(title);
|
||||
let genre = sanitize_path_component(genre);
|
||||
|
||||
let path = self
|
||||
.template
|
||||
.pattern
|
||||
.replace("$artist", &artist)
|
||||
.replace("$album", &album)
|
||||
.replace("$title", &title)
|
||||
.replace("$track", &format!("{:02}", track))
|
||||
.replace("$disc", &disc.to_string())
|
||||
.replace("$year", &year)
|
||||
.replace("$genre", &genre)
|
||||
.replace("$format_upper", &format_upper)
|
||||
.replace("$format", &format);
|
||||
|
||||
VirtualPath::new(path)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PathResolver {
|
||||
fn default() -> Self {
|
||||
Self::new(PathTemplate::default())
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_path_component(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| match c {
|
||||
'/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' | '\0' => '_',
|
||||
c => c,
|
||||
})
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::AudioFormat;
|
||||
|
||||
#[test]
|
||||
fn test_resolve_complete_metadata() {
|
||||
let resolver = PathResolver::default();
|
||||
let meta = AudioMeta {
|
||||
artist: Some("Metallica".to_string()),
|
||||
album: Some("Master of Puppets".to_string()),
|
||||
title: Some("Battery".to_string()),
|
||||
track: Some(1),
|
||||
year: Some(1986),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let path = resolver.resolve(&meta, "flac");
|
||||
assert_eq!(
|
||||
path.as_str(),
|
||||
"Metallica/Master of Puppets (1986) [FLAC]/01 - Battery.flac"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_missing_album() {
|
||||
let resolver = PathResolver::default();
|
||||
let meta = AudioMeta {
|
||||
artist: Some("Artist".to_string()),
|
||||
title: Some("Track".to_string()),
|
||||
track: Some(5),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let path = resolver.resolve(&meta, "mp3");
|
||||
assert_eq!(
|
||||
path.as_str(),
|
||||
"Artist/Unknown Album (Unknown) [MP3]/05 - Track.mp3"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_special_chars() {
|
||||
let resolver = PathResolver::default();
|
||||
let meta = AudioMeta {
|
||||
artist: Some("AC/DC".to_string()),
|
||||
album: Some("Who Made Who?".to_string()),
|
||||
title: Some("Test:Track".to_string()),
|
||||
track: Some(1),
|
||||
year: Some(1986),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let path = resolver.resolve(&meta, "flac");
|
||||
assert!(!path.as_str().contains(':'));
|
||||
assert!(!path.as_str().contains('?'));
|
||||
assert!(path.as_str().contains("AC_DC"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_template() {
|
||||
let template = PathTemplate {
|
||||
pattern: "$genre/$artist - $album/$track $title.$format".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let resolver = PathResolver::new(template);
|
||||
let meta = AudioMeta {
|
||||
artist: Some("Artist".to_string()),
|
||||
album: Some("Album".to_string()),
|
||||
title: Some("Song".to_string()),
|
||||
genre: Some("Rock".to_string()),
|
||||
track: Some(3),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let path = resolver.resolve(&meta, "flac");
|
||||
assert_eq!(path.as_str(), "Rock/Artist - Album/03 Song.flac");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{error, warn};
|
||||
|
||||
pub struct TaskSupervisor {
|
||||
tasks: Arc<RwLock<HashMap<String, TaskEntry>>>,
|
||||
}
|
||||
|
||||
struct TaskEntry {
|
||||
handle: JoinHandle<()>,
|
||||
status: TaskStatus,
|
||||
restart_count: u32,
|
||||
last_restart: Option<Instant>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum TaskStatus {
|
||||
Running,
|
||||
Failed { error: String, at: Instant },
|
||||
Restarting { attempt: u32 },
|
||||
Stopped,
|
||||
}
|
||||
|
||||
impl Default for TaskSupervisor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl TaskSupervisor {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tasks: Arc::new(RwLock::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn spawn_supervised<F>(&self, name: &str, future: F)
|
||||
where
|
||||
F: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let name_owned = name.to_string();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
future.await;
|
||||
});
|
||||
|
||||
self.tasks.write().insert(
|
||||
name_owned,
|
||||
TaskEntry {
|
||||
handle,
|
||||
status: TaskStatus::Running,
|
||||
restart_count: 0,
|
||||
last_restart: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn spawn_critical<F, Fut>(&self, name: &str, factory: F)
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync + 'static,
|
||||
Fut: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let tasks = self.tasks.clone();
|
||||
let name_owned = name.to_string();
|
||||
|
||||
let monitor_handle = tokio::spawn(async move {
|
||||
let mut restart_count = 0u32;
|
||||
let max_restarts = 5u32;
|
||||
let backoff_durations = [
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(30),
|
||||
];
|
||||
|
||||
loop {
|
||||
let handle = tokio::spawn(factory());
|
||||
|
||||
{
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Running;
|
||||
}
|
||||
}
|
||||
|
||||
match handle.await {
|
||||
Ok(()) => {
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Stopped;
|
||||
}
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
restart_count += 1;
|
||||
|
||||
if restart_count > max_restarts {
|
||||
error!(task = %name_owned, "Task exceeded max restarts ({}), giving up", max_restarts);
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: format!("Exceeded max restarts: {}", e),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
let backoff_idx =
|
||||
(restart_count as usize - 1).min(backoff_durations.len() - 1);
|
||||
let backoff = backoff_durations[backoff_idx];
|
||||
|
||||
warn!(
|
||||
task = %name_owned,
|
||||
error = %e,
|
||||
attempt = restart_count,
|
||||
backoff_ms = backoff.as_millis() as u64,
|
||||
"Critical task failed, restarting with backoff"
|
||||
);
|
||||
|
||||
{
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Restarting {
|
||||
attempt: restart_count,
|
||||
};
|
||||
entry.restart_count = restart_count;
|
||||
entry.last_restart = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::sleep(backoff).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
self.tasks.write().insert(
|
||||
name.to_string(),
|
||||
TaskEntry {
|
||||
handle: monitor_handle,
|
||||
status: TaskStatus::Running,
|
||||
restart_count: 0,
|
||||
last_restart: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn task_status(&self, name: &str) -> TaskStatus {
|
||||
let mut tasks = self.tasks.write();
|
||||
if let Some(entry) = tasks.get_mut(name) {
|
||||
if entry.handle.is_finished() {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: "Task exited".into(),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
entry.status.clone()
|
||||
} else {
|
||||
TaskStatus::Stopped
|
||||
}
|
||||
}
|
||||
|
||||
pub fn check_all(&self) -> Vec<(String, TaskStatus)> {
|
||||
let mut tasks = self.tasks.write();
|
||||
tasks
|
||||
.iter_mut()
|
||||
.map(|(name, entry)| {
|
||||
if entry.handle.is_finished() {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: "Task exited".into(),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
(name.clone(), entry.status.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::PathBuf;
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct OriginId(pub String);
|
||||
|
||||
impl From<&str> for OriginId {
|
||||
fn from(s: &str) -> Self {
|
||||
Self(s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for OriginId {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct FileId(pub i64);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct VirtualPath(pub PathBuf);
|
||||
|
||||
impl VirtualPath {
|
||||
pub fn new(path: impl Into<PathBuf>) -> Self {
|
||||
Self(path.into())
|
||||
}
|
||||
|
||||
pub fn as_path(&self) -> &std::path::Path {
|
||||
&self.0
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
self.0.to_str().unwrap_or("")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RealPath {
|
||||
pub origin_id: OriginId,
|
||||
pub path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ContentHash(pub [u8; 8]);
|
||||
|
||||
impl ContentHash {
|
||||
pub fn from_bytes(data: &[u8]) -> Self {
|
||||
use xxhash_rust::xxh64::xxh64;
|
||||
Self(xxh64(data, 0).to_le_bytes())
|
||||
}
|
||||
|
||||
pub fn to_hex(&self) -> String {
|
||||
hex::encode(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ChunkHash(pub [u8; 8]);
|
||||
|
||||
impl ChunkHash {
|
||||
pub fn from_bytes(data: &[u8]) -> Self {
|
||||
use xxhash_rust::xxh64::xxh64;
|
||||
Self(xxh64(data, 0).to_le_bytes())
|
||||
}
|
||||
|
||||
pub fn as_hex(&self) -> String {
|
||||
hex::encode(self.0)
|
||||
}
|
||||
|
||||
pub fn to_hex(&self) -> String {
|
||||
self.as_hex()
|
||||
}
|
||||
|
||||
pub fn from_hex(s: &str) -> Option<Self> {
|
||||
let bytes = hex::decode(s).ok()?;
|
||||
if bytes.len() != 8 {
|
||||
return None;
|
||||
}
|
||||
let mut arr = [0u8; 8];
|
||||
arr.copy_from_slice(&bytes);
|
||||
Some(Self(arr))
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ChunkHash {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.as_hex())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub enum AudioFormat {
|
||||
Flac,
|
||||
Mp3,
|
||||
Opus,
|
||||
Vorbis,
|
||||
Aac,
|
||||
Alac,
|
||||
Wav,
|
||||
#[default]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AudioFormat {
|
||||
pub fn from_extension(ext: &str) -> Self {
|
||||
match ext.to_lowercase().as_str() {
|
||||
"flac" => Self::Flac,
|
||||
"mp3" => Self::Mp3,
|
||||
"opus" => Self::Opus,
|
||||
"ogg" => Self::Vorbis,
|
||||
"m4a" | "aac" => Self::Aac,
|
||||
"wav" => Self::Wav,
|
||||
_ => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct AudioMeta {
|
||||
pub title: Option<String>,
|
||||
pub artist: Option<String>,
|
||||
pub album: Option<String>,
|
||||
pub album_artist: Option<String>,
|
||||
pub genre: Option<String>,
|
||||
pub year: Option<u32>,
|
||||
pub track: Option<u32>,
|
||||
pub disc: Option<u32>,
|
||||
pub duration_ms: Option<u64>,
|
||||
pub bitrate: Option<u32>,
|
||||
pub sample_rate: Option<u32>,
|
||||
pub format: AudioFormat,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FileMeta {
|
||||
pub id: FileId,
|
||||
pub virtual_path: VirtualPath,
|
||||
pub real_path: RealPath,
|
||||
pub size: u64,
|
||||
pub mtime: SystemTime,
|
||||
pub content_hash: Option<ContentHash>,
|
||||
pub audio: Option<AudioMeta>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DirEntry {
|
||||
pub name: String,
|
||||
pub is_dir: bool,
|
||||
pub size: u64,
|
||||
pub mtime: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileStat {
|
||||
pub size: u64,
|
||||
pub mtime: SystemTime,
|
||||
pub is_dir: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum HealthStatus {
|
||||
Healthy,
|
||||
Degraded,
|
||||
Unhealthy,
|
||||
#[default]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_content_hash() {
|
||||
let data = b"hello world";
|
||||
let hash1 = ContentHash::from_bytes(data);
|
||||
let hash2 = ContentHash::from_bytes(data);
|
||||
assert_eq!(hash1, hash2);
|
||||
|
||||
let hash3 = ContentHash::from_bytes(b"different");
|
||||
assert_ne!(hash1, hash3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_audio_format_from_extension() {
|
||||
assert_eq!(AudioFormat::from_extension("flac"), AudioFormat::Flac);
|
||||
assert_eq!(AudioFormat::from_extension("MP3"), AudioFormat::Mp3);
|
||||
assert_eq!(AudioFormat::from_extension("unknown"), AudioFormat::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_virtual_path() {
|
||||
let path = VirtualPath::new("/Artist/Album/Track.flac");
|
||||
assert_eq!(path.as_str(), "/Artist/Album/Track.flac");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "musicfs-fuse"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-cache = { path = "../musicfs-cache" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-search = { path = "../musicfs-search" }
|
||||
fuser.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
moka.workspace = true
|
||||
parking_lot.workspace = true
|
||||
libc = "0.2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -0,0 +1,613 @@
|
||||
use crate::ops::SearchOps;
|
||||
use fuser::{
|
||||
FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, ReplyOpen,
|
||||
Request,
|
||||
};
|
||||
use musicfs_cache::{VirtualNode, VirtualTree, ROOT_INODE};
|
||||
use musicfs_cas::FileReader;
|
||||
use musicfs_core::Result;
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::OsStr;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::runtime::Handle;
|
||||
use tracing::{debug, info, instrument, trace, warn};
|
||||
|
||||
const TTL: Duration = Duration::from_secs(1);
|
||||
const BLOCK_SIZE: u32 = 512;
|
||||
const SEARCH_QUERY_INODE_BASE: u64 = 0xFFFF_FFFF_0000_0100;
|
||||
|
||||
pub struct MusicFs {
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
reader: Option<Arc<FileReader>>,
|
||||
runtime_handle: Handle,
|
||||
search_ops: Option<SearchOps>,
|
||||
query_inodes: RwLock<HashMap<String, u64>>,
|
||||
inode_queries: RwLock<HashMap<u64, String>>,
|
||||
next_query_inode: RwLock<u64>,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
}
|
||||
|
||||
impl MusicFs {
|
||||
pub fn new(tree: Arc<RwLock<VirtualTree>>, runtime_handle: Handle) -> Self {
|
||||
Self {
|
||||
tree,
|
||||
reader: None,
|
||||
runtime_handle,
|
||||
search_ops: None,
|
||||
query_inodes: RwLock::new(HashMap::new()),
|
||||
inode_queries: RwLock::new(HashMap::new()),
|
||||
next_query_inode: RwLock::new(SEARCH_QUERY_INODE_BASE),
|
||||
uid: unsafe { libc::getuid() },
|
||||
gid: unsafe { libc::getgid() },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_reader(
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
reader: Arc<FileReader>,
|
||||
runtime_handle: Handle,
|
||||
) -> Self {
|
||||
Self {
|
||||
tree,
|
||||
reader: Some(reader),
|
||||
runtime_handle,
|
||||
search_ops: None,
|
||||
query_inodes: RwLock::new(HashMap::new()),
|
||||
inode_queries: RwLock::new(HashMap::new()),
|
||||
next_query_inode: RwLock::new(SEARCH_QUERY_INODE_BASE),
|
||||
uid: unsafe { libc::getuid() },
|
||||
gid: unsafe { libc::getgid() },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_search(mut self, search_ops: SearchOps) -> Self {
|
||||
self.search_ops = Some(search_ops);
|
||||
self
|
||||
}
|
||||
|
||||
fn get_or_create_query_inode(&self, query: &str) -> u64 {
|
||||
let query_inodes = self.query_inodes.read();
|
||||
if let Some(&inode) = query_inodes.get(query) {
|
||||
return inode;
|
||||
}
|
||||
drop(query_inodes);
|
||||
|
||||
let mut query_inodes = self.query_inodes.write();
|
||||
let mut inode_queries = self.inode_queries.write();
|
||||
let mut next_inode = self.next_query_inode.write();
|
||||
|
||||
if let Some(&inode) = query_inodes.get(query) {
|
||||
return inode;
|
||||
}
|
||||
|
||||
let inode = *next_inode;
|
||||
*next_inode += 1;
|
||||
query_inodes.insert(query.to_string(), inode);
|
||||
inode_queries.insert(inode, query.to_string());
|
||||
inode
|
||||
}
|
||||
|
||||
fn get_query_for_inode(&self, inode: u64) -> Option<String> {
|
||||
self.inode_queries.read().get(&inode).cloned()
|
||||
}
|
||||
|
||||
pub fn mount(self, mountpoint: &Path) -> Result<()> {
|
||||
info!("Mounting MusicFS at {:?}", mountpoint);
|
||||
|
||||
let options = vec![
|
||||
fuser::MountOption::RO,
|
||||
fuser::MountOption::FSName("musicfs".to_string()),
|
||||
fuser::MountOption::AutoUnmount,
|
||||
fuser::MountOption::AllowOther,
|
||||
];
|
||||
|
||||
fuser::mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn spawn_mount(self, mountpoint: &Path) -> Result<fuser::BackgroundSession> {
|
||||
info!("Mounting MusicFS at {:?}", mountpoint);
|
||||
|
||||
let options = vec![
|
||||
fuser::MountOption::RO,
|
||||
fuser::MountOption::FSName("musicfs".to_string()),
|
||||
fuser::MountOption::AutoUnmount,
|
||||
fuser::MountOption::AllowOther,
|
||||
];
|
||||
|
||||
let session =
|
||||
fuser::spawn_mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
|
||||
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
fn node_to_attr(&self, node: &VirtualNode) -> FileAttr {
|
||||
match node {
|
||||
VirtualNode::Directory(dir) => FileAttr {
|
||||
ino: dir.inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: dir.mtime,
|
||||
mtime: dir.mtime,
|
||||
ctime: dir.mtime,
|
||||
crtime: dir.mtime,
|
||||
kind: FileType::Directory,
|
||||
perm: 0o755,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: BLOCK_SIZE,
|
||||
flags: 0,
|
||||
},
|
||||
VirtualNode::File(file) => FileAttr {
|
||||
ino: file.inode,
|
||||
size: file.size,
|
||||
blocks: (file.size + BLOCK_SIZE as u64 - 1) / BLOCK_SIZE as u64,
|
||||
atime: file.mtime,
|
||||
mtime: file.mtime,
|
||||
ctime: file.mtime,
|
||||
crtime: file.mtime,
|
||||
kind: FileType::RegularFile,
|
||||
perm: 0o644,
|
||||
nlink: 1,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: BLOCK_SIZE,
|
||||
flags: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Filesystem for MusicFs {
|
||||
fn init(
|
||||
&mut self,
|
||||
_req: &Request<'_>,
|
||||
_config: &mut fuser::KernelConfig,
|
||||
) -> std::result::Result<(), libc::c_int> {
|
||||
info!("MusicFS initialized");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn destroy(&mut self) {
|
||||
info!("MusicFS destroyed");
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) {
|
||||
let name_str = name.to_string_lossy();
|
||||
|
||||
if parent == ROOT_INODE && SearchOps::is_search_dir_name(&name_str) {
|
||||
trace!(parent, name = %name_str, "search_dir_name matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.lookup_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if parent == SearchOps::search_dir_inode() {
|
||||
trace!(parent, name = %name_str, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
let inode = self.get_or_create_query_inode(&name_str);
|
||||
search_ops.lookup_query_dir(&name_str, inode, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(query) = self.get_query_for_inode(parent) {
|
||||
trace!(parent, name = %name_str, query = %query, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
let inode = self.get_or_create_query_inode(&format!("{}:{}", query, name_str));
|
||||
search_ops.lookup_result(inode, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(inode) = tree.lookup(parent, name) {
|
||||
trace!(parent, name = %name_str, ino = inode, "file found in tree");
|
||||
if let Some(node) = tree.get(inode) {
|
||||
let attr = self.node_to_attr(node);
|
||||
reply.entry(&TTL, &attr, 0);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
trace!(parent, name = %name_str, "file not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) {
|
||||
if ino == SearchOps::search_dir_inode() {
|
||||
trace!(ino, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if SearchOps::is_search_inode(ino) {
|
||||
trace!(ino, "search_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_result(ino, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if self.get_query_for_inode(ino).is_some() {
|
||||
trace!(ino, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(node) = tree.get(ino) {
|
||||
trace!(ino, "inode found in tree");
|
||||
let attr = self.node_to_attr(node);
|
||||
reply.attr(&TTL, &attr);
|
||||
} else {
|
||||
trace!(ino, "inode not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn readdir(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
offset: i64,
|
||||
mut reply: ReplyDirectory,
|
||||
) {
|
||||
if ino == SearchOps::search_dir_inode() {
|
||||
trace!(ino, offset, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readdir_search_root(offset, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(query) = self.get_query_for_inode(ino) {
|
||||
trace!(ino, offset, query = %query, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readdir_query(&query, offset, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(children) = tree.readdir(ino) {
|
||||
trace!(
|
||||
ino,
|
||||
offset,
|
||||
children_count = children.len(),
|
||||
"directory found"
|
||||
);
|
||||
let parent_ino = tree.get_parent(ino).unwrap_or(ROOT_INODE);
|
||||
|
||||
let entries: Vec<(u64, FileType, &str)> = vec![
|
||||
(ino, FileType::Directory, "."),
|
||||
(parent_ino, FileType::Directory, ".."),
|
||||
];
|
||||
|
||||
let child_entries: Vec<(u64, FileType, String)> = children
|
||||
.iter()
|
||||
.map(|(name, child_ino, is_dir)| {
|
||||
let kind = if *is_dir {
|
||||
FileType::Directory
|
||||
} else {
|
||||
FileType::RegularFile
|
||||
};
|
||||
(*child_ino, kind, name.to_string_lossy().to_string())
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, name) {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let base_offset = entries.len();
|
||||
for (i, (inode, kind, name)) in child_entries.iter().enumerate() {
|
||||
let entry_offset = base_offset + i;
|
||||
if entry_offset < offset as usize {
|
||||
continue;
|
||||
}
|
||||
if reply.add(*inode, (entry_offset + 1) as i64, *kind, name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
reply.ok();
|
||||
} else {
|
||||
trace!(ino, offset, "directory not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn open(&mut self, _req: &Request, ino: u64, flags: i32, reply: ReplyOpen) {
|
||||
let write_flags = libc::O_WRONLY | libc::O_RDWR | libc::O_APPEND | libc::O_TRUNC;
|
||||
if flags & write_flags != 0 {
|
||||
trace!(ino, flags, "write flags detected");
|
||||
reply.error(libc::EROFS);
|
||||
return;
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if tree.get(ino).is_some() {
|
||||
trace!(ino, "inode found");
|
||||
reply.opened(0, 0);
|
||||
} else {
|
||||
trace!(ino, "inode not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn read(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
offset: i64,
|
||||
size: u32,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
reply: ReplyData,
|
||||
) {
|
||||
let file_id = {
|
||||
let tree = self.tree.read();
|
||||
if let Some(VirtualNode::File(file)) = tree.get(ino) {
|
||||
trace!(ino, "file found in tree");
|
||||
file.file_id
|
||||
} else {
|
||||
trace!(ino, "file not found");
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let Some(reader) = &self.reader else {
|
||||
trace!(ino, "no reader available");
|
||||
reply.data(&[]);
|
||||
return;
|
||||
};
|
||||
|
||||
let reader = reader.clone();
|
||||
let handle = self.runtime_handle.clone();
|
||||
let result = std::thread::scope(|_| {
|
||||
handle.block_on(async {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
reader.read(file_id, offset as u64, size),
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(Ok(data)) => {
|
||||
trace!(
|
||||
ino,
|
||||
offset,
|
||||
size_bytes = size,
|
||||
bytes_read = data.len(),
|
||||
"read successful"
|
||||
);
|
||||
reply.data(&data);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!(ino, offset, size_bytes = size, error = %e, "read failed");
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
Err(_timeout) => {
|
||||
warn!(ino, offset, size_bytes = size, "read timed out after 30s");
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn release(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
_flush: bool,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
trace!(ino, "releasing file handle");
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn readlink(&mut self, _req: &Request, ino: u64, reply: ReplyData) {
|
||||
debug!("readlink(ino={})", ino);
|
||||
|
||||
if SearchOps::is_search_inode(ino) {
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readlink(ino, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
reply.error(libc::EINVAL);
|
||||
}
|
||||
|
||||
fn write(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_fh: u64,
|
||||
_offset: i64,
|
||||
_data: &[u8],
|
||||
_write_flags: u32,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
reply: fuser::ReplyWrite,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn mkdir(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn unlink(&mut self, _req: &Request, _parent: u64, _name: &OsStr, reply: fuser::ReplyEmpty) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn rmdir(&mut self, _req: &Request, _parent: u64, _name: &OsStr, reply: fuser::ReplyEmpty) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn rename(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_newparent: u64,
|
||||
_newname: &OsStr,
|
||||
_flags: u32,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn create(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
_flags: i32,
|
||||
reply: fuser::ReplyCreate,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn setattr(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_mode: Option<u32>,
|
||||
_uid: Option<u32>,
|
||||
_gid: Option<u32>,
|
||||
_size: Option<u64>,
|
||||
_atime: Option<fuser::TimeOrNow>,
|
||||
_mtime: Option<fuser::TimeOrNow>,
|
||||
_ctime: Option<SystemTime>,
|
||||
_fh: Option<u64>,
|
||||
_crtime: Option<SystemTime>,
|
||||
_chgtime: Option<SystemTime>,
|
||||
_bkuptime: Option<SystemTime>,
|
||||
_flags: Option<u32>,
|
||||
reply: ReplyAttr,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn symlink(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_link: &Path,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn link(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_newparent: u64,
|
||||
_newname: &OsStr,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn mknod(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
_rdev: u32,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from("/test"),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tree_integration() {
|
||||
let runtime = tokio::runtime::Runtime::new().unwrap();
|
||||
let handle = runtime.handle().clone();
|
||||
|
||||
let mut builder = TreeBuilder::new();
|
||||
builder.add_file(&make_file_meta(1, "/Artist/Album/Track.flac", 30_000_000));
|
||||
let tree = Arc::new(RwLock::new(builder.build()));
|
||||
|
||||
let _fs = MusicFs::new(tree.clone(), handle);
|
||||
|
||||
let tree_read = tree.read();
|
||||
assert!(tree_read.get(ROOT_INODE).is_some());
|
||||
assert!(tree_read
|
||||
.get_by_path(&VirtualPath::new("/Artist"))
|
||||
.is_some());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mod filesystem;
|
||||
pub mod ops;
|
||||
|
||||
pub use filesystem::MusicFs;
|
||||
pub use ops::SearchOps;
|
||||
@@ -0,0 +1,5 @@
|
||||
mod prefetch;
|
||||
mod search;
|
||||
|
||||
pub use prefetch::PrefetchOps;
|
||||
pub use search::SearchOps;
|
||||
@@ -0,0 +1,298 @@
|
||||
use fuser::{FileAttr, FileType, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry};
|
||||
use musicfs_cache::{PatternStore, PrefetchConfig, PrefetchEngine};
|
||||
use musicfs_cas::ContentFetcher;
|
||||
use musicfs_core::{EventBus, FileId};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
const PREFETCH_DIR_INODE: u64 = 0xFFFF_FFFF_0000_0002;
|
||||
const PREFETCH_STATUS_INODE: u64 = 0xFFFF_FFFF_0000_0003;
|
||||
const PREFETCH_HINTS_BASE: u64 = 0xFFFF_FFFF_2000_0000;
|
||||
|
||||
pub struct PrefetchOps {
|
||||
pattern_store: Arc<PatternStore>,
|
||||
engine: Option<Arc<PrefetchEngine>>,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
}
|
||||
|
||||
impl PrefetchOps {
|
||||
pub fn new(pattern_store: Arc<PatternStore>, uid: u32, gid: u32) -> Self {
|
||||
Self {
|
||||
pattern_store,
|
||||
engine: None,
|
||||
uid,
|
||||
gid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_engine(
|
||||
pattern_store: Arc<PatternStore>,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
config: PrefetchConfig,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
) -> Self {
|
||||
let engine = Arc::new(PrefetchEngine::new(config, pattern_store.clone(), fetcher));
|
||||
|
||||
Self {
|
||||
pattern_store,
|
||||
engine: Some(engine),
|
||||
uid,
|
||||
gid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_engine(&self, event_bus: Arc<EventBus>) -> Option<musicfs_cache::PrefetchHandle> {
|
||||
self.engine
|
||||
.as_ref()
|
||||
.map(|e| e.clone().start(event_bus, self.pattern_store.clone()))
|
||||
}
|
||||
|
||||
pub fn is_prefetch_dir_name(name: &str) -> bool {
|
||||
name == ".prefetch"
|
||||
}
|
||||
|
||||
pub fn is_prefetch_inode(inode: u64) -> bool {
|
||||
inode == PREFETCH_DIR_INODE
|
||||
|| inode == PREFETCH_STATUS_INODE
|
||||
|| inode >= PREFETCH_HINTS_BASE
|
||||
}
|
||||
|
||||
pub fn prefetch_dir_inode() -> u64 {
|
||||
PREFETCH_DIR_INODE
|
||||
}
|
||||
|
||||
pub fn lookup_prefetch_dir(&self, reply: ReplyEntry) {
|
||||
let attr = self.dir_attr(PREFETCH_DIR_INODE);
|
||||
reply.entry(&Duration::from_secs(60), &attr, 0);
|
||||
}
|
||||
|
||||
pub fn lookup_status(&self, reply: ReplyEntry) {
|
||||
let status = self.generate_status();
|
||||
let attr = self.file_attr(PREFETCH_STATUS_INODE, status.len() as u64);
|
||||
reply.entry(&Duration::from_secs(1), &attr, 0);
|
||||
}
|
||||
|
||||
pub fn lookup_hint(&self, name: &str, reply: ReplyEntry) {
|
||||
if let Some(inode) = self.hint_name_to_inode(name) {
|
||||
let attr = self.file_attr(inode, 256);
|
||||
reply.entry(&Duration::from_secs(1), &attr, 0);
|
||||
} else {
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn getattr_prefetch_dir(&self, reply: ReplyAttr) {
|
||||
let attr = self.dir_attr(PREFETCH_DIR_INODE);
|
||||
reply.attr(&Duration::from_secs(60), &attr);
|
||||
}
|
||||
|
||||
pub fn getattr_status(&self, reply: ReplyAttr) {
|
||||
let status = self.generate_status();
|
||||
let attr = self.file_attr(PREFETCH_STATUS_INODE, status.len() as u64);
|
||||
reply.attr(&Duration::from_secs(1), &attr);
|
||||
}
|
||||
|
||||
pub fn getattr_hint(&self, inode: u64, reply: ReplyAttr) {
|
||||
let attr = self.file_attr(inode, 256);
|
||||
reply.attr(&Duration::from_secs(1), &attr);
|
||||
}
|
||||
|
||||
pub fn readdir_prefetch_root(&self, offset: i64, mut reply: ReplyDirectory) {
|
||||
let entries: Vec<(u64, FileType, &str)> = vec![
|
||||
(PREFETCH_DIR_INODE, FileType::Directory, "."),
|
||||
(1, FileType::Directory, ".."),
|
||||
(PREFETCH_STATUS_INODE, FileType::RegularFile, "status"),
|
||||
];
|
||||
|
||||
let recently_played = self.pattern_store.recently_played(7).unwrap_or_default();
|
||||
let predictions: Vec<(u64, FileType, String)> = recently_played
|
||||
.iter()
|
||||
.take(10)
|
||||
.enumerate()
|
||||
.map(|(i, file_id)| {
|
||||
let inode = PREFETCH_HINTS_BASE + i as u64;
|
||||
let name = format!("hint_{:04}", file_id.0);
|
||||
(inode, FileType::RegularFile, name)
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, *name) {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let base_offset = entries.len();
|
||||
for (i, (inode, kind, name)) in predictions.iter().enumerate() {
|
||||
let entry_offset = base_offset + i;
|
||||
if entry_offset < offset as usize {
|
||||
continue;
|
||||
}
|
||||
if reply.add(*inode, (entry_offset + 1) as i64, *kind, name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
pub fn read_status(&self, offset: i64, size: u32, reply: ReplyData) {
|
||||
let status = self.generate_status();
|
||||
let start = offset as usize;
|
||||
let end = std::cmp::min(start + size as usize, status.len());
|
||||
|
||||
if start >= status.len() {
|
||||
reply.data(&[]);
|
||||
} else {
|
||||
reply.data(&status.as_bytes()[start..end]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn read_hint(&self, inode: u64, offset: i64, size: u32, reply: ReplyData) {
|
||||
let file_id = self.inode_to_file_id(inode);
|
||||
let predictions = self.pattern_store.predict_next(file_id, 5);
|
||||
|
||||
let content = predictions
|
||||
.iter()
|
||||
.map(|id| format!("{}", id.0))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let start = offset as usize;
|
||||
let end = std::cmp::min(start + size as usize, content.len());
|
||||
|
||||
if start >= content.len() {
|
||||
reply.data(&[]);
|
||||
} else {
|
||||
reply.data(&content.as_bytes()[start..end]);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_status(&self) -> String {
|
||||
let engine_status = if let Some(engine) = &self.engine {
|
||||
format!(
|
||||
"running: {}\nin_flight: {}",
|
||||
engine.is_running(),
|
||||
engine.in_flight_count()
|
||||
)
|
||||
} else {
|
||||
"engine: disabled".to_string()
|
||||
};
|
||||
|
||||
let most_played = self
|
||||
.pattern_store
|
||||
.most_played(5)
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|id| format!("{}", id.0))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
|
||||
format!(
|
||||
"MusicFS Prefetch Status\n\
|
||||
=======================\n\
|
||||
{}\n\
|
||||
most_played: [{}]\n",
|
||||
engine_status, most_played
|
||||
)
|
||||
}
|
||||
|
||||
fn hint_name_to_inode(&self, name: &str) -> Option<u64> {
|
||||
if name.starts_with("hint_") {
|
||||
let id_str = name.strip_prefix("hint_")?;
|
||||
let id: i64 = id_str.parse().ok()?;
|
||||
Some(PREFETCH_HINTS_BASE + id as u64)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn inode_to_file_id(&self, inode: u64) -> FileId {
|
||||
FileId((inode - PREFETCH_HINTS_BASE) as i64)
|
||||
}
|
||||
|
||||
fn dir_attr(&self, inode: u64) -> FileAttr {
|
||||
FileAttr {
|
||||
ino: inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
kind: FileType::Directory,
|
||||
perm: 0o555,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: 512,
|
||||
flags: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn file_attr(&self, inode: u64, size: u64) -> FileAttr {
|
||||
FileAttr {
|
||||
ino: inode,
|
||||
size,
|
||||
blocks: (size + 511) / 512,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
kind: FileType::RegularFile,
|
||||
perm: 0o444,
|
||||
nlink: 1,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: 512,
|
||||
flags: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_prefetch_ops_new() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let pattern_store =
|
||||
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
|
||||
let _ops = PrefetchOps::new(pattern_store, 1000, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_prefetch_inode() {
|
||||
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_DIR_INODE));
|
||||
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_STATUS_INODE));
|
||||
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_HINTS_BASE));
|
||||
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_HINTS_BASE + 100));
|
||||
assert!(!PrefetchOps::is_prefetch_inode(1));
|
||||
assert!(!PrefetchOps::is_prefetch_inode(1000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hint_name_to_inode() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let pattern_store =
|
||||
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
|
||||
let ops = PrefetchOps::new(pattern_store, 1000, 1000);
|
||||
|
||||
assert_eq!(
|
||||
ops.hint_name_to_inode("hint_0001"),
|
||||
Some(PREFETCH_HINTS_BASE + 1)
|
||||
);
|
||||
assert_eq!(
|
||||
ops.hint_name_to_inode("hint_9999"),
|
||||
Some(PREFETCH_HINTS_BASE + 9999)
|
||||
);
|
||||
assert_eq!(ops.hint_name_to_inode("invalid"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
use fuser::{FileAttr, FileType, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry};
|
||||
use moka::sync::Cache;
|
||||
use musicfs_search::{SearchHit, SearchIndex};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
const SEARCH_DIR_INODE: u64 = 0xFFFF_FFFF_0000_0001;
|
||||
const SEARCH_RESULT_BASE: u64 = 0xFFFF_FFFF_1000_0000;
|
||||
const RESULT_CACHE_MAX_ENTRIES: u64 = 1000;
|
||||
const RESULT_CACHE_TTL_SECS: u64 = 300;
|
||||
const INODE_CACHE_MAX_ENTRIES: u64 = 10000;
|
||||
const MAX_QUERY_LENGTH: usize = 256;
|
||||
|
||||
pub struct SearchOps {
|
||||
index: Arc<SearchIndex>,
|
||||
result_cache: Cache<String, Vec<SearchHit>>,
|
||||
inode_to_result: Cache<u64, (String, usize)>,
|
||||
mount_point: String,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
}
|
||||
|
||||
impl SearchOps {
|
||||
pub fn new(index: Arc<SearchIndex>, mount_point: &str, uid: u32, gid: u32) -> Self {
|
||||
let result_cache = Cache::builder()
|
||||
.max_capacity(RESULT_CACHE_MAX_ENTRIES)
|
||||
.time_to_live(Duration::from_secs(RESULT_CACHE_TTL_SECS))
|
||||
.build();
|
||||
|
||||
let inode_to_result = Cache::builder()
|
||||
.max_capacity(INODE_CACHE_MAX_ENTRIES)
|
||||
.time_to_live(Duration::from_secs(RESULT_CACHE_TTL_SECS))
|
||||
.build();
|
||||
|
||||
Self {
|
||||
index,
|
||||
result_cache,
|
||||
inode_to_result,
|
||||
mount_point: mount_point.to_string(),
|
||||
uid,
|
||||
gid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_search_dir_name(name: &str) -> bool {
|
||||
name == ".search"
|
||||
}
|
||||
|
||||
pub fn is_search_inode(inode: u64) -> bool {
|
||||
inode == SEARCH_DIR_INODE || inode >= SEARCH_RESULT_BASE
|
||||
}
|
||||
|
||||
pub fn search_dir_inode() -> u64 {
|
||||
SEARCH_DIR_INODE
|
||||
}
|
||||
|
||||
pub fn lookup_search_dir(&self, reply: ReplyEntry) {
|
||||
let attr = self.dir_attr(SEARCH_DIR_INODE);
|
||||
reply.entry(&Duration::from_secs(60), &attr, 0);
|
||||
}
|
||||
|
||||
pub fn lookup_query_dir(&self, query: &str, inode: u64, reply: ReplyEntry) {
|
||||
let results = self.execute_query(query);
|
||||
if results.is_empty() {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
|
||||
let attr = self.dir_attr(inode);
|
||||
reply.entry(&Duration::from_secs(1), &attr, 0);
|
||||
}
|
||||
|
||||
pub fn lookup_result(&self, inode: u64, reply: ReplyEntry) {
|
||||
if self.inode_to_result.contains_key(&inode) {
|
||||
let attr = self.symlink_attr(inode, 256);
|
||||
reply.entry(&Duration::from_secs(1), &attr, 0);
|
||||
} else {
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn getattr_search_dir(&self, reply: ReplyAttr) {
|
||||
let attr = self.dir_attr(SEARCH_DIR_INODE);
|
||||
reply.attr(&Duration::from_secs(60), &attr);
|
||||
}
|
||||
|
||||
pub fn getattr_result(&self, inode: u64, reply: ReplyAttr) {
|
||||
let attr = self.symlink_attr(inode, 256);
|
||||
reply.attr(&Duration::from_secs(1), &attr);
|
||||
}
|
||||
|
||||
pub fn readdir_search_root(&self, offset: i64, mut reply: ReplyDirectory) {
|
||||
let entries = vec![
|
||||
(SEARCH_DIR_INODE, FileType::Directory, "."),
|
||||
(1, FileType::Directory, ".."),
|
||||
];
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
pub fn readdir_query(&self, query: &str, offset: i64, mut reply: ReplyDirectory) {
|
||||
let results = self.execute_query(query);
|
||||
|
||||
let entries = vec![
|
||||
(SEARCH_DIR_INODE + 1, FileType::Directory, ".".to_string()),
|
||||
(SEARCH_DIR_INODE, FileType::Directory, "..".to_string()),
|
||||
];
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, name) {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let base_offset = entries.len();
|
||||
for (i, hit) in results.iter().enumerate() {
|
||||
let entry_offset = base_offset + i;
|
||||
if entry_offset < offset as usize {
|
||||
continue;
|
||||
}
|
||||
|
||||
let inode = SEARCH_RESULT_BASE + i as u64;
|
||||
let name = self.result_filename(hit, i);
|
||||
|
||||
self.inode_to_result.insert(inode, (query.to_string(), i));
|
||||
|
||||
if reply.add(inode, (entry_offset + 1) as i64, FileType::Symlink, &name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
pub fn readlink(&self, inode: u64, reply: ReplyData) {
|
||||
let (query, index) = match self.inode_to_result.get(&inode) {
|
||||
Some((q, i)) => (q, i),
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let results = self.execute_query(&query);
|
||||
if let Some(hit) = results.get(index) {
|
||||
if let Some(target) = self.safe_symlink_target(hit.virtual_path.as_str()) {
|
||||
reply.data(target.as_bytes());
|
||||
} else {
|
||||
reply.error(libc::EINVAL);
|
||||
}
|
||||
} else {
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_symlink_target(&self, virtual_path: &str) -> Option<String> {
|
||||
let normalized = Path::new(virtual_path).components().fold(
|
||||
std::path::PathBuf::new(),
|
||||
|mut acc, comp| {
|
||||
match comp {
|
||||
std::path::Component::Normal(s) => acc.push(s),
|
||||
std::path::Component::RootDir => acc.push("/"),
|
||||
_ => {}
|
||||
}
|
||||
acc
|
||||
},
|
||||
);
|
||||
|
||||
let path_str = normalized.to_string_lossy();
|
||||
if path_str.contains("..") {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(format!("{}{}", self.mount_point, path_str))
|
||||
}
|
||||
|
||||
fn execute_query(&self, query: &str) -> Vec<SearchHit> {
|
||||
let query = if query.len() > MAX_QUERY_LENGTH {
|
||||
&query[..MAX_QUERY_LENGTH]
|
||||
} else {
|
||||
query
|
||||
};
|
||||
|
||||
if let Some(results) = self.result_cache.get(query) {
|
||||
return results;
|
||||
}
|
||||
|
||||
let results = self.index.search(query, 1000).unwrap_or_default();
|
||||
self.result_cache.insert(query.to_string(), results.clone());
|
||||
results
|
||||
}
|
||||
|
||||
fn result_filename(&self, hit: &SearchHit, index: usize) -> String {
|
||||
let artist = hit.artist.as_deref().unwrap_or("Unknown");
|
||||
let title = hit.title.as_deref().unwrap_or("Unknown");
|
||||
let ext = hit
|
||||
.virtual_path
|
||||
.as_str()
|
||||
.rsplit('.')
|
||||
.next()
|
||||
.unwrap_or("flac");
|
||||
format!("{:03}. {} - {}.{}", index + 1, artist, title, ext)
|
||||
}
|
||||
|
||||
fn dir_attr(&self, inode: u64) -> FileAttr {
|
||||
FileAttr {
|
||||
ino: inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
kind: FileType::Directory,
|
||||
perm: 0o555,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: 512,
|
||||
flags: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn symlink_attr(&self, inode: u64, target_len: u64) -> FileAttr {
|
||||
FileAttr {
|
||||
ino: inode,
|
||||
size: target_len,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
kind: FileType::Symlink,
|
||||
perm: 0o777,
|
||||
nlink: 1,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: 512,
|
||||
flags: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_search::SearchIndex;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_search_ops_new() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
|
||||
let _ops = SearchOps::new(index, "/mnt/music", 1000, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_search_inode() {
|
||||
assert!(SearchOps::is_search_inode(SEARCH_DIR_INODE));
|
||||
assert!(SearchOps::is_search_inode(SEARCH_RESULT_BASE));
|
||||
assert!(SearchOps::is_search_inode(SEARCH_RESULT_BASE + 100));
|
||||
assert!(!SearchOps::is_search_inode(1));
|
||||
assert!(!SearchOps::is_search_inode(1000));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
[package]
|
||||
name = "musicfs-grpc"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-search = { path = "../musicfs-search" }
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
tonic.workspace = true
|
||||
prost.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
chrono.workspace = true
|
||||
reqwest = { version = "0.11", features = ["json"] }
|
||||
hmac = "0.12"
|
||||
sha2 = "0.10"
|
||||
hex.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
tonic-build.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -0,0 +1,4 @@
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
tonic_build::compile_protos("proto/musicfs.proto")?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package musicfs.v1;
|
||||
|
||||
service MusicFS {
|
||||
rpc Search(SearchRequest) returns (SearchResponse);
|
||||
rpc SearchStream(SearchRequest) returns (stream SearchResult);
|
||||
rpc GetStatus(Empty) returns (StatusResponse);
|
||||
rpc Shutdown(ShutdownRequest) returns (Empty);
|
||||
rpc GetCacheStats(Empty) returns (CacheStats);
|
||||
rpc ClearCache(ClearCacheRequest) returns (ClearCacheResponse);
|
||||
rpc Prefetch(PrefetchRequest) returns (stream PrefetchProgress);
|
||||
rpc ListOrigins(Empty) returns (OriginsResponse);
|
||||
rpc GetOriginHealth(OriginRequest) returns (OriginHealthResponse);
|
||||
rpc RescanOrigin(OriginRequest) returns (stream SyncProgress);
|
||||
rpc SubscribeEvents(EventFilter) returns (stream Event);
|
||||
}
|
||||
|
||||
message Empty {}
|
||||
|
||||
message SearchRequest {
|
||||
string query = 1;
|
||||
optional uint32 limit = 2;
|
||||
optional uint32 offset = 3;
|
||||
optional string origin_id = 4;
|
||||
}
|
||||
|
||||
message SearchResponse {
|
||||
repeated SearchResult results = 1;
|
||||
uint64 total_matches = 2;
|
||||
uint32 query_time_ms = 3;
|
||||
}
|
||||
|
||||
message SearchResult {
|
||||
int64 file_id = 1;
|
||||
string virtual_path = 2;
|
||||
optional string artist = 3;
|
||||
optional string album = 4;
|
||||
optional string title = 5;
|
||||
float score = 6;
|
||||
map<string, string> highlights = 7;
|
||||
}
|
||||
|
||||
enum MountState {
|
||||
MOUNT_UNKNOWN = 0;
|
||||
MOUNT_MOUNTING = 1;
|
||||
MOUNT_READY = 2;
|
||||
MOUNT_SYNCING = 3;
|
||||
MOUNT_DEGRADED = 4;
|
||||
MOUNT_UNMOUNTING = 5;
|
||||
}
|
||||
|
||||
message StatusResponse {
|
||||
string version = 1;
|
||||
uint64 uptime_secs = 2;
|
||||
string mount_point = 3;
|
||||
MountState state = 4;
|
||||
uint32 open_file_handles = 5;
|
||||
uint64 fuse_ops_total = 6;
|
||||
uint64 files_indexed = 7;
|
||||
uint64 cache_size_bytes = 8;
|
||||
repeated OriginStatus origins = 9;
|
||||
}
|
||||
|
||||
message OriginStatus {
|
||||
string id = 1;
|
||||
string origin_type = 2;
|
||||
HealthStatus health = 3;
|
||||
uint64 files_count = 4;
|
||||
}
|
||||
|
||||
enum HealthStatus {
|
||||
HEALTH_UNKNOWN = 0;
|
||||
HEALTH_HEALTHY = 1;
|
||||
HEALTH_DEGRADED = 2;
|
||||
HEALTH_UNHEALTHY = 3;
|
||||
}
|
||||
|
||||
message ShutdownRequest {
|
||||
bool graceful = 1;
|
||||
uint32 timeout_secs = 2;
|
||||
}
|
||||
|
||||
message TierStats {
|
||||
uint64 entries = 1;
|
||||
uint64 size_bytes = 2;
|
||||
uint64 hits = 3;
|
||||
uint64 misses = 4;
|
||||
}
|
||||
|
||||
message CacheStats {
|
||||
uint64 total_size_bytes = 1;
|
||||
uint64 used_size_bytes = 2;
|
||||
uint64 size_limit_bytes = 3;
|
||||
uint64 chunk_count = 4;
|
||||
uint64 chunks_unique = 5;
|
||||
double dedup_ratio = 6;
|
||||
uint64 hit_count = 7;
|
||||
uint64 miss_count = 8;
|
||||
double hit_ratio = 9;
|
||||
uint64 metadata_entries = 10;
|
||||
uint64 metadata_bytes = 11;
|
||||
TierStats l1_metadata = 12;
|
||||
TierStats l2_headers = 13;
|
||||
TierStats l3_chunks = 14;
|
||||
}
|
||||
|
||||
message ClearCacheRequest {
|
||||
optional string origin_id = 1;
|
||||
bool clear_metadata = 2;
|
||||
bool clear_chunks = 3;
|
||||
}
|
||||
|
||||
message ClearCacheResponse {
|
||||
uint64 bytes_cleared = 1;
|
||||
uint64 chunks_cleared = 2;
|
||||
}
|
||||
|
||||
message PrefetchRequest {
|
||||
repeated string paths = 1;
|
||||
optional string origin_id = 2;
|
||||
}
|
||||
|
||||
message PrefetchProgress {
|
||||
string current_path = 1;
|
||||
uint32 completed = 2;
|
||||
uint32 total = 3;
|
||||
uint64 bytes_fetched = 4;
|
||||
}
|
||||
|
||||
message OriginsResponse {
|
||||
repeated OriginInfo origins = 1;
|
||||
}
|
||||
|
||||
message OriginInfo {
|
||||
string id = 1;
|
||||
string origin_type = 2;
|
||||
string display_name = 3;
|
||||
string root_path = 4;
|
||||
HealthStatus health = 5;
|
||||
uint64 files_count = 6;
|
||||
uint64 total_size_bytes = 7;
|
||||
}
|
||||
|
||||
message OriginRequest {
|
||||
string origin_id = 1;
|
||||
}
|
||||
|
||||
message OriginHealthResponse {
|
||||
string origin_id = 1;
|
||||
HealthStatus status = 2;
|
||||
optional string message = 3;
|
||||
uint64 last_check_secs = 4;
|
||||
}
|
||||
|
||||
message SyncProgress {
|
||||
string phase = 1;
|
||||
uint32 current = 2;
|
||||
uint32 total = 3;
|
||||
string current_path = 4;
|
||||
uint64 bytes_synced = 5;
|
||||
}
|
||||
|
||||
message EventFilter {
|
||||
repeated string event_types = 1;
|
||||
optional string origin_id = 2;
|
||||
}
|
||||
|
||||
message Event {
|
||||
string event_type = 1;
|
||||
int64 timestamp_ms = 2;
|
||||
optional string origin_id = 3;
|
||||
optional string path = 4;
|
||||
optional int64 file_id = 5;
|
||||
map<string, string> metadata = 6;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
pub mod proto {
|
||||
pub mod musicfs {
|
||||
pub mod v1 {
|
||||
tonic::include_proto!("musicfs.v1");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod search_service;
|
||||
mod server;
|
||||
mod webhook;
|
||||
|
||||
pub use proto::musicfs::v1::music_fs_server::{MusicFs, MusicFsServer as MusicFsGrpcServer};
|
||||
pub use proto::musicfs::v1::*;
|
||||
pub use search_service::SearchService;
|
||||
pub use server::MusicFsServer;
|
||||
pub use webhook::{WebhookConfig, WebhookHandler, WebhookPayload};
|
||||
@@ -0,0 +1,251 @@
|
||||
use crate::proto::musicfs::v1::{
|
||||
music_fs_server::MusicFs, CacheStats, ClearCacheRequest, ClearCacheResponse, Empty, Event,
|
||||
EventFilter, OriginHealthResponse, OriginRequest, OriginsResponse, PrefetchProgress,
|
||||
PrefetchRequest, SearchRequest, SearchResponse, SearchResult, ShutdownRequest, StatusResponse,
|
||||
SyncProgress,
|
||||
};
|
||||
use musicfs_search::SearchIndex;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::debug;
|
||||
|
||||
pub struct SearchService {
|
||||
index: Arc<SearchIndex>,
|
||||
}
|
||||
|
||||
impl SearchService {
|
||||
pub fn new(index: Arc<SearchIndex>) -> Self {
|
||||
Self { index }
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl MusicFs for SearchService {
|
||||
async fn search(
|
||||
&self,
|
||||
request: Request<SearchRequest>,
|
||||
) -> Result<Response<SearchResponse>, Status> {
|
||||
let start = Instant::now();
|
||||
let req = request.into_inner();
|
||||
|
||||
if req.query.is_empty() {
|
||||
return Err(Status::invalid_argument("Query cannot be empty"));
|
||||
}
|
||||
|
||||
if req.query.len() > 256 {
|
||||
return Err(Status::invalid_argument(
|
||||
"Query exceeds maximum length (256)",
|
||||
));
|
||||
}
|
||||
|
||||
let limit = req.limit.unwrap_or(100).min(10000) as usize;
|
||||
let offset = req.offset.unwrap_or(0) as usize;
|
||||
|
||||
let results = self
|
||||
.index
|
||||
.search(&req.query, limit + offset)
|
||||
.map_err(|e| Status::internal(format!("Search failed: {}", e)))?;
|
||||
|
||||
let hits: Vec<SearchResult> = results
|
||||
.into_iter()
|
||||
.skip(offset)
|
||||
.take(limit)
|
||||
.map(|hit| SearchResult {
|
||||
file_id: hit.file_id.0,
|
||||
virtual_path: hit.virtual_path.as_str().to_string(),
|
||||
artist: hit.artist,
|
||||
album: hit.album,
|
||||
title: hit.title,
|
||||
score: hit.score,
|
||||
highlights: Default::default(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total_matches = self.index.count();
|
||||
let query_time_ms = start.elapsed().as_millis() as u32;
|
||||
|
||||
debug!(
|
||||
"Search '{}' returned {} results in {}ms",
|
||||
req.query,
|
||||
hits.len(),
|
||||
query_time_ms
|
||||
);
|
||||
|
||||
Ok(Response::new(SearchResponse {
|
||||
results: hits,
|
||||
total_matches,
|
||||
query_time_ms,
|
||||
}))
|
||||
}
|
||||
|
||||
type SearchStreamStream = ReceiverStream<Result<SearchResult, Status>>;
|
||||
|
||||
async fn search_stream(
|
||||
&self,
|
||||
request: Request<SearchRequest>,
|
||||
) -> Result<Response<Self::SearchStreamStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
|
||||
if req.query.is_empty() {
|
||||
return Err(Status::invalid_argument("Query cannot be empty"));
|
||||
}
|
||||
|
||||
let limit = req.limit.unwrap_or(1000).min(10000) as usize;
|
||||
|
||||
let results = self
|
||||
.index
|
||||
.search(&req.query, limit)
|
||||
.map_err(|e| Status::internal(format!("Search failed: {}", e)))?;
|
||||
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(100);
|
||||
|
||||
tokio::spawn(async move {
|
||||
for hit in results {
|
||||
let result = SearchResult {
|
||||
file_id: hit.file_id.0,
|
||||
virtual_path: hit.virtual_path.as_str().to_string(),
|
||||
artist: hit.artist,
|
||||
album: hit.album,
|
||||
title: hit.title,
|
||||
score: hit.score,
|
||||
highlights: Default::default(),
|
||||
};
|
||||
if tx.send(Ok(result)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
async fn get_status(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<StatusResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn shutdown(
|
||||
&self,
|
||||
_request: Request<ShutdownRequest>,
|
||||
) -> Result<Response<Empty>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_cache_stats(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<CacheStats>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn clear_cache(
|
||||
&self,
|
||||
_request: Request<ClearCacheRequest>,
|
||||
) -> Result<Response<ClearCacheResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
type PrefetchStream = ReceiverStream<Result<PrefetchProgress, Status>>;
|
||||
|
||||
async fn prefetch(
|
||||
&self,
|
||||
_request: Request<PrefetchRequest>,
|
||||
) -> Result<Response<Self::PrefetchStream>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_origins(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<OriginsResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_origin_health(
|
||||
&self,
|
||||
_request: Request<OriginRequest>,
|
||||
) -> Result<Response<OriginHealthResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
type RescanOriginStream = ReceiverStream<Result<SyncProgress, Status>>;
|
||||
|
||||
async fn rescan_origin(
|
||||
&self,
|
||||
_request: Request<OriginRequest>,
|
||||
) -> Result<Response<Self::RescanOriginStream>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
|
||||
|
||||
async fn subscribe_events(
|
||||
&self,
|
||||
_request: Request<EventFilter>,
|
||||
) -> Result<Response<Self::SubscribeEventsStream>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_grpc_search_empty_query() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
|
||||
let service = SearchService::new(index);
|
||||
|
||||
let request = Request::new(SearchRequest {
|
||||
query: String::new(),
|
||||
limit: Some(10),
|
||||
offset: None,
|
||||
origin_id: None,
|
||||
});
|
||||
|
||||
let result = service.search(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_grpc_search_returns_response() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
|
||||
let service = SearchService::new(index);
|
||||
|
||||
let request = Request::new(SearchRequest {
|
||||
query: "test".to_string(),
|
||||
limit: Some(10),
|
||||
offset: None,
|
||||
origin_id: None,
|
||||
});
|
||||
|
||||
let response = service.search(request).await.unwrap();
|
||||
assert!(response.get_ref().results.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
use crate::proto::musicfs::v1::{
|
||||
music_fs_server::MusicFs, CacheStats, ClearCacheRequest, ClearCacheResponse, Empty, Event,
|
||||
EventFilter, HealthStatus, MountState, OriginHealthResponse, OriginRequest, OriginsResponse,
|
||||
PrefetchProgress, PrefetchRequest, SearchRequest, SearchResponse, SearchResult,
|
||||
ShutdownRequest, StatusResponse, SyncProgress, TierStats,
|
||||
};
|
||||
use musicfs_core::{Event as CoreEvent, EventBus};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::{debug, info, instrument};
|
||||
|
||||
pub struct MusicFsServer {
|
||||
start_time: Instant,
|
||||
event_bus: Arc<EventBus>,
|
||||
version: String,
|
||||
}
|
||||
|
||||
impl MusicFsServer {
|
||||
pub fn new(event_bus: Arc<EventBus>) -> Self {
|
||||
Self {
|
||||
start_time: Instant::now(),
|
||||
event_bus,
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn event_to_proto(event: &CoreEvent) -> Event {
|
||||
let (event_type, origin_id, path, file_id) = match event {
|
||||
CoreEvent::FileAccessed {
|
||||
file_id,
|
||||
origin_id,
|
||||
path,
|
||||
..
|
||||
} => (
|
||||
"file_accessed".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
Some(path.as_str().to_string()),
|
||||
Some(file_id.0),
|
||||
),
|
||||
CoreEvent::FileAdded { path, origin_id } => (
|
||||
"file_added".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
Some(path.as_str().to_string()),
|
||||
None,
|
||||
),
|
||||
CoreEvent::FileRemoved { path, file_id } => (
|
||||
"file_removed".to_string(),
|
||||
None,
|
||||
Some(path.as_str().to_string()),
|
||||
file_id.map(|id| id.0),
|
||||
),
|
||||
CoreEvent::FileModified { path } => (
|
||||
"file_modified".to_string(),
|
||||
None,
|
||||
Some(path.as_str().to_string()),
|
||||
None,
|
||||
),
|
||||
CoreEvent::SyncStarted { origin_id } => (
|
||||
"sync_started".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
CoreEvent::SyncCompleted {
|
||||
origin_id,
|
||||
files_changed,
|
||||
} => {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("files_changed".to_string(), files_changed.to_string());
|
||||
return Event {
|
||||
event_type: "sync_completed".to_string(),
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id: Some(origin_id.to_string()),
|
||||
path: None,
|
||||
file_id: None,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
CoreEvent::OriginHealthChanged { origin_id, healthy } => {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("healthy".to_string(), healthy.to_string());
|
||||
return Event {
|
||||
event_type: "origin_health_changed".to_string(),
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id: Some(origin_id.to_string()),
|
||||
path: None,
|
||||
file_id: None,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
CoreEvent::CacheEviction { bytes_freed } => {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("bytes_freed".to_string(), bytes_freed.to_string());
|
||||
return Event {
|
||||
event_type: "cache_eviction".to_string(),
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id: None,
|
||||
path: None,
|
||||
file_id: None,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
CoreEvent::OriginConnected { origin_id } => (
|
||||
"origin_connected".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
CoreEvent::OriginDisconnected { origin_id } => (
|
||||
"origin_disconnected".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
CoreEvent::AllOriginsUnhealthy { candidate_count } => {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("candidate_count".to_string(), candidate_count.to_string());
|
||||
return Event {
|
||||
event_type: "all_origins_unhealthy".to_string(),
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id: None,
|
||||
path: None,
|
||||
file_id: None,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
Event {
|
||||
event_type,
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id,
|
||||
path,
|
||||
file_id,
|
||||
metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_filter(event: &CoreEvent, filter: &EventFilter) -> bool {
|
||||
if !filter.event_types.is_empty() {
|
||||
let event_type = match event {
|
||||
CoreEvent::FileAccessed { .. } => "file_accessed",
|
||||
CoreEvent::FileAdded { .. } => "file_added",
|
||||
CoreEvent::FileRemoved { .. } => "file_removed",
|
||||
CoreEvent::FileModified { .. } => "file_modified",
|
||||
CoreEvent::SyncStarted { .. } => "sync_started",
|
||||
CoreEvent::SyncCompleted { .. } => "sync_completed",
|
||||
CoreEvent::OriginHealthChanged { .. } => "origin_health_changed",
|
||||
CoreEvent::CacheEviction { .. } => "cache_eviction",
|
||||
CoreEvent::OriginConnected { .. } => "origin_connected",
|
||||
CoreEvent::OriginDisconnected { .. } => "origin_disconnected",
|
||||
CoreEvent::AllOriginsUnhealthy { .. } => "all_origins_unhealthy",
|
||||
};
|
||||
|
||||
if !filter.event_types.iter().any(|t| t == event_type) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref origin_filter) = filter.origin_id {
|
||||
let event_origin = match event {
|
||||
CoreEvent::FileAccessed { origin_id, .. }
|
||||
| CoreEvent::FileAdded { origin_id, .. }
|
||||
| CoreEvent::SyncStarted { origin_id }
|
||||
| CoreEvent::SyncCompleted { origin_id, .. }
|
||||
| CoreEvent::OriginHealthChanged { origin_id, .. }
|
||||
| CoreEvent::OriginConnected { origin_id }
|
||||
| CoreEvent::OriginDisconnected { origin_id } => Some(origin_id.to_string()),
|
||||
CoreEvent::FileRemoved { .. }
|
||||
| CoreEvent::FileModified { .. }
|
||||
| CoreEvent::CacheEviction { .. }
|
||||
| CoreEvent::AllOriginsUnhealthy { .. } => None,
|
||||
};
|
||||
|
||||
if event_origin.as_ref() != Some(origin_filter) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl MusicFs for MusicFsServer {
|
||||
async fn search(
|
||||
&self,
|
||||
_request: Request<SearchRequest>,
|
||||
) -> Result<Response<SearchResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use SearchService for search operations",
|
||||
))
|
||||
}
|
||||
|
||||
type SearchStreamStream = ReceiverStream<Result<SearchResult, Status>>;
|
||||
|
||||
async fn search_stream(
|
||||
&self,
|
||||
_request: Request<SearchRequest>,
|
||||
) -> Result<Response<Self::SearchStreamStream>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use SearchService for search operations",
|
||||
))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _request), fields(method = "get_status"))]
|
||||
async fn get_status(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<StatusResponse>, Status> {
|
||||
debug!("gRPC get_status called");
|
||||
let uptime = self.start_time.elapsed().as_secs();
|
||||
|
||||
Ok(Response::new(StatusResponse {
|
||||
version: self.version.clone(),
|
||||
uptime_secs: uptime,
|
||||
mount_point: String::new(),
|
||||
state: MountState::MountReady as i32,
|
||||
open_file_handles: 0,
|
||||
fuse_ops_total: 0,
|
||||
files_indexed: 0,
|
||||
cache_size_bytes: 0,
|
||||
origins: vec![],
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "shutdown"))]
|
||||
async fn shutdown(&self, request: Request<ShutdownRequest>) -> Result<Response<Empty>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(
|
||||
graceful = req.graceful,
|
||||
timeout_secs = req.timeout_secs,
|
||||
"gRPC shutdown requested"
|
||||
);
|
||||
|
||||
Ok(Response::new(Empty {}))
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip(self, _request),
|
||||
fields(method = "get_cache_stats")
|
||||
)]
|
||||
async fn get_cache_stats(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<CacheStats>, Status> {
|
||||
debug!("gRPC get_cache_stats called");
|
||||
Ok(Response::new(CacheStats {
|
||||
total_size_bytes: 0,
|
||||
used_size_bytes: 0,
|
||||
size_limit_bytes: 0,
|
||||
chunk_count: 0,
|
||||
chunks_unique: 0,
|
||||
dedup_ratio: 0.0,
|
||||
hit_count: 0,
|
||||
miss_count: 0,
|
||||
hit_ratio: 0.0,
|
||||
metadata_entries: 0,
|
||||
metadata_bytes: 0,
|
||||
l1_metadata: Some(TierStats {
|
||||
entries: 0,
|
||||
size_bytes: 0,
|
||||
hits: 0,
|
||||
misses: 0,
|
||||
}),
|
||||
l2_headers: Some(TierStats {
|
||||
entries: 0,
|
||||
size_bytes: 0,
|
||||
hits: 0,
|
||||
misses: 0,
|
||||
}),
|
||||
l3_chunks: Some(TierStats {
|
||||
entries: 0,
|
||||
size_bytes: 0,
|
||||
hits: 0,
|
||||
misses: 0,
|
||||
}),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "clear_cache"))]
|
||||
async fn clear_cache(
|
||||
&self,
|
||||
request: Request<ClearCacheRequest>,
|
||||
) -> Result<Response<ClearCacheResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(
|
||||
origin_id = ?req.origin_id,
|
||||
clear_metadata = req.clear_metadata,
|
||||
clear_chunks = req.clear_chunks,
|
||||
"gRPC clear_cache"
|
||||
);
|
||||
|
||||
Ok(Response::new(ClearCacheResponse {
|
||||
bytes_cleared: 0,
|
||||
chunks_cleared: 0,
|
||||
}))
|
||||
}
|
||||
|
||||
type PrefetchStream = ReceiverStream<Result<PrefetchProgress, Status>>;
|
||||
|
||||
#[instrument(level = "debug", skip(self, request), fields(method = "prefetch"))]
|
||||
async fn prefetch(
|
||||
&self,
|
||||
request: Request<PrefetchRequest>,
|
||||
) -> Result<Response<Self::PrefetchStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
let total = req.paths.len() as u32;
|
||||
debug!(file_count = total, "gRPC prefetch started");
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
|
||||
tokio::spawn(async move {
|
||||
for (i, path) in req.paths.into_iter().enumerate() {
|
||||
let progress = PrefetchProgress {
|
||||
current_path: path,
|
||||
completed: i as u32 + 1,
|
||||
total,
|
||||
bytes_fetched: 0,
|
||||
};
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _request), fields(method = "list_origins"))]
|
||||
async fn list_origins(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<OriginsResponse>, Status> {
|
||||
debug!("gRPC list_origins called");
|
||||
Ok(Response::new(OriginsResponse { origins: vec![] }))
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip(self, request),
|
||||
fields(method = "get_origin_health")
|
||||
)]
|
||||
async fn get_origin_health(
|
||||
&self,
|
||||
request: Request<OriginRequest>,
|
||||
) -> Result<Response<OriginHealthResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
debug!(origin_id = %req.origin_id, "gRPC get_origin_health");
|
||||
|
||||
Ok(Response::new(OriginHealthResponse {
|
||||
origin_id: req.origin_id,
|
||||
status: HealthStatus::HealthUnknown as i32,
|
||||
message: None,
|
||||
last_check_secs: 0,
|
||||
}))
|
||||
}
|
||||
|
||||
type RescanOriginStream = ReceiverStream<Result<SyncProgress, Status>>;
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "rescan_origin"))]
|
||||
async fn rescan_origin(
|
||||
&self,
|
||||
request: Request<OriginRequest>,
|
||||
) -> Result<Response<Self::RescanOriginStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(origin_id = %req.origin_id, "gRPC rescan_origin started");
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let phases = ["scanning", "indexing", "complete"];
|
||||
for (i, phase) in phases.iter().enumerate() {
|
||||
let progress = SyncProgress {
|
||||
phase: phase.to_string(),
|
||||
current: i as u32 + 1,
|
||||
total: phases.len() as u32,
|
||||
current_path: String::new(),
|
||||
bytes_synced: 0,
|
||||
};
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
|
||||
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "subscribe_events")
|
||||
)]
|
||||
async fn subscribe_events(
|
||||
&self,
|
||||
request: Request<EventFilter>,
|
||||
) -> Result<Response<Self::SubscribeEventsStream>, Status> {
|
||||
info!("gRPC subscribe_events: client connected");
|
||||
let filter = request.into_inner();
|
||||
let mut rx = self.event_bus.subscribe();
|
||||
let (tx, out_rx) = mpsc::channel(100);
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(event) => {
|
||||
if Self::matches_filter(&event, &filter) {
|
||||
let proto_event = Self::event_to_proto(&event);
|
||||
if tx.send(Ok(proto_event)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
tracing::warn!(skipped = n, "Event subscriber lagged, skipped events");
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
tracing::debug!("Event channel closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(out_rx)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_status() {
|
||||
let event_bus = Arc::new(EventBus::new(16));
|
||||
let server = MusicFsServer::new(event_bus);
|
||||
|
||||
let response = server.get_status(Request::new(Empty {})).await.unwrap();
|
||||
let status = response.into_inner();
|
||||
|
||||
assert!(!status.version.is_empty());
|
||||
assert!(status.uptime_secs < 5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_cache_stats() {
|
||||
let event_bus = Arc::new(EventBus::new(16));
|
||||
let server = MusicFsServer::new(event_bus);
|
||||
|
||||
let response = server
|
||||
.get_cache_stats(Request::new(Empty {}))
|
||||
.await
|
||||
.unwrap();
|
||||
let stats = response.into_inner();
|
||||
|
||||
assert_eq!(stats.hit_ratio, 0.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
use musicfs_core::Event;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WebhookPayload {
|
||||
pub event_type: String,
|
||||
pub timestamp: i64,
|
||||
pub data: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
pub struct WebhookConfig {
|
||||
pub url: String,
|
||||
#[serde(skip_serializing)]
|
||||
pub secret: Option<String>,
|
||||
pub events: Vec<String>,
|
||||
#[serde(default = "default_retry_count")]
|
||||
pub retry_count: u32,
|
||||
#[serde(default = "default_timeout_ms")]
|
||||
pub timeout_ms: u64,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for WebhookConfig {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("WebhookConfig")
|
||||
.field("url", &self.url)
|
||||
.field("secret", &self.secret.as_ref().map(|_| "[REDACTED]"))
|
||||
.field("events", &self.events)
|
||||
.field("retry_count", &self.retry_count)
|
||||
.field("timeout_ms", &self.timeout_ms)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
fn default_retry_count() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
fn default_timeout_ms() -> u64 {
|
||||
5000
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum WebhookError {
|
||||
#[error("Failed to initialize HTTP client: {0}")]
|
||||
ClientInit(String),
|
||||
}
|
||||
|
||||
pub struct WebhookHandler {
|
||||
client: reqwest::Client,
|
||||
configs: Vec<WebhookConfig>,
|
||||
}
|
||||
|
||||
impl WebhookHandler {
|
||||
pub fn new(configs: Vec<WebhookConfig>) -> Result<Self, WebhookError> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "Failed to create webhook HTTP client");
|
||||
WebhookError::ClientInit(e.to_string())
|
||||
})?;
|
||||
|
||||
Ok(Self { client, configs })
|
||||
}
|
||||
|
||||
pub async fn run(&self, mut rx: broadcast::Receiver<Event>) {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(event) => {
|
||||
for config in &self.configs {
|
||||
if self.matches_filter(&event, config) {
|
||||
self.dispatch(config, &event).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
warn!(skipped = n, "Webhook handler lagged, skipped events");
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => {
|
||||
debug!("Event channel closed, webhook handler stopping");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch(&self, config: &WebhookConfig, event: &Event) {
|
||||
let payload = WebhookPayload {
|
||||
event_type: self.event_type_name(event),
|
||||
timestamp: chrono::Utc::now().timestamp_millis(),
|
||||
data: self.event_to_json(event),
|
||||
};
|
||||
|
||||
let signature = self.sign(&payload, config);
|
||||
|
||||
let mut attempts = 0u32;
|
||||
loop {
|
||||
let result = self
|
||||
.client
|
||||
.post(&config.url)
|
||||
.timeout(Duration::from_millis(config.timeout_ms))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-MusicFS-Signature", &signature)
|
||||
.header("X-MusicFS-Event", &payload.event_type)
|
||||
.json(&payload)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
debug!(
|
||||
"Webhook delivered to {} for {}",
|
||||
config.url, payload.event_type
|
||||
);
|
||||
break;
|
||||
}
|
||||
Ok(resp) => {
|
||||
warn!(
|
||||
"Webhook to {} returned status {}, attempt {}/{}",
|
||||
config.url,
|
||||
resp.status(),
|
||||
attempts + 1,
|
||||
config.retry_count + 1
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"Webhook to {} failed: {}, attempt {}/{}",
|
||||
config.url,
|
||||
e,
|
||||
attempts + 1,
|
||||
config.retry_count + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if attempts >= config.retry_count {
|
||||
warn!(
|
||||
"Webhook delivery to {} failed after {} attempts",
|
||||
config.url,
|
||||
attempts + 1
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
attempts += 1;
|
||||
let delay = Duration::from_millis(100 * 2u64.pow(attempts));
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn sign(&self, payload: &WebhookPayload, config: &WebhookConfig) -> String {
|
||||
match &config.secret {
|
||||
Some(secret) => {
|
||||
use hmac::{Hmac, Mac};
|
||||
use sha2::Sha256;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
let body = serde_json::to_string(payload).unwrap_or_default();
|
||||
let mac = match HmacSha256::new_from_slice(secret.as_bytes()) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
error!(error = %e, "Invalid HMAC key for webhook signature");
|
||||
return String::new();
|
||||
}
|
||||
};
|
||||
let mut mac = mac;
|
||||
mac.update(body.as_bytes());
|
||||
let result = mac.finalize();
|
||||
|
||||
format!("sha256={}", hex::encode(result.into_bytes()))
|
||||
}
|
||||
None => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_filter(&self, event: &Event, config: &WebhookConfig) -> bool {
|
||||
if config.events.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let event_type = self.event_type_name(event);
|
||||
config.events.iter().any(|e| e == &event_type)
|
||||
}
|
||||
|
||||
fn event_type_name(&self, event: &Event) -> String {
|
||||
match event {
|
||||
Event::FileAccessed { .. } => "file_accessed",
|
||||
Event::FileAdded { .. } => "file_added",
|
||||
Event::FileRemoved { .. } => "file_removed",
|
||||
Event::FileModified { .. } => "file_modified",
|
||||
Event::SyncStarted { .. } => "sync_started",
|
||||
Event::SyncCompleted { .. } => "sync_completed",
|
||||
Event::OriginHealthChanged { .. } => "origin_health_changed",
|
||||
Event::CacheEviction { .. } => "cache_eviction",
|
||||
Event::OriginConnected { .. } => "origin_connected",
|
||||
Event::OriginDisconnected { .. } => "origin_disconnected",
|
||||
Event::AllOriginsUnhealthy { .. } => "all_origins_unhealthy",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn event_to_json(&self, event: &Event) -> serde_json::Value {
|
||||
match event {
|
||||
Event::FileAccessed {
|
||||
file_id,
|
||||
origin_id,
|
||||
path,
|
||||
offset,
|
||||
size,
|
||||
} => serde_json::json!({
|
||||
"file_id": file_id.0,
|
||||
"origin_id": origin_id.to_string(),
|
||||
"path": path.as_str(),
|
||||
"offset": offset,
|
||||
"size": size,
|
||||
}),
|
||||
Event::FileAdded { path, origin_id } => serde_json::json!({
|
||||
"path": path.as_str(),
|
||||
"origin_id": origin_id.to_string(),
|
||||
}),
|
||||
Event::FileRemoved { path, file_id } => serde_json::json!({
|
||||
"path": path.as_str(),
|
||||
"file_id": file_id.map(|id| id.0),
|
||||
}),
|
||||
Event::FileModified { path } => serde_json::json!({
|
||||
"path": path.as_str(),
|
||||
}),
|
||||
Event::SyncStarted { origin_id } => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
}),
|
||||
Event::SyncCompleted {
|
||||
origin_id,
|
||||
files_changed,
|
||||
} => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
"files_changed": files_changed,
|
||||
}),
|
||||
Event::OriginHealthChanged { origin_id, healthy } => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
"healthy": healthy,
|
||||
}),
|
||||
Event::CacheEviction { bytes_freed } => serde_json::json!({
|
||||
"bytes_freed": bytes_freed,
|
||||
}),
|
||||
Event::OriginConnected { origin_id } => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
}),
|
||||
Event::OriginDisconnected { origin_id } => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
}),
|
||||
Event::AllOriginsUnhealthy { candidate_count } => serde_json::json!({
|
||||
"candidate_count": candidate_count,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_core::OriginId;
|
||||
|
||||
#[test]
|
||||
fn test_webhook_config_defaults() {
|
||||
let json = r#"{"url": "http://example.com", "events": []}"#;
|
||||
let config: WebhookConfig = serde_json::from_str(json).unwrap();
|
||||
|
||||
assert_eq!(config.retry_count, 3);
|
||||
assert_eq!(config.timeout_ms, 5000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_event_type_name() {
|
||||
let handler = WebhookHandler::new(vec![]);
|
||||
|
||||
let event = Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
};
|
||||
assert_eq!(handler.event_type_name(&event), "sync_started");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_matches_filter_empty() {
|
||||
let handler = WebhookHandler::new(vec![]);
|
||||
let config = WebhookConfig {
|
||||
url: "http://example.com".to_string(),
|
||||
secret: None,
|
||||
events: vec![],
|
||||
retry_count: 3,
|
||||
timeout_ms: 5000,
|
||||
};
|
||||
|
||||
let event = Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
};
|
||||
assert!(handler.matches_filter(&event, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_matches_filter_specific() {
|
||||
let handler = WebhookHandler::new(vec![]);
|
||||
let config = WebhookConfig {
|
||||
url: "http://example.com".to_string(),
|
||||
secret: None,
|
||||
events: vec!["sync_started".to_string()],
|
||||
retry_count: 3,
|
||||
timeout_ms: 5000,
|
||||
};
|
||||
|
||||
let event = Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
};
|
||||
assert!(handler.matches_filter(&event, &config));
|
||||
|
||||
let event2 = Event::SyncCompleted {
|
||||
origin_id: OriginId::from("test"),
|
||||
files_changed: 0,
|
||||
};
|
||||
assert!(!handler.matches_filter(&event2, &config));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
[package]
|
||||
name = "musicfs-metadata"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
symphonia.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
image.workspace = true
|
||||
@@ -0,0 +1,116 @@
|
||||
use image::ImageFormat;
|
||||
use std::io::Cursor;
|
||||
use symphonia::core::meta::Visual;
|
||||
use tracing::debug;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Artwork {
|
||||
pub art_type: ArtType,
|
||||
pub mime_type: String,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ArtType {
|
||||
Front,
|
||||
Back,
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum ArtSize {
|
||||
Thumbnail,
|
||||
Medium,
|
||||
Full,
|
||||
}
|
||||
|
||||
impl ArtSize {
|
||||
pub fn max_dimension(&self) -> Option<u32> {
|
||||
match self {
|
||||
ArtSize::Thumbnail => Some(150),
|
||||
ArtSize::Medium => Some(300),
|
||||
ArtSize::Full => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ArtworkExtractor;
|
||||
|
||||
impl ArtworkExtractor {
|
||||
pub fn extract_from_visual(visual: &Visual) -> Option<Artwork> {
|
||||
let data = visual.data.to_vec();
|
||||
|
||||
let img = image::load_from_memory(&data).ok()?;
|
||||
|
||||
let art_type = match visual.usage {
|
||||
Some(symphonia::core::meta::StandardVisualKey::FrontCover) => ArtType::Front,
|
||||
Some(symphonia::core::meta::StandardVisualKey::BackCover) => ArtType::Back,
|
||||
_ => ArtType::Other,
|
||||
};
|
||||
|
||||
let mime_type = if visual.media_type.is_empty() {
|
||||
"image/jpeg".to_string()
|
||||
} else {
|
||||
visual.media_type.clone()
|
||||
};
|
||||
|
||||
Some(Artwork {
|
||||
art_type,
|
||||
mime_type,
|
||||
width: img.width(),
|
||||
height: img.height(),
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn resize(artwork: &Artwork, size: ArtSize) -> Option<Artwork> {
|
||||
let max_dim = size.max_dimension()?;
|
||||
|
||||
if artwork.width <= max_dim && artwork.height <= max_dim {
|
||||
return Some(artwork.clone());
|
||||
}
|
||||
|
||||
let img = image::load_from_memory(&artwork.data).ok()?;
|
||||
let resized = img.thumbnail(max_dim, max_dim);
|
||||
|
||||
let mut output = Vec::new();
|
||||
let mut cursor = Cursor::new(&mut output);
|
||||
resized.write_to(&mut cursor, ImageFormat::Jpeg).ok()?;
|
||||
|
||||
debug!(
|
||||
"Resized artwork from {}x{} to {}x{}",
|
||||
artwork.width,
|
||||
artwork.height,
|
||||
resized.width(),
|
||||
resized.height()
|
||||
);
|
||||
|
||||
Some(Artwork {
|
||||
art_type: artwork.art_type,
|
||||
mime_type: "image/jpeg".to_string(),
|
||||
width: resized.width(),
|
||||
height: resized.height(),
|
||||
data: output,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_art_size_dimensions() {
|
||||
assert_eq!(ArtSize::Thumbnail.max_dimension(), Some(150));
|
||||
assert_eq!(ArtSize::Medium.max_dimension(), Some(300));
|
||||
assert_eq!(ArtSize::Full.max_dimension(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_art_type_equality() {
|
||||
assert_eq!(ArtType::Front, ArtType::Front);
|
||||
assert_ne!(ArtType::Front, ArtType::Back);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod artwork;
|
||||
mod parser;
|
||||
|
||||
pub use artwork::{ArtSize, ArtType, Artwork, ArtworkExtractor};
|
||||
pub use parser::MetadataParser;
|
||||
@@ -0,0 +1,132 @@
|
||||
use musicfs_core::{AudioFormat, AudioMeta, Error, Result};
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use symphonia::core::codecs::CODEC_TYPE_NULL;
|
||||
use symphonia::core::formats::FormatOptions;
|
||||
use symphonia::core::io::MediaSourceStream;
|
||||
use symphonia::core::meta::MetadataOptions;
|
||||
use symphonia::core::probe::Hint;
|
||||
use tracing::debug;
|
||||
|
||||
pub struct MetadataParser;
|
||||
|
||||
impl MetadataParser {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
pub fn parse_file(&self, path: &Path) -> Result<AudioMeta> {
|
||||
let file = File::open(path)?;
|
||||
let mss = MediaSourceStream::new(Box::new(file), Default::default());
|
||||
|
||||
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
|
||||
|
||||
let mut hint = Hint::new();
|
||||
if !ext.is_empty() {
|
||||
hint.with_extension(ext);
|
||||
}
|
||||
|
||||
let fmt_opts = FormatOptions::default();
|
||||
let meta_opts = MetadataOptions::default();
|
||||
|
||||
let probed = symphonia::default::get_probe()
|
||||
.format(&hint, mss, &fmt_opts, &meta_opts)
|
||||
.map_err(|e| Error::Metadata(format!("Failed to probe format: {}", e)))?;
|
||||
let mut format = probed.format;
|
||||
|
||||
let mut audio_meta = AudioMeta {
|
||||
format: AudioFormat::from_extension(ext),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Some(metadata) = format.metadata().current() {
|
||||
self.extract_tags(&mut audio_meta, metadata);
|
||||
}
|
||||
|
||||
if let Some(track) = format
|
||||
.tracks()
|
||||
.iter()
|
||||
.find(|t| t.codec_params.codec != CODEC_TYPE_NULL)
|
||||
{
|
||||
let params = &track.codec_params;
|
||||
|
||||
if let Some(n_frames) = params.n_frames {
|
||||
if let Some(sample_rate) = params.sample_rate {
|
||||
audio_meta.duration_ms = Some((n_frames as u64 * 1000) / sample_rate as u64);
|
||||
audio_meta.sample_rate = Some(sample_rate);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(bits_per_sample) = params.bits_per_sample {
|
||||
if let Some(sample_rate) = params.sample_rate {
|
||||
if let Some(channels) = params.channels {
|
||||
audio_meta.bitrate =
|
||||
Some(bits_per_sample * sample_rate * channels.count() as u32 / 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!(?audio_meta, "Parsed metadata");
|
||||
Ok(audio_meta)
|
||||
}
|
||||
|
||||
fn extract_tags(
|
||||
&self,
|
||||
meta: &mut AudioMeta,
|
||||
metadata: &symphonia::core::meta::MetadataRevision,
|
||||
) {
|
||||
use symphonia::core::meta::StandardTagKey;
|
||||
|
||||
for tag in metadata.tags() {
|
||||
if let Some(std_key) = tag.std_key {
|
||||
let value = tag.value.to_string();
|
||||
match std_key {
|
||||
StandardTagKey::TrackTitle => meta.title = Some(value),
|
||||
StandardTagKey::Artist => meta.artist = Some(value),
|
||||
StandardTagKey::Album => meta.album = Some(value),
|
||||
StandardTagKey::AlbumArtist => meta.album_artist = Some(value),
|
||||
StandardTagKey::Genre => meta.genre = Some(value),
|
||||
StandardTagKey::TrackNumber => {
|
||||
meta.track = value.split('/').next().and_then(|s| s.parse().ok());
|
||||
}
|
||||
StandardTagKey::DiscNumber => {
|
||||
meta.disc = value.split('/').next().and_then(|s| s.parse().ok());
|
||||
}
|
||||
StandardTagKey::Date | StandardTagKey::ReleaseDate => {
|
||||
meta.year = value.chars().take(4).collect::<String>().parse().ok();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MetadataParser {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_audio_format_detection() {
|
||||
assert_eq!(AudioFormat::from_extension("flac"), AudioFormat::Flac);
|
||||
assert_eq!(AudioFormat::from_extension("mp3"), AudioFormat::Mp3);
|
||||
assert_eq!(AudioFormat::from_extension("opus"), AudioFormat::Opus);
|
||||
assert_eq!(AudioFormat::from_extension("ogg"), AudioFormat::Vorbis);
|
||||
assert_eq!(AudioFormat::from_extension("m4a"), AudioFormat::Aac);
|
||||
assert_eq!(AudioFormat::from_extension("wav"), AudioFormat::Wav);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parser_creation() {
|
||||
let parser = MetadataParser::new();
|
||||
let default_parser = MetadataParser::default();
|
||||
assert!(std::mem::size_of_val(&parser) == std::mem::size_of_val(&default_parser));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "musicfs-origins"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
s3 = []
|
||||
sftp = []
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
async-trait.workspace = true
|
||||
dashmap.workspace = true
|
||||
futures.workspace = true
|
||||
libc.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "sync", "time"] }
|
||||
tracing.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -0,0 +1,226 @@
|
||||
use crate::registry::OriginRegistry;
|
||||
use crate::traits::Origin;
|
||||
use musicfs_core::{Error, RealPath, Result};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tracing::{trace, warn};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RetryConfig {
|
||||
pub max_attempts: u32,
|
||||
pub delays: Vec<Duration>,
|
||||
}
|
||||
|
||||
impl Default for RetryConfig {
|
||||
fn default() -> Self {
|
||||
Self::spec_compliant()
|
||||
}
|
||||
}
|
||||
|
||||
impl RetryConfig {
|
||||
pub fn spec_compliant() -> Self {
|
||||
Self {
|
||||
max_attempts: 3,
|
||||
delays: vec![
|
||||
Duration::from_millis(100),
|
||||
Duration::from_millis(500),
|
||||
Duration::from_millis(2000),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_delays(delays: Vec<Duration>) -> Self {
|
||||
Self {
|
||||
max_attempts: delays.len() as u32,
|
||||
delays,
|
||||
}
|
||||
}
|
||||
|
||||
fn delay_for_attempt(&self, attempt: u32) -> Duration {
|
||||
self.delays
|
||||
.get(attempt as usize)
|
||||
.copied()
|
||||
.unwrap_or(*self.delays.last().unwrap_or(&Duration::from_millis(100)))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FailoverExecutor {
|
||||
registry: Arc<OriginRegistry>,
|
||||
retry_config: RetryConfig,
|
||||
}
|
||||
|
||||
impl FailoverExecutor {
|
||||
pub fn new(registry: Arc<OriginRegistry>, retry_config: RetryConfig) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
retry_config,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn read_with_failover(
|
||||
&self,
|
||||
path: &RealPath,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> Result<Vec<u8>> {
|
||||
let origins = self.registry.route_all(path);
|
||||
|
||||
if origins.is_empty() {
|
||||
if let Some(origin) = self.registry.route_with_fallback(path) {
|
||||
warn!("No healthy origins, using fallback origin {}", origin.id());
|
||||
return self
|
||||
.read_with_retry(&origin, &path.path, offset, size)
|
||||
.await;
|
||||
}
|
||||
return Err(Error::NoOriginAvailable);
|
||||
}
|
||||
|
||||
let mut last_error = None;
|
||||
|
||||
for origin in origins {
|
||||
trace!(origin_id = %origin.id(), "Attempting read from origin");
|
||||
let start = std::time::Instant::now();
|
||||
match self
|
||||
.read_with_retry(&origin, &path.path, offset, size)
|
||||
.await
|
||||
{
|
||||
Ok(data) => {
|
||||
let latency = start.elapsed().as_millis() as u64;
|
||||
self.registry.record_latency(origin.id(), latency);
|
||||
return Ok(data);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(origin_id = %origin.id(), error = %e, "Origin failed, trying next");
|
||||
last_error = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_error.unwrap_or(Error::NoOriginAvailable))
|
||||
}
|
||||
|
||||
async fn read_with_retry(
|
||||
&self,
|
||||
origin: &Arc<dyn Origin>,
|
||||
path: &std::path::Path,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> Result<Vec<u8>> {
|
||||
for attempt in 0..self.retry_config.max_attempts {
|
||||
match origin.read(path, offset, size).await {
|
||||
Ok(data) => return Ok(data),
|
||||
Err(e) if attempt + 1 < self.retry_config.max_attempts => {
|
||||
let delay = self.retry_config.delay_for_attempt(attempt);
|
||||
warn!(
|
||||
origin_id = %origin.id(),
|
||||
attempt = attempt + 1,
|
||||
max_attempts = self.retry_config.max_attempts,
|
||||
error = %e,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Retrying read operation"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::MaxRetriesExceeded)
|
||||
}
|
||||
|
||||
pub async fn read_full_with_failover(&self, path: &RealPath) -> Result<Vec<u8>> {
|
||||
let origins = self.registry.route_all(path);
|
||||
|
||||
if origins.is_empty() {
|
||||
if let Some(origin) = self.registry.route_with_fallback(path) {
|
||||
warn!(
|
||||
"No healthy origins for full read, using fallback {}",
|
||||
origin.id()
|
||||
);
|
||||
return self.read_full_with_retry(&origin, &path.path).await;
|
||||
}
|
||||
return Err(Error::NoOriginAvailable);
|
||||
}
|
||||
|
||||
let mut last_error = None;
|
||||
|
||||
for origin in origins {
|
||||
trace!(origin_id = %origin.id(), "Attempting full read from origin");
|
||||
let start = std::time::Instant::now();
|
||||
match self.read_full_with_retry(&origin, &path.path).await {
|
||||
Ok(data) => {
|
||||
let latency = start.elapsed().as_millis() as u64;
|
||||
self.registry.record_latency(origin.id(), latency);
|
||||
return Ok(data);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(origin_id = %origin.id(), error = %e, "Origin failed full read, trying next");
|
||||
last_error = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_error.unwrap_or(Error::NoOriginAvailable))
|
||||
}
|
||||
|
||||
async fn read_full_with_retry(
|
||||
&self,
|
||||
origin: &Arc<dyn Origin>,
|
||||
path: &std::path::Path,
|
||||
) -> Result<Vec<u8>> {
|
||||
for attempt in 0..self.retry_config.max_attempts {
|
||||
match origin.read_full(path).await {
|
||||
Ok(data) => return Ok(data),
|
||||
Err(e) if attempt + 1 < self.retry_config.max_attempts => {
|
||||
let delay = self.retry_config.delay_for_attempt(attempt);
|
||||
warn!(
|
||||
origin_id = %origin.id(),
|
||||
attempt = attempt + 1,
|
||||
max_attempts = self.retry_config.max_attempts,
|
||||
error = %e,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Retrying full read operation"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::MaxRetriesExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_retry_config_default() {
|
||||
let config = RetryConfig::default();
|
||||
assert_eq!(config.max_attempts, 3);
|
||||
assert_eq!(config.delays[0], Duration::from_millis(100));
|
||||
assert_eq!(config.delays[1], Duration::from_millis(500));
|
||||
assert_eq!(config.delays[2], Duration::from_millis(2000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delay_for_attempt() {
|
||||
let config = RetryConfig::spec_compliant();
|
||||
|
||||
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(100));
|
||||
assert_eq!(config.delay_for_attempt(1), Duration::from_millis(500));
|
||||
assert_eq!(config.delay_for_attempt(2), Duration::from_millis(2000));
|
||||
assert_eq!(config.delay_for_attempt(10), Duration::from_millis(2000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_delays() {
|
||||
let config =
|
||||
RetryConfig::with_delays(vec![Duration::from_millis(50), Duration::from_millis(100)]);
|
||||
|
||||
assert_eq!(config.max_attempts, 2);
|
||||
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(50));
|
||||
assert_eq!(config.delay_for_attempt(1), Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
use crate::traits::Origin;
|
||||
use dashmap::DashMap;
|
||||
use futures::future::join_all;
|
||||
use musicfs_core::{Event, EventBus, HealthStatus, OriginId, OriginType};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, info_span, warn, Instrument};
|
||||
|
||||
pub struct HealthMonitor {
|
||||
origins: DashMap<OriginId, Arc<dyn Origin>>,
|
||||
state: DashMap<OriginId, OriginHealthState>,
|
||||
check_interval: Duration,
|
||||
default_threshold: u32,
|
||||
per_type_thresholds: HashMap<OriginType, u32>,
|
||||
event_bus: Option<Arc<EventBus>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OriginHealthState {
|
||||
pub status: HealthStatus,
|
||||
pub last_check: Instant,
|
||||
pub consecutive_failures: u32,
|
||||
pub last_latency_ms: Option<u64>,
|
||||
}
|
||||
|
||||
impl Default for OriginHealthState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
status: HealthStatus::Unknown,
|
||||
last_check: Instant::now(),
|
||||
consecutive_failures: 0,
|
||||
last_latency_ms: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealthSnapshot {
|
||||
pub healthy: Vec<OriginId>,
|
||||
pub degraded: Vec<OriginId>,
|
||||
pub unhealthy: Vec<OriginId>,
|
||||
pub failure_counts: HashMap<OriginId, u32>,
|
||||
}
|
||||
|
||||
impl HealthSnapshot {
|
||||
pub fn is_healthy(&self, id: &OriginId) -> bool {
|
||||
self.healthy.contains(id)
|
||||
}
|
||||
|
||||
pub fn is_degraded(&self, id: &OriginId) -> bool {
|
||||
self.degraded.contains(id)
|
||||
}
|
||||
|
||||
pub fn is_unhealthy(&self, id: &OriginId) -> bool {
|
||||
self.unhealthy.contains(id)
|
||||
}
|
||||
|
||||
pub fn failure_count(&self, id: &OriginId) -> Option<u32> {
|
||||
self.failure_counts.get(id).copied()
|
||||
}
|
||||
|
||||
pub fn all_unhealthy(&self) -> bool {
|
||||
self.healthy.is_empty() && self.degraded.is_empty()
|
||||
}
|
||||
|
||||
pub fn total_candidates(&self) -> usize {
|
||||
self.healthy.len() + self.degraded.len() + self.unhealthy.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthMonitor {
|
||||
pub fn new(check_interval: Duration) -> Self {
|
||||
let mut per_type = HashMap::new();
|
||||
per_type.insert(OriginType::Local, 1);
|
||||
per_type.insert(OriginType::Nfs, 3);
|
||||
per_type.insert(OriginType::Smb, 3);
|
||||
per_type.insert(OriginType::S3, 3);
|
||||
per_type.insert(OriginType::Sftp, 3);
|
||||
|
||||
Self {
|
||||
origins: DashMap::new(),
|
||||
state: DashMap::new(),
|
||||
check_interval,
|
||||
default_threshold: 3,
|
||||
per_type_thresholds: per_type,
|
||||
event_bus: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_threshold(mut self, threshold: u32) -> Self {
|
||||
self.default_threshold = threshold;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_per_type_thresholds(mut self, thresholds: HashMap<OriginType, u32>) -> Self {
|
||||
self.per_type_thresholds = thresholds;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_event_bus(mut self, bus: Arc<EventBus>) -> Self {
|
||||
self.event_bus = Some(bus);
|
||||
self
|
||||
}
|
||||
|
||||
fn threshold_for(&self, origin_type: OriginType) -> u32 {
|
||||
self.per_type_thresholds
|
||||
.get(&origin_type)
|
||||
.copied()
|
||||
.unwrap_or(self.default_threshold)
|
||||
}
|
||||
|
||||
pub fn add_origin(&self, origin: Arc<dyn Origin>) {
|
||||
let id = origin.id().clone();
|
||||
self.origins.insert(id.clone(), origin);
|
||||
self.state.insert(id, OriginHealthState::default());
|
||||
}
|
||||
|
||||
pub fn remove_origin(&self, id: &OriginId) {
|
||||
self.origins.remove(id);
|
||||
self.state.remove(id);
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> HealthSnapshot {
|
||||
let mut healthy = Vec::new();
|
||||
let mut degraded = Vec::new();
|
||||
let mut unhealthy = Vec::new();
|
||||
let mut failure_counts = HashMap::new();
|
||||
|
||||
for entry in self.state.iter() {
|
||||
let id = entry.key().clone();
|
||||
failure_counts.insert(id.clone(), entry.value().consecutive_failures);
|
||||
|
||||
match entry.value().status {
|
||||
HealthStatus::Healthy => healthy.push(id),
|
||||
HealthStatus::Degraded => degraded.push(id),
|
||||
HealthStatus::Unhealthy => unhealthy.push(id),
|
||||
HealthStatus::Unknown => degraded.push(id),
|
||||
}
|
||||
}
|
||||
|
||||
HealthSnapshot {
|
||||
healthy,
|
||||
degraded,
|
||||
unhealthy,
|
||||
failure_counts,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(self: Arc<Self>) -> HealthCheckHandle {
|
||||
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
|
||||
let monitor = self.clone();
|
||||
let interval_secs = monitor.check_interval.as_secs();
|
||||
|
||||
info!(
|
||||
interval_secs = interval_secs,
|
||||
origin_count = monitor.origins.len(),
|
||||
"Health monitor starting"
|
||||
);
|
||||
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let mut interval = tokio::time::interval(monitor.check_interval);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
monitor.check_all().await;
|
||||
}
|
||||
_ = stop_rx.recv() => {
|
||||
info!("Health monitor stopping");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.instrument(info_span!("health_monitor")),
|
||||
);
|
||||
|
||||
HealthCheckHandle { stop_tx }
|
||||
}
|
||||
|
||||
pub async fn check_all(&self) {
|
||||
let origins: Vec<_> = self
|
||||
.origins
|
||||
.iter()
|
||||
.map(|e| (e.key().clone(), e.value().clone()))
|
||||
.collect();
|
||||
|
||||
let checks: Vec<_> = origins
|
||||
.iter()
|
||||
.map(|(id, origin)| self.check_one(id, origin))
|
||||
.collect();
|
||||
|
||||
join_all(checks).await;
|
||||
}
|
||||
|
||||
async fn check_one(&self, id: &OriginId, origin: &Arc<dyn Origin>) {
|
||||
let start = Instant::now();
|
||||
let health_timeout = Duration::from_millis(1500);
|
||||
|
||||
let status = match tokio::time::timeout(health_timeout, origin.health()).await {
|
||||
Ok(status) => status,
|
||||
Err(_) => {
|
||||
warn!(
|
||||
origin_id = %id,
|
||||
timeout_ms = health_timeout.as_millis() as u64,
|
||||
"Health check timed out"
|
||||
);
|
||||
HealthStatus::Unhealthy
|
||||
}
|
||||
};
|
||||
|
||||
let latency_ms = start.elapsed().as_millis() as u64;
|
||||
|
||||
let threshold = self.threshold_for(origin.origin_type());
|
||||
let prev_healthy = self
|
||||
.state
|
||||
.get(id)
|
||||
.map(|s| s.status == HealthStatus::Healthy)
|
||||
.unwrap_or(false);
|
||||
|
||||
let mut state = self.state.entry(id.clone()).or_default();
|
||||
|
||||
match status {
|
||||
HealthStatus::Healthy => {
|
||||
if state.status != HealthStatus::Healthy {
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to healthy"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Healthy;
|
||||
state.consecutive_failures = 0;
|
||||
}
|
||||
HealthStatus::Degraded => {
|
||||
if state.status != HealthStatus::Degraded {
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to degraded"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Degraded;
|
||||
}
|
||||
HealthStatus::Unhealthy => {
|
||||
state.consecutive_failures += 1;
|
||||
if state.consecutive_failures >= threshold {
|
||||
if state.status != HealthStatus::Unhealthy {
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
consecutive_failures = state.consecutive_failures,
|
||||
threshold = threshold,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to unhealthy"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Unhealthy;
|
||||
} else {
|
||||
debug!(
|
||||
origin_id = %id,
|
||||
consecutive_failures = state.consecutive_failures,
|
||||
threshold = threshold,
|
||||
"Origin health check failed"
|
||||
);
|
||||
state.status = HealthStatus::Degraded;
|
||||
}
|
||||
}
|
||||
HealthStatus::Unknown => {
|
||||
state.status = HealthStatus::Unknown;
|
||||
}
|
||||
}
|
||||
|
||||
state.last_check = Instant::now();
|
||||
state.last_latency_ms = Some(latency_ms);
|
||||
|
||||
let now_healthy = state.status == HealthStatus::Healthy;
|
||||
if prev_healthy != now_healthy {
|
||||
if let Some(bus) = &self.event_bus {
|
||||
bus.publish(Event::OriginHealthChanged {
|
||||
origin_id: id.clone(),
|
||||
healthy: now_healthy,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn check_now(&self, id: &OriginId) {
|
||||
if let Some(origin) = self.origins.get(id) {
|
||||
self.check_one(id, &origin.clone()).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_state(&self, id: &OriginId) -> Option<OriginHealthState> {
|
||||
self.state.get(id).map(|e| e.value().clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HealthCheckHandle {
|
||||
stop_tx: mpsc::Sender<()>,
|
||||
}
|
||||
|
||||
impl HealthCheckHandle {
|
||||
pub async fn stop(self) {
|
||||
let _ = self.stop_tx.send(()).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::LocalOrigin;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_health_monitor_basic() {
|
||||
let monitor = HealthMonitor::new(Duration::from_secs(30));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
|
||||
|
||||
monitor.add_origin(origin);
|
||||
|
||||
let snapshot = monitor.snapshot();
|
||||
assert!(!snapshot.is_healthy(&OriginId::from("test")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_health_check() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
|
||||
|
||||
monitor.add_origin(origin);
|
||||
monitor.check_now(&OriginId::from("test")).await;
|
||||
|
||||
let snapshot = monitor.snapshot();
|
||||
assert!(snapshot.is_healthy(&OriginId::from("test")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_failure_tracking() {
|
||||
let mut thresholds = HashMap::new();
|
||||
thresholds.insert(OriginType::Local, 3);
|
||||
|
||||
let monitor =
|
||||
HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds);
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
"missing",
|
||||
std::path::Path::new("/nonexistent"),
|
||||
));
|
||||
monitor.add_origin(origin);
|
||||
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
assert_eq!(state.status, HealthStatus::Degraded);
|
||||
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
|
||||
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 3);
|
||||
assert_eq!(state.status, HealthStatus::Unhealthy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_threshold_is_one() {
|
||||
let monitor = HealthMonitor::new(Duration::from_secs(30));
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
"missing",
|
||||
std::path::Path::new("/nonexistent"),
|
||||
));
|
||||
monitor.add_origin(origin);
|
||||
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
assert_eq!(state.status, HealthStatus::Unhealthy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_snapshot_all_unhealthy() {
|
||||
let snapshot = HealthSnapshot {
|
||||
healthy: Vec::new(),
|
||||
degraded: Vec::new(),
|
||||
unhealthy: vec![OriginId::from("a")],
|
||||
failure_counts: HashMap::new(),
|
||||
};
|
||||
assert!(snapshot.all_unhealthy());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
mod failover;
|
||||
mod health;
|
||||
mod local;
|
||||
mod nfs;
|
||||
mod registry;
|
||||
mod router;
|
||||
mod s3;
|
||||
mod sftp;
|
||||
mod smb;
|
||||
mod traits;
|
||||
|
||||
pub use failover::{FailoverExecutor, RetryConfig};
|
||||
pub use health::{HealthCheckHandle, HealthMonitor, HealthSnapshot, OriginHealthState};
|
||||
pub use local::LocalOrigin;
|
||||
pub use musicfs_core::OriginType;
|
||||
pub use nfs::NfsOrigin;
|
||||
pub use registry::OriginRegistry;
|
||||
pub use router::{LatencyStats, Router};
|
||||
pub use smb::SmbOrigin;
|
||||
pub use traits::{Origin, WatchCallback, WatchEvent, WatchHandle};
|
||||
@@ -0,0 +1,218 @@
|
||||
use crate::traits::{Origin, WatchCallback, WatchHandle};
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::fs;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::debug;
|
||||
|
||||
pub struct LocalOrigin {
|
||||
id: OriginId,
|
||||
root: PathBuf,
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
impl LocalOrigin {
|
||||
pub fn new(id: impl Into<OriginId>, root: impl Into<PathBuf>) -> Self {
|
||||
let root = root.into();
|
||||
let display_name = format!("Local: {}", root.display());
|
||||
Self {
|
||||
id: id.into(),
|
||||
root,
|
||||
display_name,
|
||||
}
|
||||
}
|
||||
|
||||
fn full_path(&self, path: &Path) -> PathBuf {
|
||||
if path.as_os_str().is_empty() || path == Path::new("/") {
|
||||
self.root.clone()
|
||||
} else {
|
||||
self.root.join(path.strip_prefix("/").unwrap_or(path))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for LocalOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
&self.id
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
OriginType::Local
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
&self.display_name
|
||||
}
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
|
||||
let full_path = self.full_path(path);
|
||||
debug!("LocalOrigin::readdir({:?})", full_path);
|
||||
|
||||
let mut entries = Vec::new();
|
||||
let mut dir = fs::read_dir(&full_path).await?;
|
||||
|
||||
while let Some(entry) = dir.next_entry().await? {
|
||||
let metadata = entry.metadata().await?;
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
|
||||
entries.push(DirEntry {
|
||||
name,
|
||||
is_dir: metadata.is_dir(),
|
||||
size: metadata.len(),
|
||||
mtime: metadata.modified().unwrap_or(std::time::UNIX_EPOCH),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat> {
|
||||
let full_path = self.full_path(path);
|
||||
debug!("LocalOrigin::stat({:?})", full_path);
|
||||
|
||||
let metadata = fs::metadata(&full_path).await?;
|
||||
|
||||
Ok(FileStat {
|
||||
size: metadata.len(),
|
||||
mtime: metadata.modified().unwrap_or(std::time::UNIX_EPOCH),
|
||||
is_dir: metadata.is_dir(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||
|
||||
let full_path = self.full_path(path);
|
||||
debug!(
|
||||
"LocalOrigin::read({:?}, offset={}, size={})",
|
||||
full_path, offset, size
|
||||
);
|
||||
|
||||
let mut file = fs::File::open(&full_path).await?;
|
||||
file.seek(std::io::SeekFrom::Start(offset)).await?;
|
||||
|
||||
// FIX: Loop until all requested bytes are read or EOF
|
||||
// Single read() only returns kernel buffer (~2MB), not full request
|
||||
let mut buffer = Vec::with_capacity(size as usize);
|
||||
let mut temp_buf = vec![0u8; 64 * 1024]; // 64KB chunks
|
||||
let mut total_read = 0usize;
|
||||
|
||||
while total_read < size as usize {
|
||||
let to_read = std::cmp::min(temp_buf.len(), size as usize - total_read);
|
||||
let n = file.read(&mut temp_buf[..to_read]).await?;
|
||||
if n == 0 {
|
||||
break; // EOF
|
||||
}
|
||||
buffer.extend_from_slice(&temp_buf[..n]);
|
||||
total_read += n;
|
||||
}
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
let full_path = self.full_path(path);
|
||||
debug!("LocalOrigin::read_full({:?})", full_path);
|
||||
Ok(fs::read(&full_path).await?)
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
let full_path = self.full_path(path);
|
||||
Ok(fs::try_exists(&full_path).await?)
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
match fs::try_exists(&self.root).await {
|
||||
Ok(true) => HealthStatus::Healthy,
|
||||
Ok(false) => HealthStatus::Unhealthy,
|
||||
Err(_) => HealthStatus::Unhealthy,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
|
||||
let full_path = self.full_path(path);
|
||||
let file = fs::File::open(&full_path).await?;
|
||||
Ok(Box::new(file))
|
||||
}
|
||||
|
||||
async fn watch(&self, path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
|
||||
debug!("LocalOrigin::watch({:?}) - stub implementation", path);
|
||||
let (tx, _rx) = tokio::sync::oneshot::channel();
|
||||
Ok(WatchHandle::new(tx))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_readdir() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello").unwrap();
|
||||
std::fs::create_dir(dir.path().join("subdir")).unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
let entries = origin.readdir(Path::new("/")).await.unwrap();
|
||||
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert!(entries.iter().any(|e| e.name == "test.txt" && !e.is_dir));
|
||||
assert!(entries.iter().any(|e| e.name == "subdir" && e.is_dir));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_stat() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
let stat = origin.stat(Path::new("/test.txt")).await.unwrap();
|
||||
|
||||
assert_eq!(stat.size, 11);
|
||||
assert!(!stat.is_dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_read() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
let data = origin.read(Path::new("/test.txt"), 0, 5).await.unwrap();
|
||||
|
||||
assert_eq!(data, b"hello");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_read_offset() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
let data = origin.read(Path::new("/test.txt"), 6, 5).await.unwrap();
|
||||
|
||||
assert_eq!(data, b"world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_exists() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello").unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
|
||||
assert!(origin.exists(Path::new("/test.txt")).await.unwrap());
|
||||
assert!(!origin.exists(Path::new("/nonexistent.txt")).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_health() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
|
||||
assert_eq!(origin.health().await, HealthStatus::Healthy);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
use crate::local::LocalOrigin;
|
||||
use crate::traits::{Origin, WatchCallback, WatchHandle};
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
pub struct NfsOrigin {
|
||||
inner: LocalOrigin,
|
||||
max_retries: u32,
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
impl NfsOrigin {
|
||||
pub fn new(id: impl Into<OriginId>, mount_point: impl Into<PathBuf>) -> Self {
|
||||
let mount_point = mount_point.into();
|
||||
let display_name = format!("NFS: {}", mount_point.display());
|
||||
|
||||
Self {
|
||||
inner: LocalOrigin::new(id, &mount_point),
|
||||
max_retries: 3,
|
||||
display_name,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_max_retries(mut self, retries: u32) -> Self {
|
||||
self.max_retries = retries;
|
||||
self
|
||||
}
|
||||
|
||||
async fn retry_on_stale<T, F, Fut>(&self, op: F) -> Result<T>
|
||||
where
|
||||
F: Fn() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
let mut delay = Duration::from_millis(100);
|
||||
|
||||
for attempt in 0..self.max_retries {
|
||||
match op().await {
|
||||
Ok(result) => return Ok(result),
|
||||
Err(e) => {
|
||||
if let Some(io_err) = e.downcast_io() {
|
||||
#[cfg(unix)]
|
||||
if io_err.raw_os_error() == Some(libc::ESTALE) {
|
||||
warn!(
|
||||
"NFS stale handle (attempt {}/{}), retrying after {:?}",
|
||||
attempt + 1,
|
||||
self.max_retries,
|
||||
delay
|
||||
);
|
||||
sleep(delay).await;
|
||||
delay *= 2;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(musicfs_core::Error::NfsStaleHandle)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for NfsOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
self.inner.id()
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
OriginType::Nfs
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
&self.display_name
|
||||
}
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
|
||||
self.retry_on_stale(|| self.inner.readdir(path)).await
|
||||
}
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat> {
|
||||
self.retry_on_stale(|| self.inner.stat(path)).await
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
self.retry_on_stale(|| self.inner.read(path, offset, size))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
self.retry_on_stale(|| self.inner.read_full(path)).await
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
self.retry_on_stale(|| self.inner.exists(path)).await
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
let health_timeout = Duration::from_secs(5);
|
||||
match tokio::time::timeout(health_timeout, self.inner.stat(Path::new("/"))).await {
|
||||
Ok(Ok(_)) => HealthStatus::Healthy,
|
||||
Ok(Err(_)) | Err(_) => HealthStatus::Unhealthy,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn tokio::io::AsyncRead + Send + Unpin>> {
|
||||
self.inner.open_read(path).await
|
||||
}
|
||||
|
||||
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
|
||||
debug!("NFS watch - inotify may be unreliable over NFS, consider polling");
|
||||
self.inner.watch(path, callback).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nfs_origin_basic() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.flac"), b"audio").unwrap();
|
||||
|
||||
let origin = NfsOrigin::new("nfs-test", dir.path());
|
||||
|
||||
let entries = origin.readdir(Path::new("/")).await.unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
|
||||
let data = origin.read(Path::new("/test.flac"), 0, 5).await.unwrap();
|
||||
assert_eq!(&data, b"audio");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nfs_origin_health() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = NfsOrigin::new("nfs-test", dir.path());
|
||||
|
||||
assert_eq!(origin.health().await, HealthStatus::Healthy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nfs_origin_type() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = NfsOrigin::new("nfs-test", dir.path());
|
||||
|
||||
assert_eq!(origin.origin_type(), OriginType::Nfs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retry_uses_fn_not_fnmut() {
|
||||
fn assert_fn<F: Fn() -> Fut, Fut>(_: F) {}
|
||||
|
||||
let closure = || async { Ok::<_, musicfs_core::Error>(()) };
|
||||
assert_fn(closure);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
use crate::health::{HealthMonitor, HealthSnapshot};
|
||||
use crate::router::Router;
|
||||
use crate::traits::{Origin, WatchHandle};
|
||||
use musicfs_core::{OriginId, RealPath};
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub struct OriginRegistry {
|
||||
origins: RwLock<HashMap<OriginId, Arc<dyn Origin>>>,
|
||||
router: Router,
|
||||
health_monitor: Arc<HealthMonitor>,
|
||||
watch_handles: RwLock<HashMap<OriginId, Vec<WatchHandle>>>,
|
||||
}
|
||||
|
||||
impl OriginRegistry {
|
||||
pub fn new(health_monitor: Arc<HealthMonitor>) -> Self {
|
||||
Self {
|
||||
origins: RwLock::new(HashMap::new()),
|
||||
router: Router::new(),
|
||||
health_monitor,
|
||||
watch_handles: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register(&self, origin: Arc<dyn Origin>, priority: u8) {
|
||||
let id = origin.id().clone();
|
||||
info!("Registering origin {} with priority {}", id, priority);
|
||||
|
||||
self.router.set_priority(id.clone(), priority);
|
||||
self.health_monitor.add_origin(origin.clone());
|
||||
self.origins.write().insert(id, origin);
|
||||
}
|
||||
|
||||
pub fn unregister(&self, id: &OriginId) {
|
||||
info!("Unregistering origin {}", id);
|
||||
|
||||
if let Some(handles) = self.watch_handles.write().remove(id) {
|
||||
info!("Dropping {} watch handles for origin {}", handles.len(), id);
|
||||
}
|
||||
|
||||
self.origins.write().remove(id);
|
||||
self.router.remove_priority(id);
|
||||
self.health_monitor.remove_origin(id);
|
||||
}
|
||||
|
||||
pub fn register_watch(&self, origin_id: &OriginId, handle: WatchHandle) {
|
||||
self.watch_handles
|
||||
.write()
|
||||
.entry(origin_id.clone())
|
||||
.or_default()
|
||||
.push(handle);
|
||||
}
|
||||
|
||||
pub fn get(&self, id: &OriginId) -> Option<Arc<dyn Origin>> {
|
||||
self.origins.read().get(id).cloned()
|
||||
}
|
||||
|
||||
pub fn list(&self) -> Vec<Arc<dyn Origin>> {
|
||||
self.origins.read().values().cloned().collect()
|
||||
}
|
||||
|
||||
pub fn route(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let candidates: Vec<_> = origins
|
||||
.iter()
|
||||
.filter(|(id, _)| self.can_serve(id, path))
|
||||
.map(|(id, origin)| (id.clone(), origin.clone()))
|
||||
.collect();
|
||||
|
||||
if candidates.is_empty() {
|
||||
warn!("No origin can serve path: {:?}", path);
|
||||
return None;
|
||||
}
|
||||
|
||||
let candidate_ids: Vec<_> = candidates.iter().map(|(id, _)| id.clone()).collect();
|
||||
let selected = self.router.select(&candidate_ids, &health)?;
|
||||
|
||||
candidates
|
||||
.into_iter()
|
||||
.find(|(id, _)| id == &selected)
|
||||
.map(|(_, origin)| origin)
|
||||
}
|
||||
|
||||
pub fn route_with_fallback(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let candidates: Vec<_> = origins
|
||||
.iter()
|
||||
.filter(|(id, _)| self.can_serve(id, path))
|
||||
.map(|(id, origin)| (id.clone(), origin.clone()))
|
||||
.collect();
|
||||
|
||||
if candidates.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let candidate_ids: Vec<_> = candidates.iter().map(|(id, _)| id.clone()).collect();
|
||||
let selected = self.router.select_with_fallback(&candidate_ids, &health)?;
|
||||
|
||||
candidates
|
||||
.into_iter()
|
||||
.find(|(id, _)| id == &selected)
|
||||
.map(|(_, origin)| origin)
|
||||
}
|
||||
|
||||
pub fn route_all(&self, path: &RealPath) -> Vec<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let mut result: Vec<_> = origins
|
||||
.iter()
|
||||
.filter(|(id, _)| self.can_serve(id, path) && health.is_healthy(id))
|
||||
.map(|(_, origin)| origin.clone())
|
||||
.collect();
|
||||
|
||||
result.sort_by_key(|o| self.router.get_priority(o.id()));
|
||||
result
|
||||
}
|
||||
|
||||
fn can_serve(&self, origin_id: &OriginId, path: &RealPath) -> bool {
|
||||
path.origin_id == *origin_id
|
||||
}
|
||||
|
||||
pub fn health(&self) -> HealthSnapshot {
|
||||
self.health_monitor.snapshot()
|
||||
}
|
||||
|
||||
pub fn record_latency(&self, id: &OriginId, latency_ms: u64) {
|
||||
self.router.record_latency(id, latency_ms);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::LocalOrigin;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_register_and_get() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
let registry = OriginRegistry::new(monitor);
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
|
||||
|
||||
registry.register(origin.clone(), 1);
|
||||
|
||||
let retrieved = registry.get(&OriginId::from("test"));
|
||||
assert!(retrieved.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unregister() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
let registry = OriginRegistry::new(monitor);
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
|
||||
|
||||
registry.register(origin, 1);
|
||||
registry.unregister(&OriginId::from("test"));
|
||||
|
||||
assert!(registry.get(&OriginId::from("test")).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_route_by_priority() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
let registry = OriginRegistry::new(monitor.clone());
|
||||
|
||||
let dir1 = TempDir::new().unwrap();
|
||||
let dir2 = TempDir::new().unwrap();
|
||||
|
||||
let origin1 = Arc::new(LocalOrigin::new("primary", dir1.path()));
|
||||
let origin2 = Arc::new(LocalOrigin::new("backup", dir2.path()));
|
||||
|
||||
registry.register(origin1, 1);
|
||||
registry.register(origin2, 2);
|
||||
|
||||
monitor.check_now(&OriginId::from("primary")).await;
|
||||
monitor.check_now(&OriginId::from("backup")).await;
|
||||
|
||||
let path = RealPath {
|
||||
origin_id: OriginId::from("primary"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
};
|
||||
|
||||
let routed = registry.route(&path);
|
||||
assert!(routed.is_some());
|
||||
assert_eq!(routed.unwrap().id(), &OriginId::from("primary"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_origins() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
let registry = OriginRegistry::new(monitor);
|
||||
|
||||
let dir1 = TempDir::new().unwrap();
|
||||
let dir2 = TempDir::new().unwrap();
|
||||
|
||||
registry.register(Arc::new(LocalOrigin::new("a", dir1.path())), 1);
|
||||
registry.register(Arc::new(LocalOrigin::new("b", dir2.path())), 2);
|
||||
|
||||
let list = registry.list();
|
||||
assert_eq!(list.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
use crate::health::HealthSnapshot;
|
||||
use dashmap::DashMap;
|
||||
use musicfs_core::{Event, EventBus, OriginId};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
pub struct Router {
|
||||
priorities: DashMap<OriginId, u8>,
|
||||
latency_stats: DashMap<OriginId, LatencyStats>,
|
||||
event_bus: Option<Arc<EventBus>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct LatencyStats {
|
||||
pub samples: Vec<u64>,
|
||||
pub p50_ms: u64,
|
||||
pub p99_ms: u64,
|
||||
pub last_update: Option<Instant>,
|
||||
}
|
||||
|
||||
impl LatencyStats {
|
||||
pub fn record(&mut self, latency_ms: u64) {
|
||||
self.samples.push(latency_ms);
|
||||
|
||||
if self.samples.len() > 100 {
|
||||
self.samples.remove(0);
|
||||
}
|
||||
|
||||
if !self.samples.is_empty() {
|
||||
let mut sorted = self.samples.clone();
|
||||
sorted.sort_unstable();
|
||||
|
||||
let p50_idx = sorted.len() / 2;
|
||||
let p99_idx = (sorted.len() * 99) / 100;
|
||||
|
||||
self.p50_ms = sorted[p50_idx];
|
||||
self.p99_ms = sorted.get(p99_idx).copied().unwrap_or(self.p50_ms);
|
||||
}
|
||||
|
||||
self.last_update = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
impl Router {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
priorities: DashMap::new(),
|
||||
latency_stats: DashMap::new(),
|
||||
event_bus: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_event_bus(mut self, bus: Arc<EventBus>) -> Self {
|
||||
self.event_bus = Some(bus);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn set_priority(&self, id: OriginId, priority: u8) {
|
||||
self.priorities.insert(id, priority);
|
||||
}
|
||||
|
||||
pub fn remove_priority(&self, id: &OriginId) {
|
||||
self.priorities.remove(id);
|
||||
self.latency_stats.remove(id);
|
||||
}
|
||||
|
||||
pub fn get_priority(&self, id: &OriginId) -> u8 {
|
||||
self.priorities.get(id).map(|p| *p).unwrap_or(100)
|
||||
}
|
||||
|
||||
pub fn record_latency(&self, id: &OriginId, latency_ms: u64) {
|
||||
self.latency_stats
|
||||
.entry(id.clone())
|
||||
.or_default()
|
||||
.record(latency_ms);
|
||||
}
|
||||
|
||||
pub fn select(&self, candidates: &[OriginId], health: &HealthSnapshot) -> Option<OriginId> {
|
||||
let selected = candidates
|
||||
.iter()
|
||||
.filter(|id| health.is_healthy(id))
|
||||
.min_by_key(|id| {
|
||||
let priority = self.get_priority(id);
|
||||
let latency = self.latency_stats.get(*id).map(|s| s.p50_ms).unwrap_or(0);
|
||||
(priority, latency)
|
||||
})
|
||||
.cloned();
|
||||
|
||||
if let Some(ref id) = selected {
|
||||
let priority = self.get_priority(id);
|
||||
let latency = self.latency_stats.get(id).map(|s| s.p50_ms).unwrap_or(0);
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
priority = priority,
|
||||
latency_ms = latency,
|
||||
"Selected healthy origin"
|
||||
);
|
||||
}
|
||||
|
||||
selected
|
||||
}
|
||||
|
||||
pub fn select_with_fallback(
|
||||
&self,
|
||||
candidates: &[OriginId],
|
||||
health: &HealthSnapshot,
|
||||
) -> Option<OriginId> {
|
||||
if let Some(id) = self.select(candidates, health) {
|
||||
return Some(id);
|
||||
}
|
||||
|
||||
debug!("No healthy origins, trying degraded");
|
||||
if let Some(id) = candidates
|
||||
.iter()
|
||||
.filter(|id| health.is_degraded(id))
|
||||
.min_by_key(|id| self.get_priority(id))
|
||||
.cloned()
|
||||
{
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
priority = self.get_priority(&id),
|
||||
"Selected degraded origin as fallback"
|
||||
);
|
||||
return Some(id);
|
||||
}
|
||||
|
||||
warn!("All origins unhealthy, selecting least-bad by failure count");
|
||||
|
||||
if let Some(bus) = &self.event_bus {
|
||||
bus.publish(Event::AllOriginsUnhealthy {
|
||||
candidate_count: candidates.len(),
|
||||
});
|
||||
}
|
||||
|
||||
let selected = candidates
|
||||
.iter()
|
||||
.min_by_key(|id| {
|
||||
let failures = health.failure_count(id).unwrap_or(u32::MAX);
|
||||
let priority = self.get_priority(id);
|
||||
(failures, priority)
|
||||
})
|
||||
.cloned();
|
||||
|
||||
if let Some(ref id) = selected {
|
||||
let failures = health.failure_count(id).unwrap_or(u32::MAX);
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
failure_count = failures,
|
||||
priority = self.get_priority(id),
|
||||
"Selected least-bad unhealthy origin"
|
||||
);
|
||||
}
|
||||
|
||||
selected
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Router {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn mock_health(healthy: &[&str], degraded: &[&str]) -> HealthSnapshot {
|
||||
HealthSnapshot {
|
||||
healthy: healthy.iter().map(|s| OriginId::from(*s)).collect(),
|
||||
degraded: degraded.iter().map(|s| OriginId::from(*s)).collect(),
|
||||
unhealthy: Vec::new(),
|
||||
failure_counts: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_select_by_priority() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("high"), 1);
|
||||
router.set_priority(OriginId::from("low"), 2);
|
||||
|
||||
let candidates = vec![OriginId::from("low"), OriginId::from("high")];
|
||||
let health = mock_health(&["high", "low"], &[]);
|
||||
|
||||
let selected = router.select(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("high")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_select_skips_unhealthy() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("high"), 1);
|
||||
router.set_priority(OriginId::from("low"), 2);
|
||||
|
||||
let candidates = vec![OriginId::from("high"), OriginId::from("low")];
|
||||
let health = mock_health(&["low"], &[]);
|
||||
|
||||
let selected = router.select(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("low")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_latency_affects_tiebreak() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("a"), 1);
|
||||
router.set_priority(OriginId::from("b"), 1);
|
||||
|
||||
router.record_latency(&OriginId::from("a"), 100);
|
||||
router.record_latency(&OriginId::from("b"), 10);
|
||||
|
||||
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
|
||||
let health = mock_health(&["a", "b"], &[]);
|
||||
|
||||
let selected = router.select(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("b")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fallback_to_degraded() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("a"), 1);
|
||||
router.set_priority(OriginId::from("b"), 2);
|
||||
|
||||
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
|
||||
let health = mock_health(&[], &["b"]);
|
||||
|
||||
let selected = router.select_with_fallback(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("b")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fallback_least_bad() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("a"), 1);
|
||||
router.set_priority(OriginId::from("b"), 2);
|
||||
|
||||
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
|
||||
let mut failure_counts = HashMap::new();
|
||||
failure_counts.insert(OriginId::from("a"), 5);
|
||||
failure_counts.insert(OriginId::from("b"), 2);
|
||||
|
||||
let health = HealthSnapshot {
|
||||
healthy: Vec::new(),
|
||||
degraded: Vec::new(),
|
||||
unhealthy: vec![OriginId::from("a"), OriginId::from("b")],
|
||||
failure_counts,
|
||||
};
|
||||
|
||||
let selected = router.select_with_fallback(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("b")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
//! S3-compatible object storage origin
|
||||
//!
|
||||
//! This module is feature-gated behind the `s3` feature to avoid heavy AWS SDK dependencies.
|
||||
//!
|
||||
//! # Oracle Security Fixes (MUST IMPLEMENT)
|
||||
//!
|
||||
//! 1. **Range EOF** - Clamp range to `min(requested_end, file_size)` to avoid 416 errors
|
||||
//! 2. **Health check** - Use `head_bucket` not `list_objects_v2` (lighter operation)
|
||||
//! 3. **Timeout handling** - Wrap all remote calls with `tokio::time::timeout(30s)`
|
||||
//!
|
||||
//! # Example Implementation (when feature enabled)
|
||||
//!
|
||||
//! ```ignore
|
||||
//! async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
//! // Oracle fix: Clamp range to file size to avoid 416 error
|
||||
//! let file_size = self.stat(path).await?.size;
|
||||
//! let end = std::cmp::min(offset + size as u64, file_size).saturating_sub(1);
|
||||
//!
|
||||
//! if offset >= file_size {
|
||||
//! return Ok(Vec::new()); // EOF
|
||||
//! }
|
||||
//!
|
||||
//! let range = format!("bytes={}-{}", offset, end);
|
||||
//!
|
||||
//! // Oracle fix: Add timeout to prevent hung connections
|
||||
//! let resp = tokio::time::timeout(
|
||||
//! Duration::from_secs(30),
|
||||
//! self.client.get_object().bucket(&self.bucket).key(&key).range(range).send()
|
||||
//! )
|
||||
//! .await
|
||||
//! .map_err(|_| Error::Timeout("S3 read timed out".into()))?
|
||||
//! .map_err(|e| Error::S3(e.to_string()))?;
|
||||
//!
|
||||
//! // ...
|
||||
//! }
|
||||
//!
|
||||
//! async fn health(&self) -> HealthStatus {
|
||||
//! // Oracle fix: Use head_bucket instead of list_objects_v2 (lighter)
|
||||
//! match self.client.head_bucket().bucket(&self.bucket).send().await {
|
||||
//! Ok(_) => HealthStatus::Healthy,
|
||||
//! Err(_) => HealthStatus::Unhealthy,
|
||||
//! }
|
||||
//! }
|
||||
//! ```
|
||||
|
||||
#[cfg(feature = "s3")]
|
||||
mod implementation {
|
||||
// Full S3 implementation would go here when aws-sdk-s3 is enabled
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#![allow(dead_code)]
|
||||
//! SFTP origin - feature-gated to avoid russh/deadpool dependencies
|
||||
|
||||
#[cfg(feature = "sftp")]
|
||||
mod implementation {
|
||||
// Full SFTP implementation with connection pooling
|
||||
// Oracle fixes to implement:
|
||||
// 1. Use deadpool connection pool, not Arc<Mutex<SftpSession>>
|
||||
// 2. Verify SSH host keys against ~/.ssh/known_hosts
|
||||
// 3. Wrap all operations with tokio::time::timeout(30s)
|
||||
// 4. Cap open_read to actual file size, not u32::MAX
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
use crate::local::LocalOrigin;
|
||||
use crate::traits::{Origin, WatchCallback, WatchHandle};
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
pub struct SmbOrigin {
|
||||
inner: LocalOrigin,
|
||||
share_path: String,
|
||||
}
|
||||
|
||||
impl SmbOrigin {
|
||||
pub fn from_mount(
|
||||
id: impl Into<OriginId>,
|
||||
mount_point: impl Into<PathBuf>,
|
||||
share_path: impl Into<String>,
|
||||
) -> Self {
|
||||
let mount_point = mount_point.into();
|
||||
let share_path = share_path.into();
|
||||
|
||||
Self {
|
||||
inner: LocalOrigin::new(id, &mount_point),
|
||||
share_path,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn is_mounted(&self) -> bool {
|
||||
self.inner.exists(Path::new("/")).await.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn retry_on_disconnect<T, F, Fut>(&self, op: F) -> Result<T>
|
||||
where
|
||||
F: Fn() -> Fut,
|
||||
Fut: Future<Output = Result<T>>,
|
||||
{
|
||||
const MAX_RETRIES: u32 = 3;
|
||||
|
||||
for attempt in 0..MAX_RETRIES {
|
||||
match op().await {
|
||||
Ok(val) => return Ok(val),
|
||||
Err(e) => {
|
||||
if Self::is_enotconn(&e) && attempt < MAX_RETRIES - 1 {
|
||||
debug!(attempt, "SMB ENOTCONN, retrying");
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
continue;
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn is_enotconn(err: &musicfs_core::Error) -> bool {
|
||||
if let musicfs_core::Error::Io(io_err) = err {
|
||||
io_err.raw_os_error() == Some(libc::ENOTCONN)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn is_enotconn(_err: &musicfs_core::Error) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for SmbOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
self.inner.id()
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
OriginType::Smb
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
&self.share_path
|
||||
}
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
|
||||
self.retry_on_disconnect(|| self.inner.readdir(path)).await
|
||||
}
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat> {
|
||||
self.retry_on_disconnect(|| self.inner.stat(path)).await
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
self.retry_on_disconnect(|| self.inner.read(path, offset, size))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
self.retry_on_disconnect(|| self.inner.read_full(path))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
self.retry_on_disconnect(|| self.inner.exists(path)).await
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
let health_timeout = std::time::Duration::from_secs(5);
|
||||
match tokio::time::timeout(health_timeout, self.is_mounted()).await {
|
||||
Ok(true) => HealthStatus::Healthy,
|
||||
Ok(false) | Err(_) => HealthStatus::Unhealthy,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn tokio::io::AsyncRead + Send + Unpin>> {
|
||||
self.inner.open_read(path).await
|
||||
}
|
||||
|
||||
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
|
||||
warn!("SMB watch using inotify - may be unreliable. Consider polling for remote mounts.");
|
||||
self.inner.watch(path, callback).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_smb_origin_basic() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.flac"), b"audio").unwrap();
|
||||
|
||||
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/share");
|
||||
|
||||
let entries = origin.readdir(Path::new("/")).await.unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_smb_origin_type() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/share");
|
||||
|
||||
assert_eq!(origin.origin_type(), OriginType::Smb);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_smb_display_name() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/music");
|
||||
|
||||
assert_eq!(origin.display_name(), "//server/music");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
#[async_trait]
|
||||
pub trait Origin: Send + Sync {
|
||||
fn id(&self) -> &OriginId;
|
||||
|
||||
fn origin_type(&self) -> OriginType;
|
||||
|
||||
fn display_name(&self) -> &str;
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>>;
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat>;
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>>;
|
||||
|
||||
/// Read entire file content (for CDC chunking of files <4GB)
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>>;
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool>;
|
||||
|
||||
async fn health(&self) -> HealthStatus;
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>>;
|
||||
|
||||
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle>;
|
||||
}
|
||||
|
||||
pub type WatchCallback = Box<dyn Fn(WatchEvent) + Send + Sync>;
|
||||
|
||||
pub struct WatchHandle {
|
||||
_cancel: tokio::sync::oneshot::Sender<()>,
|
||||
}
|
||||
|
||||
impl WatchHandle {
|
||||
pub fn new(cancel: tokio::sync::oneshot::Sender<()>) -> Self {
|
||||
Self { _cancel: cancel }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum WatchEvent {
|
||||
Created(PathBuf),
|
||||
Modified(PathBuf),
|
||||
Deleted(PathBuf),
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "musicfs-plugins"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
async-trait.workspace = true
|
||||
tokio.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tracing.workspace = true
|
||||
libloading = "0.8"
|
||||
wasmtime = { version = "19", optional = true }
|
||||
semver = "1"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
wasm = ["wasmtime"]
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -0,0 +1,42 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum PluginError {
|
||||
#[error("Plugin not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
#[error("Plugin load failed: {0}")]
|
||||
LoadFailed(String),
|
||||
|
||||
#[error("Plugin initialization failed: {0}")]
|
||||
InitFailed(String),
|
||||
|
||||
#[error("Plugin API version mismatch: expected {expected}, got {actual}")]
|
||||
VersionMismatch { expected: String, actual: String },
|
||||
|
||||
#[error("Plugin already loaded: {0}")]
|
||||
AlreadyLoaded(String),
|
||||
|
||||
#[error("Plugin symbol not found: {0}")]
|
||||
SymbolNotFound(String),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("Plugin execution error: {0}")]
|
||||
Execution(String),
|
||||
|
||||
#[error("Plugin shutdown error: {0}")]
|
||||
Shutdown(String),
|
||||
|
||||
#[error("Configuration error: {0}")]
|
||||
Config(String),
|
||||
|
||||
#[error("WASM error: {0}")]
|
||||
Wasm(String),
|
||||
|
||||
#[error("Resource limit exceeded: {0}")]
|
||||
ResourceLimit(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, PluginError>;
|
||||
@@ -0,0 +1,15 @@
|
||||
pub mod error;
|
||||
pub mod manager;
|
||||
pub mod native;
|
||||
pub mod traits;
|
||||
pub mod wasm;
|
||||
|
||||
pub use error::{PluginError, Result};
|
||||
pub use manager::{PluginConfig, PluginEntry, PluginManager, WasmConfig};
|
||||
pub use native::NativePluginHost;
|
||||
pub use traits::{
|
||||
ExternalMetadata, FormatPlugin, MetadataPlugin, MetadataQuery, MetadataQueryType,
|
||||
OriginDirEntry, OriginHealth, OriginInstance, OriginPlugin, OriginStat, Plugin, PluginId,
|
||||
PluginInfo, PluginType, WatchEvent, WatchHandle, PLUGIN_API_VERSION,
|
||||
};
|
||||
pub use wasm::{ResourceLimits, WasmPluginHost};
|
||||
@@ -0,0 +1,346 @@
|
||||
use crate::error::{PluginError, Result};
|
||||
use crate::native::NativePluginHost;
|
||||
use crate::traits::{Plugin, PluginId, PluginInfo, PluginType};
|
||||
use crate::wasm::{ResourceLimits, WasmPluginHost};
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use tracing::{debug, info};
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
pub struct PluginConfig {
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub search_paths: Vec<PathBuf>,
|
||||
|
||||
#[serde(default)]
|
||||
pub plugins: HashMap<String, PluginEntry>,
|
||||
|
||||
#[serde(default)]
|
||||
pub wasm: WasmConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct PluginEntry {
|
||||
pub path: PathBuf,
|
||||
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub config: Value,
|
||||
}
|
||||
|
||||
impl Default for PluginEntry {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
path: PathBuf::new(),
|
||||
enabled: true,
|
||||
config: Value::Null,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
pub struct WasmConfig {
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub max_memory_mb: Option<u32>,
|
||||
|
||||
#[serde(default)]
|
||||
pub max_cpu_time_ms: Option<u32>,
|
||||
}
|
||||
|
||||
pub struct PluginManager {
|
||||
native_host: NativePluginHost,
|
||||
wasm_host: WasmPluginHost,
|
||||
registry: PluginRegistry,
|
||||
config: PluginConfig,
|
||||
}
|
||||
|
||||
struct PluginRegistry {
|
||||
origin_plugins: Vec<PluginId>,
|
||||
metadata_plugins: Vec<PluginId>,
|
||||
format_plugins: Vec<PluginId>,
|
||||
}
|
||||
|
||||
impl PluginRegistry {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
origin_plugins: Vec::new(),
|
||||
metadata_plugins: Vec::new(),
|
||||
format_plugins: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn register(&mut self, id: PluginId, plugin_type: PluginType) {
|
||||
match plugin_type {
|
||||
PluginType::Origin => {
|
||||
if !self.origin_plugins.contains(&id) {
|
||||
self.origin_plugins.push(id);
|
||||
}
|
||||
}
|
||||
PluginType::Metadata => {
|
||||
if !self.metadata_plugins.contains(&id) {
|
||||
self.metadata_plugins.push(id);
|
||||
}
|
||||
}
|
||||
PluginType::Format => {
|
||||
if !self.format_plugins.contains(&id) {
|
||||
self.format_plugins.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn unregister(&mut self, id: PluginId) {
|
||||
self.origin_plugins.retain(|&x| x != id);
|
||||
self.metadata_plugins.retain(|&x| x != id);
|
||||
self.format_plugins.retain(|&x| x != id);
|
||||
}
|
||||
}
|
||||
|
||||
impl PluginManager {
|
||||
pub fn new() -> Result<Self> {
|
||||
Ok(Self {
|
||||
native_host: NativePluginHost::new(),
|
||||
wasm_host: WasmPluginHost::new()?,
|
||||
registry: PluginRegistry::new(),
|
||||
config: PluginConfig::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn init(config: &PluginConfig) -> Result<Self> {
|
||||
let mut manager = Self::new()?;
|
||||
manager.config = config.clone();
|
||||
|
||||
if !config.enabled {
|
||||
info!("Plugin system disabled");
|
||||
return Ok(manager);
|
||||
}
|
||||
|
||||
info!("Initializing plugin system");
|
||||
|
||||
for path in &config.search_paths {
|
||||
manager.native_host.add_search_path(path.clone());
|
||||
}
|
||||
|
||||
if config.wasm.enabled {
|
||||
let limits = ResourceLimits {
|
||||
max_memory_mb: config.wasm.max_memory_mb.unwrap_or(64),
|
||||
max_cpu_time_ms: config.wasm.max_cpu_time_ms.unwrap_or(5000),
|
||||
..Default::default()
|
||||
};
|
||||
manager.wasm_host.set_limits(limits);
|
||||
}
|
||||
|
||||
for (name, entry) in &config.plugins {
|
||||
if !entry.enabled {
|
||||
debug!("Skipping disabled plugin: {}", name);
|
||||
continue;
|
||||
}
|
||||
|
||||
match manager.load_and_init(&entry.path, &entry.config) {
|
||||
Ok(id) => {
|
||||
info!("Loaded plugin '{}' with id {:?}", name, id);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to load plugin '{}': {}", name, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let discovered = manager.native_host.discover()?;
|
||||
for id in discovered {
|
||||
if let Some(info) = manager.native_host.list().iter().find(|i| i.id == id) {
|
||||
manager.registry.register(id, info.plugin_type);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(manager)
|
||||
}
|
||||
|
||||
pub fn load_and_init(&mut self, path: &PathBuf, config: &Value) -> Result<PluginId> {
|
||||
let id = self.native_host.load(path)?;
|
||||
|
||||
if let Some(plugin) = self.native_host.get_mut(id) {
|
||||
plugin.init(config.clone())?;
|
||||
}
|
||||
|
||||
if let Some(info) = self.native_host.list().iter().find(|i| i.id == id) {
|
||||
self.registry.register(id, info.plugin_type);
|
||||
}
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub fn load_wasm(&mut self, wasm_bytes: &[u8]) -> Result<PluginId> {
|
||||
if !self.config.wasm.enabled {
|
||||
return Err(PluginError::Config("WASM plugins disabled".to_string()));
|
||||
}
|
||||
|
||||
self.wasm_host.load(wasm_bytes)
|
||||
}
|
||||
|
||||
pub fn unload(&mut self, id: PluginId) -> Result<()> {
|
||||
self.registry.unregister(id);
|
||||
|
||||
if let Err(native_err) = self.native_host.unload(id) {
|
||||
if let Err(wasm_err) = self.wasm_host.unload(id) {
|
||||
return Err(PluginError::NotFound(format!(
|
||||
"Plugin {:?} not found in native ({}) or WASM ({}) hosts",
|
||||
id, native_err, wasm_err
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn reload(&mut self, id: PluginId) -> Result<()> {
|
||||
self.native_host.reload(id)
|
||||
}
|
||||
|
||||
pub fn reload_all(&mut self) -> Result<()> {
|
||||
let ids: Vec<PluginId> = self.native_host.list().iter().map(|i| i.id).collect();
|
||||
|
||||
for id in ids {
|
||||
self.reload(id)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list(&self) -> Vec<PluginInfo> {
|
||||
let mut all = self.native_host.list();
|
||||
|
||||
for (id, name) in self.wasm_host.list() {
|
||||
all.push(PluginInfo {
|
||||
id,
|
||||
name: name.to_string(),
|
||||
version: semver::Version::new(0, 0, 0),
|
||||
description: "WASM plugin".to_string(),
|
||||
plugin_type: PluginType::Origin,
|
||||
});
|
||||
}
|
||||
|
||||
all
|
||||
}
|
||||
|
||||
pub fn get(&self, id: PluginId) -> Option<&dyn Plugin> {
|
||||
self.native_host.get(id)
|
||||
}
|
||||
|
||||
pub fn get_mut(&mut self, id: PluginId) -> Option<&mut dyn Plugin> {
|
||||
self.native_host.get_mut(id)
|
||||
}
|
||||
|
||||
pub fn origin_plugin_ids(&self) -> &[PluginId] {
|
||||
&self.registry.origin_plugins
|
||||
}
|
||||
|
||||
pub fn metadata_plugin_ids(&self) -> &[PluginId] {
|
||||
&self.registry.metadata_plugins
|
||||
}
|
||||
|
||||
pub fn format_plugin_ids(&self) -> &[PluginId] {
|
||||
&self.registry.format_plugins
|
||||
}
|
||||
|
||||
pub fn shutdown(&mut self) -> Result<()> {
|
||||
info!("Shutting down plugin system");
|
||||
|
||||
let ids: Vec<PluginId> = self.list().iter().map(|i| i.id).collect();
|
||||
|
||||
for id in ids {
|
||||
if let Err(e) = self.unload(id) {
|
||||
tracing::warn!("Failed to unload plugin {:?}: {}", id, e);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PluginManager {
|
||||
fn default() -> Self {
|
||||
Self::new().expect("Failed to create plugin manager")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PluginManager {
|
||||
fn drop(&mut self) {
|
||||
debug!(plugin_count = self.list().len(), "PluginManager dropping");
|
||||
let _ = self.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_plugin_manager_new() {
|
||||
let manager = PluginManager::new();
|
||||
assert!(manager.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_plugin_manager_disabled() {
|
||||
let config = PluginConfig {
|
||||
enabled: false,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let manager = PluginManager::init(&config);
|
||||
assert!(manager.is_ok());
|
||||
|
||||
let manager = manager.unwrap();
|
||||
assert!(manager.list().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_registry() {
|
||||
let mut registry = PluginRegistry::new();
|
||||
|
||||
let id1 = PluginId::new(1);
|
||||
let id2 = PluginId::new(2);
|
||||
|
||||
registry.register(id1, PluginType::Origin);
|
||||
registry.register(id2, PluginType::Metadata);
|
||||
|
||||
assert_eq!(registry.origin_plugins.len(), 1);
|
||||
assert_eq!(registry.metadata_plugins.len(), 1);
|
||||
|
||||
registry.unregister(id1);
|
||||
assert!(registry.origin_plugins.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_plugin_config_deserialize() {
|
||||
let json = r#"{
|
||||
"enabled": true,
|
||||
"search_paths": ["/usr/lib/musicfs/plugins"],
|
||||
"plugins": {
|
||||
"example": {
|
||||
"path": "/path/to/plugin.so",
|
||||
"enabled": true,
|
||||
"config": {"key": "value"}
|
||||
}
|
||||
},
|
||||
"wasm": {
|
||||
"enabled": false
|
||||
}
|
||||
}"#;
|
||||
|
||||
let config: PluginConfig = serde_json::from_str(json).unwrap();
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.search_paths.len(), 1);
|
||||
assert!(config.plugins.contains_key("example"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
use crate::error::{PluginError, Result};
|
||||
use crate::traits::{Plugin, PluginId, PluginInfo, PluginType, PLUGIN_API_VERSION};
|
||||
use libloading::{Library, Symbol};
|
||||
use semver::Version;
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::CStr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
static NEXT_PLUGIN_ID: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
fn next_plugin_id() -> PluginId {
|
||||
PluginId::new(NEXT_PLUGIN_ID.fetch_add(1, Ordering::SeqCst))
|
||||
}
|
||||
|
||||
struct LoadedPlugin {
|
||||
id: PluginId,
|
||||
path: PathBuf,
|
||||
library: Library,
|
||||
instance: Box<dyn Plugin>,
|
||||
plugin_type: PluginType,
|
||||
}
|
||||
|
||||
pub struct NativePluginHost {
|
||||
plugins: HashMap<PluginId, LoadedPlugin>,
|
||||
search_paths: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
impl NativePluginHost {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
plugins: HashMap::new(),
|
||||
search_paths: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_search_path(&mut self, path: PathBuf) {
|
||||
if !self.search_paths.contains(&path) {
|
||||
self.search_paths.push(path);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(&mut self, path: &Path) -> Result<PluginId> {
|
||||
let canonical = path.canonicalize().map_err(|e| {
|
||||
PluginError::LoadFailed(format!("Cannot resolve path {}: {}", path.display(), e))
|
||||
})?;
|
||||
|
||||
for plugin in self.plugins.values() {
|
||||
if plugin.path == canonical {
|
||||
return Err(PluginError::AlreadyLoaded(canonical.display().to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
info!("Loading native plugin from {:?}", canonical);
|
||||
|
||||
let library = unsafe {
|
||||
Library::new(&canonical)
|
||||
.map_err(|e| PluginError::LoadFailed(format!("Failed to load library: {}", e)))?
|
||||
};
|
||||
|
||||
self.verify_api_version(&library)?;
|
||||
|
||||
let instance = self.create_plugin_instance(&library)?;
|
||||
let id = next_plugin_id();
|
||||
|
||||
let plugin_type = self.detect_plugin_type(&*instance);
|
||||
|
||||
debug!(
|
||||
"Loaded plugin '{}' v{} as {:?}",
|
||||
instance.name(),
|
||||
instance.version(),
|
||||
plugin_type
|
||||
);
|
||||
|
||||
self.plugins.insert(
|
||||
id,
|
||||
LoadedPlugin {
|
||||
id,
|
||||
path: canonical,
|
||||
library,
|
||||
instance,
|
||||
plugin_type,
|
||||
},
|
||||
);
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub fn unload(&mut self, id: PluginId) -> Result<()> {
|
||||
let mut plugin = self
|
||||
.plugins
|
||||
.remove(&id)
|
||||
.ok_or_else(|| PluginError::NotFound(format!("Plugin {:?}", id)))?;
|
||||
|
||||
info!("Unloading plugin '{}'", plugin.instance.name());
|
||||
|
||||
plugin.instance.shutdown()?;
|
||||
|
||||
drop(plugin.instance);
|
||||
drop(plugin.library);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn reload(&mut self, id: PluginId) -> Result<()> {
|
||||
let plugin = self
|
||||
.plugins
|
||||
.get(&id)
|
||||
.ok_or_else(|| PluginError::NotFound(format!("Plugin {:?}", id)))?;
|
||||
|
||||
let path = plugin.path.clone();
|
||||
|
||||
info!("Hot-reloading plugin from {:?}", path);
|
||||
|
||||
self.unload(id)?;
|
||||
|
||||
let new_id = self.load(&path)?;
|
||||
|
||||
if let Some(plugin) = self.plugins.remove(&new_id) {
|
||||
self.plugins.insert(id, LoadedPlugin { id, ..plugin });
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get(&self, id: PluginId) -> Option<&dyn Plugin> {
|
||||
self.plugins.get(&id).map(|p| &*p.instance as &dyn Plugin)
|
||||
}
|
||||
|
||||
pub fn get_mut(&mut self, id: PluginId) -> Option<&mut dyn Plugin> {
|
||||
self.plugins
|
||||
.get_mut(&id)
|
||||
.map(|p| &mut *p.instance as &mut dyn Plugin)
|
||||
}
|
||||
|
||||
pub fn list(&self) -> Vec<PluginInfo> {
|
||||
self.plugins
|
||||
.values()
|
||||
.map(|p| PluginInfo {
|
||||
id: p.id,
|
||||
name: p.instance.name().to_string(),
|
||||
version: p.instance.version(),
|
||||
description: p.instance.description().to_string(),
|
||||
plugin_type: p.plugin_type,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn find_by_name(&self, name: &str) -> Option<PluginId> {
|
||||
self.plugins
|
||||
.iter()
|
||||
.find(|(_, p)| p.instance.name() == name)
|
||||
.map(|(id, _)| *id)
|
||||
}
|
||||
|
||||
pub fn discover(&mut self) -> Result<Vec<PluginId>> {
|
||||
let mut loaded = Vec::new();
|
||||
|
||||
for search_path in self.search_paths.clone() {
|
||||
if !search_path.exists() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let entries = std::fs::read_dir(&search_path).map_err(|e| {
|
||||
PluginError::LoadFailed(format!(
|
||||
"Cannot read plugin directory {}: {}",
|
||||
search_path.display(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
|
||||
if self.is_plugin_library(&path) {
|
||||
match self.load(&path) {
|
||||
Ok(id) => loaded.push(id),
|
||||
Err(e) => {
|
||||
warn!("Failed to load plugin {:?}: {}", path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(loaded)
|
||||
}
|
||||
|
||||
fn verify_api_version(&self, library: &Library) -> Result<()> {
|
||||
let version_fn: Symbol<unsafe extern "C" fn() -> *const std::ffi::c_char> = unsafe {
|
||||
library.get(b"musicfs_plugin_api_version").map_err(|_| {
|
||||
PluginError::SymbolNotFound("musicfs_plugin_api_version".to_string())
|
||||
})?
|
||||
};
|
||||
|
||||
let version_ptr = unsafe { version_fn() };
|
||||
let version_str = unsafe { CStr::from_ptr(version_ptr) }
|
||||
.to_str()
|
||||
.map_err(|_| PluginError::VersionMismatch {
|
||||
expected: PLUGIN_API_VERSION.to_string(),
|
||||
actual: "<invalid UTF-8>".to_string(),
|
||||
})?;
|
||||
|
||||
let plugin_version =
|
||||
Version::parse(version_str).map_err(|_| PluginError::VersionMismatch {
|
||||
expected: PLUGIN_API_VERSION.to_string(),
|
||||
actual: version_str.to_string(),
|
||||
})?;
|
||||
|
||||
let expected_version = Version::parse(PLUGIN_API_VERSION).unwrap();
|
||||
|
||||
if plugin_version.major != expected_version.major {
|
||||
return Err(PluginError::VersionMismatch {
|
||||
expected: PLUGIN_API_VERSION.to_string(),
|
||||
actual: version_str.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_plugin_instance(&self, library: &Library) -> Result<Box<dyn Plugin>> {
|
||||
let create_fn: Symbol<unsafe extern "C" fn() -> *mut dyn Plugin> = unsafe {
|
||||
library
|
||||
.get(b"musicfs_plugin_create")
|
||||
.map_err(|_| PluginError::SymbolNotFound("musicfs_plugin_create".to_string()))?
|
||||
};
|
||||
|
||||
let plugin_ptr = unsafe { create_fn() };
|
||||
if plugin_ptr.is_null() {
|
||||
return Err(PluginError::LoadFailed(
|
||||
"Plugin factory returned null".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let plugin = unsafe { Box::from_raw(plugin_ptr) };
|
||||
Ok(plugin)
|
||||
}
|
||||
|
||||
fn detect_plugin_type(&self, plugin: &dyn Plugin) -> PluginType {
|
||||
plugin.plugin_type()
|
||||
}
|
||||
|
||||
fn is_plugin_library(&self, path: &Path) -> bool {
|
||||
let extension = path.extension().and_then(|e| e.to_str());
|
||||
|
||||
match extension {
|
||||
Some("so") => true,
|
||||
Some("dylib") => true,
|
||||
Some("dll") => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for NativePluginHost {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_native_host_creation() {
|
||||
let host = NativePluginHost::new();
|
||||
assert!(host.plugins.is_empty());
|
||||
assert!(host.search_paths.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_search_path() {
|
||||
let mut host = NativePluginHost::new();
|
||||
host.add_search_path(PathBuf::from("/usr/lib/musicfs/plugins"));
|
||||
host.add_search_path(PathBuf::from("/usr/lib/musicfs/plugins"));
|
||||
|
||||
assert_eq!(host.search_paths.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_plugin_library() {
|
||||
let host = NativePluginHost::new();
|
||||
|
||||
assert!(host.is_plugin_library(Path::new("plugin.so")));
|
||||
assert!(host.is_plugin_library(Path::new("plugin.dylib")));
|
||||
assert!(host.is_plugin_library(Path::new("plugin.dll")));
|
||||
assert!(!host.is_plugin_library(Path::new("plugin.txt")));
|
||||
assert!(!host.is_plugin_library(Path::new("plugin")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_nonexistent() {
|
||||
let mut host = NativePluginHost::new();
|
||||
let result = host.load(Path::new("/nonexistent/plugin.so"));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
//! Plugin trait definitions (FR-23.1-23.4)
|
||||
//!
|
||||
//! Per architecture.md section 4.3.4:
|
||||
//! - Plugin trait: Base interface for all plugins
|
||||
//! - OriginPlugin: Creates Origin instances for storage backends
|
||||
//! - MetadataPlugin: Provides external metadata lookup
|
||||
//! - FormatPlugin: Handles custom audio format parsing
|
||||
|
||||
use crate::error::Result;
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::AudioMeta;
|
||||
use semver::Version;
|
||||
use serde_json::Value;
|
||||
use std::io::Read;
|
||||
|
||||
/// Current plugin API version
|
||||
pub const PLUGIN_API_VERSION: &str = "0.1.0";
|
||||
|
||||
/// Unique identifier for a loaded plugin
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct PluginId(pub u64);
|
||||
|
||||
impl PluginId {
|
||||
pub fn new(id: u64) -> Self {
|
||||
Self(id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Plugin metadata returned by plugins
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PluginInfo {
|
||||
pub id: PluginId,
|
||||
pub name: String,
|
||||
pub version: Version,
|
||||
pub description: String,
|
||||
pub plugin_type: PluginType,
|
||||
}
|
||||
|
||||
/// Type of plugin
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PluginType {
|
||||
Origin,
|
||||
Metadata,
|
||||
Format,
|
||||
}
|
||||
|
||||
/// Base plugin interface (FR-23.1)
|
||||
///
|
||||
/// All plugins must implement this trait. It provides:
|
||||
/// - Plugin identification (name, version)
|
||||
/// - Lifecycle management (init, shutdown)
|
||||
pub trait Plugin: Send + Sync {
|
||||
/// Unique plugin name (e.g., "s3-origin", "musicbrainz-metadata")
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Plugin version following semver
|
||||
fn version(&self) -> Version;
|
||||
|
||||
/// Human-readable description
|
||||
fn description(&self) -> &str {
|
||||
""
|
||||
}
|
||||
|
||||
/// Plugin type for registry categorization
|
||||
fn plugin_type(&self) -> PluginType;
|
||||
|
||||
/// Initialize plugin with configuration
|
||||
///
|
||||
/// Called once after loading. The config value contains
|
||||
/// plugin-specific configuration from the main config file.
|
||||
fn init(&mut self, config: Value) -> Result<()>;
|
||||
|
||||
/// Shutdown plugin and release resources
|
||||
///
|
||||
/// Called before unloading. Plugins should clean up any
|
||||
/// resources (connections, file handles, etc).
|
||||
fn shutdown(&mut self) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Origin plugin interface (FR-23.3)
|
||||
///
|
||||
/// Per architecture.md section 4.3.4:
|
||||
/// Origin plugins create `Box<dyn Origin>` instances for custom storage backends.
|
||||
///
|
||||
/// Example use cases:
|
||||
/// - Google Drive origin
|
||||
/// - Dropbox origin
|
||||
/// - Custom NAS protocol
|
||||
#[async_trait]
|
||||
pub trait OriginPlugin: Plugin {
|
||||
/// Origin type identifier (e.g., "gdrive", "dropbox")
|
||||
fn origin_type(&self) -> &str;
|
||||
|
||||
/// Create a new Origin instance with the given configuration
|
||||
///
|
||||
/// The config contains origin-specific settings (credentials, paths, etc).
|
||||
/// Returns a boxed Origin that can be used by the OriginRouter.
|
||||
async fn create_origin(&self, id: &str, config: Value) -> Result<Box<dyn OriginInstance>>;
|
||||
}
|
||||
|
||||
/// Instance created by OriginPlugin
|
||||
///
|
||||
/// This is a simplified async interface that maps to the full Origin trait.
|
||||
/// The plugin host wraps this to provide the full Origin implementation.
|
||||
#[async_trait]
|
||||
pub trait OriginInstance: Send + Sync {
|
||||
/// List directory contents
|
||||
async fn readdir(&self, path: &str) -> Result<Vec<OriginDirEntry>>;
|
||||
|
||||
/// Get file/directory stats
|
||||
async fn stat(&self, path: &str) -> Result<OriginStat>;
|
||||
|
||||
/// Read file data
|
||||
async fn read(&self, path: &str, offset: u64, size: u32) -> Result<Vec<u8>>;
|
||||
|
||||
/// Check if path exists
|
||||
async fn exists(&self, path: &str) -> Result<bool>;
|
||||
|
||||
/// Health check
|
||||
async fn health(&self) -> OriginHealth;
|
||||
|
||||
/// Watch path for changes (FR-10.2)
|
||||
async fn watch(
|
||||
&self,
|
||||
path: &str,
|
||||
callback: Box<dyn Fn(WatchEvent) + Send + Sync>,
|
||||
) -> Result<WatchHandle>;
|
||||
}
|
||||
|
||||
pub struct WatchHandle {
|
||||
_cancel: tokio::sync::oneshot::Sender<()>,
|
||||
}
|
||||
|
||||
impl WatchHandle {
|
||||
pub fn new(cancel: tokio::sync::oneshot::Sender<()>) -> Self {
|
||||
Self { _cancel: cancel }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum WatchEvent {
|
||||
Created(String),
|
||||
Modified(String),
|
||||
Deleted(String),
|
||||
}
|
||||
|
||||
/// Directory entry from plugin origin
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OriginDirEntry {
|
||||
pub name: String,
|
||||
pub is_dir: bool,
|
||||
pub size: u64,
|
||||
pub mtime_secs: u64,
|
||||
}
|
||||
|
||||
/// File stats from plugin origin
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OriginStat {
|
||||
pub size: u64,
|
||||
pub mtime_secs: u64,
|
||||
pub is_dir: bool,
|
||||
}
|
||||
|
||||
/// Origin health status
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OriginHealth {
|
||||
Healthy,
|
||||
Degraded,
|
||||
Unhealthy,
|
||||
}
|
||||
|
||||
/// Metadata plugin interface (FR-23.3)
|
||||
///
|
||||
/// Metadata plugins provide external metadata lookup from services like
|
||||
/// MusicBrainz, Discogs, Last.fm, etc.
|
||||
#[async_trait]
|
||||
pub trait MetadataPlugin: Plugin {
|
||||
/// Lookup metadata for a query
|
||||
///
|
||||
/// Returns enriched metadata if found, None otherwise.
|
||||
async fn lookup(&self, query: &MetadataQuery) -> Result<Option<ExternalMetadata>>;
|
||||
|
||||
/// Supported query types
|
||||
fn supported_queries(&self) -> &[MetadataQueryType] {
|
||||
&[MetadataQueryType::ByTitleArtist]
|
||||
}
|
||||
}
|
||||
|
||||
/// Query for metadata lookup
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MetadataQuery {
|
||||
pub query_type: MetadataQueryType,
|
||||
pub title: Option<String>,
|
||||
pub artist: Option<String>,
|
||||
pub album: Option<String>,
|
||||
pub fingerprint: Option<String>,
|
||||
pub duration_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// Type of metadata query
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MetadataQueryType {
|
||||
ByTitleArtist,
|
||||
ByFingerprint,
|
||||
ByAlbum,
|
||||
}
|
||||
|
||||
/// External metadata returned by plugins
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ExternalMetadata {
|
||||
pub title: Option<String>,
|
||||
pub artist: Option<String>,
|
||||
pub album: Option<String>,
|
||||
pub album_artist: Option<String>,
|
||||
pub genre: Option<String>,
|
||||
pub year: Option<u32>,
|
||||
pub track: Option<u32>,
|
||||
pub disc: Option<u32>,
|
||||
pub musicbrainz_id: Option<String>,
|
||||
pub artwork_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Format plugin interface (FR-24.1)
|
||||
///
|
||||
/// Format plugins handle custom audio formats not supported by symphonia.
|
||||
///
|
||||
/// Example use cases:
|
||||
/// - Custom lossless codecs
|
||||
/// - Proprietary formats
|
||||
/// - Game audio formats
|
||||
pub trait FormatPlugin: Plugin {
|
||||
/// File extensions this plugin handles
|
||||
fn extensions(&self) -> &[&str];
|
||||
|
||||
/// Check if plugin can handle a specific extension
|
||||
fn can_handle(&self, extension: &str) -> bool {
|
||||
self.extensions()
|
||||
.iter()
|
||||
.any(|ext| ext.eq_ignore_ascii_case(extension))
|
||||
}
|
||||
|
||||
/// Parse audio metadata from reader
|
||||
///
|
||||
/// The reader provides the raw file bytes. Plugin should parse
|
||||
/// and return AudioMeta with whatever metadata it can extract.
|
||||
fn parse(&self, reader: &mut dyn Read) -> Result<AudioMeta>;
|
||||
|
||||
/// Synthesize file header with updated metadata (FR-5.3)
|
||||
///
|
||||
/// Creates a new file header containing the provided metadata.
|
||||
/// Used for metadata overlay - serving cached metadata without
|
||||
/// modifying the original file.
|
||||
fn synthesize_header(&self, metadata: &AudioMeta) -> Result<Vec<u8>>;
|
||||
}
|
||||
|
||||
/// Declaration macro for native plugins
|
||||
///
|
||||
/// Native plugins must export a function with this signature:
|
||||
/// ```ignore
|
||||
/// #[no_mangle]
|
||||
/// pub extern "C" fn musicfs_plugin_create() -> *mut dyn Plugin
|
||||
/// ```
|
||||
#[macro_export]
|
||||
macro_rules! declare_plugin {
|
||||
($plugin_type:ty, $constructor:expr) => {
|
||||
#[no_mangle]
|
||||
pub extern "C" fn musicfs_plugin_create() -> *mut dyn $crate::Plugin {
|
||||
let plugin = $constructor;
|
||||
let boxed: Box<dyn $crate::Plugin> = Box::new(plugin);
|
||||
Box::into_raw(boxed)
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "C" fn musicfs_plugin_api_version() -> *const std::ffi::c_char {
|
||||
concat!($crate::PLUGIN_API_VERSION, "\0").as_ptr() as *const std::ffi::c_char
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
struct TestPlugin {
|
||||
name: String,
|
||||
initialized: bool,
|
||||
}
|
||||
|
||||
impl Plugin for TestPlugin {
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn version(&self) -> Version {
|
||||
Version::new(1, 0, 0)
|
||||
}
|
||||
|
||||
fn plugin_type(&self) -> PluginType {
|
||||
PluginType::Origin
|
||||
}
|
||||
|
||||
fn init(&mut self, _config: Value) -> Result<()> {
|
||||
self.initialized = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn shutdown(&mut self) -> Result<()> {
|
||||
self.initialized = false;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_plugin_lifecycle() {
|
||||
let mut plugin = TestPlugin {
|
||||
name: "test".to_string(),
|
||||
initialized: false,
|
||||
};
|
||||
|
||||
assert_eq!(plugin.name(), "test");
|
||||
assert!(!plugin.initialized);
|
||||
|
||||
plugin.init(Value::Null).unwrap();
|
||||
assert!(plugin.initialized);
|
||||
|
||||
plugin.shutdown().unwrap();
|
||||
assert!(!plugin.initialized);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_plugin_id() {
|
||||
let id1 = PluginId::new(1);
|
||||
let id2 = PluginId::new(1);
|
||||
let id3 = PluginId::new(2);
|
||||
|
||||
assert_eq!(id1, id2);
|
||||
assert_ne!(id1, id3);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
use crate::error::{PluginError, Result};
|
||||
use crate::traits::PluginId;
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
static NEXT_WASM_PLUGIN_ID: AtomicU64 = AtomicU64::new(1_000_000);
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
fn next_wasm_plugin_id() -> PluginId {
|
||||
PluginId::new(NEXT_WASM_PLUGIN_ID.fetch_add(1, Ordering::SeqCst))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ResourceLimits {
|
||||
pub max_memory_mb: u32,
|
||||
pub max_cpu_time_ms: u32,
|
||||
pub allow_network: bool,
|
||||
pub allow_filesystem: bool,
|
||||
}
|
||||
|
||||
impl Default for ResourceLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_memory_mb: 64,
|
||||
max_cpu_time_ms: 5000,
|
||||
allow_network: false,
|
||||
allow_filesystem: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
mod wasm_impl {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
use tracing::info;
|
||||
use wasmtime::{Config, Engine, Linker, Module, Store};
|
||||
|
||||
pub struct PluginState {
|
||||
limits: ResourceLimits,
|
||||
}
|
||||
|
||||
pub struct WasmPlugin {
|
||||
id: PluginId,
|
||||
name: String,
|
||||
_module: Module,
|
||||
}
|
||||
|
||||
impl WasmPlugin {
|
||||
pub fn id(&self) -> PluginId {
|
||||
self.id
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WasmPluginHost {
|
||||
engine: Engine,
|
||||
linker: Linker<PluginState>,
|
||||
plugins: HashMap<PluginId, WasmPlugin>,
|
||||
limits: ResourceLimits,
|
||||
}
|
||||
|
||||
impl WasmPluginHost {
|
||||
pub fn new() -> Result<Self> {
|
||||
let mut config = Config::new();
|
||||
config.consume_fuel(true);
|
||||
config.epoch_interruption(true);
|
||||
|
||||
let engine = Engine::new(&config)
|
||||
.map_err(|e| PluginError::Wasm(format!("Failed to create WASM engine: {}", e)))?;
|
||||
|
||||
let linker = Linker::new(&engine);
|
||||
|
||||
Ok(Self {
|
||||
engine,
|
||||
linker,
|
||||
plugins: HashMap::new(),
|
||||
limits: ResourceLimits::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn set_limits(&mut self, limits: ResourceLimits) {
|
||||
self.limits = limits;
|
||||
}
|
||||
|
||||
pub fn load(&mut self, wasm_bytes: &[u8]) -> Result<PluginId> {
|
||||
info!("Loading WASM plugin ({} bytes)", wasm_bytes.len());
|
||||
|
||||
let module = Module::new(&self.engine, wasm_bytes)
|
||||
.map_err(|e| PluginError::Wasm(format!("Failed to compile WASM module: {}", e)))?;
|
||||
|
||||
let id = next_wasm_plugin_id();
|
||||
let name = module.name().unwrap_or("unnamed").to_string();
|
||||
|
||||
let plugin = WasmPlugin {
|
||||
id,
|
||||
name,
|
||||
_module: module,
|
||||
};
|
||||
|
||||
self.plugins.insert(id, plugin);
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub fn unload(&mut self, id: PluginId) -> Result<()> {
|
||||
self.plugins
|
||||
.remove(&id)
|
||||
.ok_or_else(|| PluginError::NotFound(format!("WASM plugin {:?}", id)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get(&self, id: PluginId) -> Option<&WasmPlugin> {
|
||||
self.plugins.get(&id)
|
||||
}
|
||||
|
||||
pub fn list(&self) -> Vec<(PluginId, &str)> {
|
||||
self.plugins.iter().map(|(id, p)| (*id, p.name())).collect()
|
||||
}
|
||||
|
||||
fn create_store(&self) -> Store<PluginState> {
|
||||
let state = PluginState {
|
||||
limits: self.limits.clone(),
|
||||
};
|
||||
|
||||
let mut store = Store::new(&self.engine, state);
|
||||
|
||||
let fuel = (self.limits.max_cpu_time_ms as u64) * 1_000_000;
|
||||
store.set_fuel(fuel).ok();
|
||||
|
||||
store
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WasmPluginHost {
|
||||
fn default() -> Self {
|
||||
Self::new().expect("Failed to create WASM host")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "wasm"))]
|
||||
mod wasm_stub {
|
||||
use super::*;
|
||||
|
||||
pub struct WasmPluginHost {
|
||||
limits: ResourceLimits,
|
||||
}
|
||||
|
||||
impl WasmPluginHost {
|
||||
pub fn new() -> Result<Self> {
|
||||
Ok(Self {
|
||||
limits: ResourceLimits::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn set_limits(&mut self, limits: ResourceLimits) {
|
||||
self.limits = limits;
|
||||
}
|
||||
|
||||
pub fn load(&mut self, _wasm_bytes: &[u8]) -> Result<PluginId> {
|
||||
Err(PluginError::Wasm(
|
||||
"WASM support not enabled. Compile with --features wasm".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn unload(&mut self, _id: PluginId) -> Result<()> {
|
||||
Err(PluginError::Wasm("WASM support not enabled".to_string()))
|
||||
}
|
||||
|
||||
pub fn list(&self) -> Vec<(PluginId, &str)> {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WasmPluginHost {
|
||||
fn default() -> Self {
|
||||
Self::new().expect("Failed to create WASM stub host")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
pub use wasm_impl::*;
|
||||
|
||||
#[cfg(not(feature = "wasm"))]
|
||||
pub use wasm_stub::*;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_resource_limits_default() {
|
||||
let limits = ResourceLimits::default();
|
||||
assert_eq!(limits.max_memory_mb, 64);
|
||||
assert_eq!(limits.max_cpu_time_ms, 5000);
|
||||
assert!(!limits.allow_network);
|
||||
assert!(!limits.allow_filesystem);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wasm_host_creation() {
|
||||
let host = WasmPluginHost::new();
|
||||
assert!(host.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(feature = "wasm"))]
|
||||
fn test_wasm_disabled_load_fails() {
|
||||
let mut host = WasmPluginHost::new().unwrap();
|
||||
let result = host.load(&[0x00, 0x61, 0x73, 0x6d]);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "musicfs-search"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
|
||||
tantivy.workspace = true
|
||||
moka.workspace = true
|
||||
parking_lot.workspace = true
|
||||
tokio = { workspace = true, features = ["sync", "time"] }
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
rusqlite.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
|
||||
@@ -0,0 +1,321 @@
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SmartCollection {
|
||||
pub id: i64,
|
||||
pub name: String,
|
||||
pub query: CollectionQuery,
|
||||
pub created_at: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum CollectionQuery {
|
||||
Match {
|
||||
field: String,
|
||||
pattern: String,
|
||||
},
|
||||
DateRange {
|
||||
field: String,
|
||||
start: i32,
|
||||
end: i32,
|
||||
},
|
||||
RecentlyAdded {
|
||||
days: u32,
|
||||
},
|
||||
RecentlyPlayed {
|
||||
days: u32,
|
||||
},
|
||||
MostPlayed {
|
||||
limit: u32,
|
||||
},
|
||||
Genre {
|
||||
genre: String,
|
||||
},
|
||||
Compound {
|
||||
op: BoolOp,
|
||||
children: Vec<CollectionQuery>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub enum BoolOp {
|
||||
And,
|
||||
Or,
|
||||
}
|
||||
|
||||
impl CollectionQuery {
|
||||
pub fn to_tantivy_query(&self) -> String {
|
||||
match self {
|
||||
CollectionQuery::Match { field, pattern } => {
|
||||
format!("{}:{}", field, pattern)
|
||||
}
|
||||
CollectionQuery::DateRange { field, start, end } => {
|
||||
format!("{}:[{} TO {}]", field, start, end)
|
||||
}
|
||||
CollectionQuery::Genre { genre } => {
|
||||
format!("genre:{}", genre)
|
||||
}
|
||||
CollectionQuery::Compound { op, children } => {
|
||||
let sep = match op {
|
||||
BoolOp::And => " AND ",
|
||||
BoolOp::Or => " OR ",
|
||||
};
|
||||
let parts: Vec<_> = children
|
||||
.iter()
|
||||
.map(|c| format!("({})", c.to_tantivy_query()))
|
||||
.collect();
|
||||
parts.join(sep)
|
||||
}
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_dynamic(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
CollectionQuery::RecentlyAdded { .. }
|
||||
| CollectionQuery::RecentlyPlayed { .. }
|
||||
| CollectionQuery::MostPlayed { .. }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CollectionStore {
|
||||
db: Mutex<rusqlite::Connection>,
|
||||
}
|
||||
|
||||
impl CollectionStore {
|
||||
pub fn new(db_path: &Path) -> Result<Self, CollectionError> {
|
||||
let db = rusqlite::Connection::open(db_path)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS collections (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
query_json TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
info!(path = ?db_path, "Collection store opened");
|
||||
Ok(Self { db: Mutex::new(db) })
|
||||
}
|
||||
|
||||
pub fn create(
|
||||
&self,
|
||||
name: &str,
|
||||
query: CollectionQuery,
|
||||
) -> Result<SmartCollection, CollectionError> {
|
||||
info!(name = %name, "Creating collection");
|
||||
let query_json = serde_json::to_string(&query)?;
|
||||
let now = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs() as i64;
|
||||
|
||||
let db = self.db.lock();
|
||||
db.execute(
|
||||
"INSERT INTO collections (name, query_json, created_at) VALUES (?1, ?2, ?3)",
|
||||
rusqlite::params![name, query_json, now],
|
||||
)?;
|
||||
|
||||
let id = db.last_insert_rowid();
|
||||
debug!(id = id, name = %name, "Collection created");
|
||||
|
||||
Ok(SmartCollection {
|
||||
id,
|
||||
name: name.to_string(),
|
||||
query,
|
||||
created_at: SystemTime::UNIX_EPOCH + Duration::from_secs(now as u64),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn list(&self) -> Result<Vec<SmartCollection>, CollectionError> {
|
||||
let db = self.db.lock();
|
||||
let mut stmt = db.prepare("SELECT id, name, query_json, created_at FROM collections")?;
|
||||
|
||||
let collections = stmt.query_map([], |row| {
|
||||
let query_json: String = row.get(2)?;
|
||||
let created_secs: i64 = row.get(3)?;
|
||||
|
||||
let query = match serde_json::from_str(&query_json) {
|
||||
Ok(q) => q,
|
||||
Err(e) => {
|
||||
warn!("Failed to parse collection query JSON: {}", e);
|
||||
CollectionQuery::Match {
|
||||
field: "title".to_string(),
|
||||
pattern: "*".to_string(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(SmartCollection {
|
||||
id: row.get(0)?,
|
||||
name: row.get(1)?,
|
||||
query,
|
||||
created_at: SystemTime::UNIX_EPOCH + Duration::from_secs(created_secs as u64),
|
||||
})
|
||||
})?;
|
||||
|
||||
collections
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(CollectionError::from)
|
||||
}
|
||||
|
||||
pub fn get(&self, name: &str) -> Result<Option<SmartCollection>, CollectionError> {
|
||||
let db = self.db.lock();
|
||||
let mut stmt =
|
||||
db.prepare("SELECT id, name, query_json, created_at FROM collections WHERE name = ?1")?;
|
||||
|
||||
let result = stmt
|
||||
.query_row([name], |row| {
|
||||
let query_json: String = row.get(2)?;
|
||||
let created_secs: i64 = row.get(3)?;
|
||||
|
||||
let query = match serde_json::from_str(&query_json) {
|
||||
Ok(q) => q,
|
||||
Err(e) => {
|
||||
warn!("Failed to parse collection query JSON: {}", e);
|
||||
CollectionQuery::Match {
|
||||
field: "title".to_string(),
|
||||
pattern: "*".to_string(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(SmartCollection {
|
||||
id: row.get(0)?,
|
||||
name: row.get(1)?,
|
||||
query,
|
||||
created_at: SystemTime::UNIX_EPOCH + Duration::from_secs(created_secs as u64),
|
||||
})
|
||||
})
|
||||
.ok();
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn delete(&self, name: &str) -> Result<(), CollectionError> {
|
||||
info!(name = %name, "Deleting collection");
|
||||
let db = self.db.lock();
|
||||
db.execute("DELETE FROM collections WHERE name = ?1", [name])?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn builtin_collections() -> Vec<SmartCollection> {
|
||||
vec![
|
||||
SmartCollection {
|
||||
id: -1,
|
||||
name: "Recently Added".to_string(),
|
||||
query: CollectionQuery::RecentlyAdded { days: 30 },
|
||||
created_at: SystemTime::UNIX_EPOCH,
|
||||
},
|
||||
SmartCollection {
|
||||
id: -2,
|
||||
name: "80s Music".to_string(),
|
||||
query: CollectionQuery::DateRange {
|
||||
field: "year".to_string(),
|
||||
start: 1980,
|
||||
end: 1989,
|
||||
},
|
||||
created_at: SystemTime::UNIX_EPOCH,
|
||||
},
|
||||
SmartCollection {
|
||||
id: -3,
|
||||
name: "90s Music".to_string(),
|
||||
query: CollectionQuery::DateRange {
|
||||
field: "year".to_string(),
|
||||
start: 1990,
|
||||
end: 1999,
|
||||
},
|
||||
created_at: SystemTime::UNIX_EPOCH,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CollectionError {
|
||||
#[error("database error: {0}")]
|
||||
Database(#[from] rusqlite::Error),
|
||||
|
||||
#[error("serialization error: {0}")]
|
||||
Serialization(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_collection_crud() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("collections.db");
|
||||
let store = CollectionStore::new(&db_path).unwrap();
|
||||
|
||||
let collection = store
|
||||
.create(
|
||||
"Jazz",
|
||||
CollectionQuery::Genre {
|
||||
genre: "Jazz".to_string(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(collection.name, "Jazz");
|
||||
|
||||
let collections = store.list().unwrap();
|
||||
assert_eq!(collections.len(), 1);
|
||||
|
||||
store.delete("Jazz").unwrap();
|
||||
let collections = store.list().unwrap();
|
||||
assert_eq!(collections.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compound_query() {
|
||||
let query = CollectionQuery::Compound {
|
||||
op: BoolOp::And,
|
||||
children: vec![
|
||||
CollectionQuery::Genre {
|
||||
genre: "Metal".to_string(),
|
||||
},
|
||||
CollectionQuery::DateRange {
|
||||
field: "year".to_string(),
|
||||
start: 1980,
|
||||
end: 1989,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
let tantivy_query = query.to_tantivy_query();
|
||||
assert!(tantivy_query.contains("genre:Metal"));
|
||||
assert!(tantivy_query.contains("year:[1980 TO 1989]"));
|
||||
assert!(tantivy_query.contains(" AND "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_builtin_collections() {
|
||||
let builtins = builtin_collections();
|
||||
assert_eq!(builtins.len(), 3);
|
||||
assert!(builtins.iter().any(|c| c.name == "Recently Added"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dynamic_query_detection() {
|
||||
assert!(CollectionQuery::RecentlyAdded { days: 30 }.is_dynamic());
|
||||
assert!(CollectionQuery::RecentlyPlayed { days: 7 }.is_dynamic());
|
||||
assert!(CollectionQuery::MostPlayed { limit: 100 }.is_dynamic());
|
||||
assert!(!CollectionQuery::Genre {
|
||||
genre: "Rock".to_string()
|
||||
}
|
||||
.is_dynamic());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
use musicfs_core::{AudioMeta, FileId, FileMeta, VirtualPath};
|
||||
use parking_lot::RwLock;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tantivy::collector::TopDocs;
|
||||
use tantivy::query::{BooleanQuery, FuzzyTermQuery, Occur, Query, QueryParser};
|
||||
use tantivy::schema::{Field, Schema, Value, INDEXED, STORED, TEXT};
|
||||
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const SCHEMA_VERSION: u32 = 1;
|
||||
|
||||
pub struct SearchIndex {
|
||||
index: Index,
|
||||
reader: IndexReader,
|
||||
writer: Arc<RwLock<IndexWriter>>,
|
||||
schema: SearchSchema,
|
||||
pub schema_version: u32,
|
||||
}
|
||||
|
||||
struct SearchSchema {
|
||||
schema: Schema,
|
||||
file_id: Field,
|
||||
virtual_path: Field,
|
||||
artist: Field,
|
||||
album: Field,
|
||||
album_artist: Field,
|
||||
title: Field,
|
||||
genre: Field,
|
||||
composer: Field,
|
||||
year: Field,
|
||||
duration_ms: Field,
|
||||
bitrate: Field,
|
||||
sample_rate: Field,
|
||||
}
|
||||
|
||||
impl SearchSchema {
|
||||
fn new() -> Self {
|
||||
let mut builder = Schema::builder();
|
||||
|
||||
Self {
|
||||
file_id: builder.add_u64_field("file_id", INDEXED | STORED),
|
||||
virtual_path: builder.add_text_field("virtual_path", STORED),
|
||||
artist: builder.add_text_field("artist", TEXT | STORED),
|
||||
album: builder.add_text_field("album", TEXT | STORED),
|
||||
album_artist: builder.add_text_field("album_artist", TEXT | STORED),
|
||||
title: builder.add_text_field("title", TEXT | STORED),
|
||||
genre: builder.add_text_field("genre", TEXT | STORED),
|
||||
composer: builder.add_text_field("composer", TEXT | STORED),
|
||||
year: builder.add_u64_field("year", INDEXED | STORED),
|
||||
duration_ms: builder.add_u64_field("duration_ms", STORED),
|
||||
bitrate: builder.add_u64_field("bitrate", STORED),
|
||||
sample_rate: builder.add_u64_field("sample_rate", STORED),
|
||||
schema: builder.build(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SearchHit {
|
||||
pub file_id: FileId,
|
||||
pub virtual_path: VirtualPath,
|
||||
pub artist: Option<String>,
|
||||
pub album: Option<String>,
|
||||
pub title: Option<String>,
|
||||
pub score: f32,
|
||||
}
|
||||
|
||||
impl SearchIndex {
|
||||
pub fn open(index_path: &Path) -> Result<Self, SearchError> {
|
||||
let schema_obj = SearchSchema::new();
|
||||
|
||||
let index = if index_path.exists() && index_path.join("meta.json").exists() {
|
||||
Index::open_in_dir(index_path)?
|
||||
} else {
|
||||
std::fs::create_dir_all(index_path)?;
|
||||
Index::create_in_dir(index_path, schema_obj.schema.clone())?
|
||||
};
|
||||
|
||||
let reader = index
|
||||
.reader_builder()
|
||||
.reload_policy(ReloadPolicy::OnCommitWithDelay)
|
||||
.try_into()?;
|
||||
|
||||
let writer = index.writer(50_000_000)?;
|
||||
|
||||
info!("Search index opened at {:?}", index_path);
|
||||
|
||||
Ok(Self {
|
||||
index,
|
||||
reader,
|
||||
writer: Arc::new(RwLock::new(writer)),
|
||||
schema: schema_obj,
|
||||
schema_version: SCHEMA_VERSION,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_with_recovery(index_path: &Path) -> Result<Self, SearchError> {
|
||||
match Self::open(index_path) {
|
||||
Ok(index) => {
|
||||
let docs = index.reader.searcher().num_docs();
|
||||
info!(docs, "Search index opened successfully");
|
||||
Ok(index)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
error = %e,
|
||||
path = ?index_path,
|
||||
"Search index corrupted, rebuilding from scratch"
|
||||
);
|
||||
if index_path.exists() {
|
||||
std::fs::remove_dir_all(index_path).map_err(SearchError::Io)?;
|
||||
}
|
||||
Self::open(index_path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn index_file(&self, file: &FileMeta) -> Result<(), SearchError> {
|
||||
let mut doc = TantivyDocument::new();
|
||||
|
||||
doc.add_u64(self.schema.file_id, file.id.0 as u64);
|
||||
doc.add_text(self.schema.virtual_path, file.virtual_path.as_str());
|
||||
|
||||
if let Some(ref audio) = file.audio {
|
||||
Self::add_audio_fields(&mut doc, &self.schema, audio);
|
||||
}
|
||||
|
||||
self.writer.read().add_document(doc)?;
|
||||
debug!("Indexed file {:?}", file.id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_audio_fields(doc: &mut TantivyDocument, schema: &SearchSchema, audio: &AudioMeta) {
|
||||
if let Some(ref v) = audio.artist {
|
||||
doc.add_text(schema.artist, v);
|
||||
}
|
||||
if let Some(ref v) = audio.album {
|
||||
doc.add_text(schema.album, v);
|
||||
}
|
||||
if let Some(ref v) = audio.album_artist {
|
||||
doc.add_text(schema.album_artist, v);
|
||||
}
|
||||
if let Some(ref v) = audio.title {
|
||||
doc.add_text(schema.title, v);
|
||||
}
|
||||
if let Some(ref v) = audio.genre {
|
||||
doc.add_text(schema.genre, v);
|
||||
}
|
||||
if let Some(ref v) = audio.year {
|
||||
doc.add_u64(schema.year, *v as u64);
|
||||
}
|
||||
if let Some(v) = audio.duration_ms {
|
||||
doc.add_u64(schema.duration_ms, v);
|
||||
}
|
||||
if let Some(v) = audio.bitrate {
|
||||
doc.add_u64(schema.bitrate, v as u64);
|
||||
}
|
||||
if let Some(v) = audio.sample_rate {
|
||||
doc.add_u64(schema.sample_rate, v as u64);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove_file(&self, file_id: FileId) -> Result<(), SearchError> {
|
||||
let term = tantivy::Term::from_field_u64(self.schema.file_id, file_id.0 as u64);
|
||||
self.writer.read().delete_term(term);
|
||||
debug!("Removed file {:?} from index", file_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn remove_by_path(&self, path: &VirtualPath) -> Result<bool, SearchError> {
|
||||
let searcher = self.reader.searcher();
|
||||
let query_parser = QueryParser::for_index(&self.index, vec![self.schema.virtual_path]);
|
||||
let query = query_parser.parse_query(&format!("\"{}\"", path.as_str()))?;
|
||||
let top_docs = searcher.search(&query, &TopDocs::with_limit(1))?;
|
||||
|
||||
if let Some((_, doc_address)) = top_docs.first() {
|
||||
let doc: TantivyDocument = searcher.doc(*doc_address)?;
|
||||
if let Some(file_id) = doc.get_first(self.schema.file_id).and_then(|v| v.as_u64()) {
|
||||
self.remove_file(FileId(file_id as i64))?;
|
||||
debug!("Removed file by path {:?}", path);
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub fn commit(&self) -> Result<(), SearchError> {
|
||||
self.writer.write().commit()?;
|
||||
self.reader.reload()?;
|
||||
info!("Search index committed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn search(&self, query_str: &str, limit: usize) -> Result<Vec<SearchHit>, SearchError> {
|
||||
let searcher = self.reader.searcher();
|
||||
|
||||
let default_fields = vec![
|
||||
self.schema.artist,
|
||||
self.schema.album,
|
||||
self.schema.album_artist,
|
||||
self.schema.title,
|
||||
self.schema.genre,
|
||||
self.schema.composer,
|
||||
];
|
||||
|
||||
let query: Box<dyn Query> =
|
||||
if let Some((term, distance)) = Self::parse_fuzzy_query(query_str) {
|
||||
let subqueries: Vec<(Occur, Box<dyn Query>)> = default_fields
|
||||
.iter()
|
||||
.map(|&field| {
|
||||
let term = Term::from_field_text(field, &term);
|
||||
let fuzzy = FuzzyTermQuery::new(term, distance, true);
|
||||
(Occur::Should, Box::new(fuzzy) as Box<dyn Query>)
|
||||
})
|
||||
.collect();
|
||||
Box::new(BooleanQuery::new(subqueries))
|
||||
} else {
|
||||
let query_parser = QueryParser::for_index(&self.index, default_fields);
|
||||
query_parser.parse_query(query_str)?
|
||||
};
|
||||
|
||||
let top_docs = searcher.search(&*query, &TopDocs::with_limit(limit))?;
|
||||
|
||||
let mut results = Vec::with_capacity(top_docs.len());
|
||||
for (score, doc_address) in top_docs {
|
||||
let doc: TantivyDocument = searcher.doc(doc_address)?;
|
||||
|
||||
let file_id = doc
|
||||
.get_first(self.schema.file_id)
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|id| FileId(id as i64))
|
||||
.ok_or(SearchError::CorruptedIndex)?;
|
||||
|
||||
let virtual_path = doc
|
||||
.get_first(self.schema.virtual_path)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| VirtualPath::new(s))
|
||||
.ok_or(SearchError::CorruptedIndex)?;
|
||||
|
||||
results.push(SearchHit {
|
||||
file_id,
|
||||
virtual_path,
|
||||
artist: doc
|
||||
.get_first(self.schema.artist)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from),
|
||||
album: doc
|
||||
.get_first(self.schema.album)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from),
|
||||
title: doc
|
||||
.get_first(self.schema.title)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from),
|
||||
score,
|
||||
});
|
||||
}
|
||||
|
||||
debug!("Search '{}' returned {} results", query_str, results.len());
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub fn count(&self) -> u64 {
|
||||
self.reader.searcher().num_docs()
|
||||
}
|
||||
|
||||
fn parse_fuzzy_query(query_str: &str) -> Option<(String, u8)> {
|
||||
let query_str = query_str.trim();
|
||||
if let Some(tilde_pos) = query_str.rfind('~') {
|
||||
let term = &query_str[..tilde_pos];
|
||||
let distance_str = &query_str[tilde_pos + 1..];
|
||||
if !term.is_empty() && !term.contains(':') && !term.contains(' ') {
|
||||
if let Ok(distance) = distance_str.parse::<u8>() {
|
||||
if distance <= 2 {
|
||||
return Some((term.to_lowercase(), distance));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum SearchError {
|
||||
#[error("tantivy error: {0}")]
|
||||
Tantivy(#[from] tantivy::TantivyError),
|
||||
|
||||
#[error("query parse error: {0}")]
|
||||
QueryParse(#[from] tantivy::query::QueryParserError),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("corrupted search index")]
|
||||
CorruptedIndex,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_core::{AudioFormat, OriginId, RealPath};
|
||||
use std::path::PathBuf;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_file(id: i64, artist: &str, album: &str, title: &str) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(format!("/{}/{}/{}.flac", artist, album, title)),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from("test.flac"),
|
||||
},
|
||||
size: 1000,
|
||||
mtime: std::time::SystemTime::UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: Some(AudioMeta {
|
||||
artist: Some(artist.to_string()),
|
||||
album: Some(album.to_string()),
|
||||
title: Some(title.to_string()),
|
||||
genre: Some("Metal".to_string()),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_search_basic() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
|
||||
index
|
||||
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
|
||||
.unwrap();
|
||||
index
|
||||
.index_file(&make_file(2, "Metallica", "Master of Puppets", "Battery"))
|
||||
.unwrap();
|
||||
index
|
||||
.index_file(&make_file(3, "Iron Maiden", "Powerslave", "Aces High"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
let results = index.search("metallica", 10).unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
|
||||
let results = index.search("sandman", 10).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].title.as_deref(), Some("Enter Sandman"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_search_fuzzy() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
|
||||
index
|
||||
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
let results = index.search("metalica~1", 10).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_search_genre() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
|
||||
index
|
||||
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
let results = index.search("genre:Metal", 10).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_file() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
|
||||
index
|
||||
.index_file(&make_file(1, "Test", "Album", "Song"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
assert_eq!(index.search("test", 10).unwrap().len(), 1);
|
||||
|
||||
index.remove_file(FileId(1)).unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
assert_eq!(index.search("test", 10).unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_index_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
index
|
||||
.index_file(&make_file(1, "Artist", "Album", "Track"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
let results = index.search("artist", 10).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
use crate::index::{SearchError, SearchIndex};
|
||||
use musicfs_core::{Event, EventBus, FileMeta};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, error, info, info_span, warn, Instrument};
|
||||
|
||||
pub trait MetadataLookup: Send + Sync {
|
||||
fn lookup(&self, path: &musicfs_core::VirtualPath) -> Option<FileMeta>;
|
||||
}
|
||||
|
||||
pub struct Indexer<M: MetadataLookup> {
|
||||
index: Arc<SearchIndex>,
|
||||
event_bus: Arc<EventBus>,
|
||||
metadata_lookup: Arc<M>,
|
||||
}
|
||||
|
||||
impl<M: MetadataLookup + 'static> Indexer<M> {
|
||||
pub fn new(index: Arc<SearchIndex>, event_bus: Arc<EventBus>, metadata_lookup: Arc<M>) -> Self {
|
||||
Self {
|
||||
index,
|
||||
event_bus,
|
||||
metadata_lookup,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(self) -> IndexerHandle {
|
||||
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
|
||||
let mut event_rx = self.event_bus.subscribe();
|
||||
|
||||
info!("Search indexer starting");
|
||||
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let mut pending_commit = false;
|
||||
let mut commit_timer = tokio::time::interval(std::time::Duration::from_secs(5));
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = event_rx.recv() => {
|
||||
match result {
|
||||
Ok(event) => {
|
||||
if let Err(e) = self.handle_event(&event) {
|
||||
error!("Indexer error: {}", e);
|
||||
}
|
||||
pending_commit = true;
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
warn!(skipped = n, "Indexer lagged, skipped events");
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
debug!("Event channel closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = commit_timer.tick() => {
|
||||
if pending_commit {
|
||||
if let Err(e) = self.index.commit() {
|
||||
error!("Index commit error: {}", e);
|
||||
}
|
||||
pending_commit = false;
|
||||
}
|
||||
}
|
||||
_ = stop_rx.recv() => {
|
||||
info!("Indexer stopping");
|
||||
if pending_commit {
|
||||
let _ = self.index.commit();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.instrument(info_span!("search_indexer")),
|
||||
);
|
||||
|
||||
IndexerHandle { stop_tx }
|
||||
}
|
||||
|
||||
fn handle_event(&self, event: &Event) -> Result<(), SearchError> {
|
||||
match event {
|
||||
Event::FileAdded { path, .. } => {
|
||||
debug!("Indexing added file: {:?}", path);
|
||||
if let Some(meta) = self.metadata_lookup.lookup(path) {
|
||||
self.index.index_file(&meta)?;
|
||||
} else {
|
||||
warn!("No metadata found for added file: {:?}", path);
|
||||
}
|
||||
}
|
||||
Event::FileRemoved { path, file_id } => {
|
||||
debug!("Removing from index: {:?}", path);
|
||||
if let Some(id) = file_id {
|
||||
self.index.remove_file(*id)?;
|
||||
} else if let Some(meta) = self.metadata_lookup.lookup(path) {
|
||||
self.index.remove_file(meta.id)?;
|
||||
} else {
|
||||
self.index.remove_by_path(path)?;
|
||||
}
|
||||
}
|
||||
Event::FileModified { path } => {
|
||||
debug!("Re-indexing modified file: {:?}", path);
|
||||
if let Some(meta) = self.metadata_lookup.lookup(path) {
|
||||
self.index.remove_file(meta.id)?;
|
||||
self.index.index_file(&meta)?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn index_batch(&self, files: &[FileMeta]) -> Result<usize, SearchError> {
|
||||
let mut count = 0;
|
||||
for file in files {
|
||||
self.index.index_file(file)?;
|
||||
count += 1;
|
||||
}
|
||||
self.index.commit()?;
|
||||
info!("Indexed {} files", count);
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct IndexerHandle {
|
||||
stop_tx: mpsc::Sender<()>,
|
||||
}
|
||||
|
||||
impl IndexerHandle {
|
||||
pub async fn stop(self) {
|
||||
let _ = self.stop_tx.send(()).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_core::{AudioFormat, AudioMeta, FileId, OriginId, RealPath, VirtualPath};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::RwLock;
|
||||
use tempfile::TempDir;
|
||||
|
||||
struct MockMetadataLookup {
|
||||
files: RwLock<HashMap<String, FileMeta>>,
|
||||
}
|
||||
|
||||
impl MockMetadataLookup {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
files: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn insert(&self, meta: FileMeta) {
|
||||
self.files
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(meta.virtual_path.as_str().to_string(), meta);
|
||||
}
|
||||
}
|
||||
|
||||
impl MetadataLookup for MockMetadataLookup {
|
||||
fn lookup(&self, path: &VirtualPath) -> Option<FileMeta> {
|
||||
self.files.read().unwrap().get(path.as_str()).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
fn make_file(id: i64, path: &str, artist: &str, title: &str) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(path),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from("test.flac"),
|
||||
},
|
||||
size: 1000,
|
||||
mtime: std::time::SystemTime::UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: Some(AudioMeta {
|
||||
artist: Some(artist.to_string()),
|
||||
title: Some(title.to_string()),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_indexer_handles_file_added() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
|
||||
let event_bus = Arc::new(EventBus::default());
|
||||
let metadata = Arc::new(MockMetadataLookup::new());
|
||||
|
||||
let file = make_file(1, "/Artist/Album/Track.flac", "Artist", "Track");
|
||||
metadata.insert(file.clone());
|
||||
|
||||
let indexer = Indexer::new(index.clone(), event_bus.clone(), metadata);
|
||||
let handle = indexer.start();
|
||||
|
||||
event_bus.publish(Event::FileAdded {
|
||||
path: VirtualPath::new("/Artist/Album/Track.flac"),
|
||||
origin_id: OriginId::from("test"),
|
||||
});
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
index.commit().unwrap();
|
||||
|
||||
let results = index.search("artist", 10).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
|
||||
handle.stop().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_index_batch() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
|
||||
let event_bus = Arc::new(EventBus::default());
|
||||
let metadata = Arc::new(MockMetadataLookup::new());
|
||||
|
||||
let indexer = Indexer::new(index.clone(), event_bus, metadata);
|
||||
|
||||
let files = vec![
|
||||
make_file(1, "/a.flac", "Artist1", "Song1"),
|
||||
make_file(2, "/b.flac", "Artist2", "Song2"),
|
||||
make_file(3, "/c.flac", "Artist3", "Song3"),
|
||||
];
|
||||
|
||||
let count = indexer.index_batch(&files).unwrap();
|
||||
assert_eq!(count, 3);
|
||||
|
||||
let results = index.search("artist1", 10).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
mod collections;
|
||||
mod index;
|
||||
mod indexer;
|
||||
mod query;
|
||||
|
||||
pub use collections::{
|
||||
builtin_collections, BoolOp, CollectionError, CollectionQuery, CollectionStore, SmartCollection,
|
||||
};
|
||||
pub use index::{SearchError, SearchHit, SearchIndex};
|
||||
pub use indexer::{Indexer, IndexerHandle, MetadataLookup};
|
||||
pub use query::SearchQueryBuilder;
|
||||
@@ -0,0 +1,78 @@
|
||||
use tantivy::query::{BooleanQuery, FuzzyTermQuery, Occur, Query};
|
||||
use tantivy::schema::Field;
|
||||
use tantivy::Term;
|
||||
|
||||
pub struct SearchQueryBuilder {
|
||||
fields: Vec<Field>,
|
||||
default_fuzziness: u8,
|
||||
}
|
||||
|
||||
impl SearchQueryBuilder {
|
||||
pub fn new(fields: Vec<Field>) -> Self {
|
||||
Self {
|
||||
fields,
|
||||
default_fuzziness: 1,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_fuzziness(mut self, fuzziness: u8) -> Self {
|
||||
self.default_fuzziness = fuzziness;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn build_fuzzy(&self, query_text: &str) -> Box<dyn Query> {
|
||||
let terms: Vec<_> = query_text
|
||||
.split_whitespace()
|
||||
.filter(|t| !t.is_empty())
|
||||
.collect();
|
||||
|
||||
if terms.is_empty() {
|
||||
return Box::new(tantivy::query::AllQuery);
|
||||
}
|
||||
|
||||
let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new();
|
||||
|
||||
for term in terms {
|
||||
let mut field_queries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
|
||||
|
||||
for field in &self.fields {
|
||||
let fuzzy = FuzzyTermQuery::new(
|
||||
Term::from_field_text(*field, &term.to_lowercase()),
|
||||
self.default_fuzziness,
|
||||
true,
|
||||
);
|
||||
field_queries.push((Occur::Should, Box::new(fuzzy)));
|
||||
}
|
||||
|
||||
let field_union = BooleanQuery::new(field_queries);
|
||||
clauses.push((Occur::Must, Box::new(field_union)));
|
||||
}
|
||||
|
||||
Box::new(BooleanQuery::new(clauses))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tantivy::schema::{Schema, TEXT};
|
||||
|
||||
#[test]
|
||||
fn test_query_builder() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let artist = schema_builder.add_text_field("artist", TEXT);
|
||||
let title = schema_builder.add_text_field("title", TEXT);
|
||||
|
||||
let builder = SearchQueryBuilder::new(vec![artist, title]);
|
||||
let _query = builder.build_fuzzy("metallica sandman");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_query() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let artist = schema_builder.add_text_field("artist", TEXT);
|
||||
|
||||
let builder = SearchQueryBuilder::new(vec![artist]);
|
||||
let _query = builder.build_fuzzy("");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "musicfs-sync"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-origins = { path = "../musicfs-origins" }
|
||||
|
||||
fastcdc = "3"
|
||||
xxhash-rust = { version = "0.8", features = ["xxh64"] }
|
||||
notify = "6"
|
||||
rmp-serde = "1"
|
||||
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
@@ -0,0 +1,232 @@
|
||||
use fastcdc::v2020::FastCDC;
|
||||
use musicfs_core::ChunkHash;
|
||||
|
||||
pub struct CdcChunker {
|
||||
min_size: u32,
|
||||
avg_size: u32,
|
||||
max_size: u32,
|
||||
}
|
||||
|
||||
impl Default for CdcChunker {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
min_size: 16 * 1024,
|
||||
avg_size: 64 * 1024,
|
||||
max_size: 256 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Chunk {
|
||||
pub hash: ChunkHash,
|
||||
pub offset: u64,
|
||||
pub length: u32,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ChunkRef<'a> {
|
||||
pub hash: ChunkHash,
|
||||
pub offset: u64,
|
||||
pub length: u32,
|
||||
pub data: &'a [u8],
|
||||
}
|
||||
|
||||
impl CdcChunker {
|
||||
pub fn new(min_size: u32, avg_size: u32, max_size: u32) -> Self {
|
||||
Self {
|
||||
min_size,
|
||||
avg_size,
|
||||
max_size,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chunk(&self, data: &[u8]) -> Vec<Chunk> {
|
||||
let chunker = FastCDC::new(data, self.min_size, self.avg_size, self.max_size);
|
||||
|
||||
chunker
|
||||
.map(|c| {
|
||||
let chunk_data = &data[c.offset..c.offset + c.length];
|
||||
Chunk {
|
||||
hash: ChunkHash::from_bytes(chunk_data),
|
||||
offset: c.offset as u64,
|
||||
length: c.length as u32,
|
||||
data: chunk_data.to_vec(),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn chunk_refs<'a>(&self, data: &'a [u8]) -> Vec<ChunkRef<'a>> {
|
||||
let chunker = FastCDC::new(data, self.min_size, self.avg_size, self.max_size);
|
||||
|
||||
chunker
|
||||
.map(|c| {
|
||||
let chunk_data = &data[c.offset..c.offset + c.length];
|
||||
ChunkRef {
|
||||
hash: ChunkHash::from_bytes(chunk_data),
|
||||
offset: c.offset as u64,
|
||||
length: c.length as u32,
|
||||
data: chunk_data,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn chunk_streaming<F>(&self, data: &[u8], mut processor: F) -> usize
|
||||
where
|
||||
F: FnMut(ChunkRef<'_>),
|
||||
{
|
||||
let chunker = FastCDC::new(data, self.min_size, self.avg_size, self.max_size);
|
||||
let mut count = 0;
|
||||
|
||||
for c in chunker {
|
||||
let chunk_data = &data[c.offset..c.offset + c.length];
|
||||
processor(ChunkRef {
|
||||
hash: ChunkHash::from_bytes(chunk_data),
|
||||
offset: c.offset as u64,
|
||||
length: c.length as u32,
|
||||
data: chunk_data,
|
||||
});
|
||||
count += 1;
|
||||
}
|
||||
count
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_cdc_basic() {
|
||||
let chunker = CdcChunker::default();
|
||||
let data = vec![0u8; 256 * 1024];
|
||||
|
||||
let chunks = chunker.chunk(&data);
|
||||
|
||||
assert!(!chunks.is_empty());
|
||||
|
||||
let total: u64 = chunks.iter().map(|c| c.length as u64).sum();
|
||||
assert_eq!(total, data.len() as u64);
|
||||
|
||||
let mut offset = 0u64;
|
||||
for chunk in &chunks {
|
||||
assert_eq!(chunk.offset, offset);
|
||||
offset += chunk.length as u64;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cdc_stable_boundaries() {
|
||||
let chunker = CdcChunker::new(4 * 1024, 16 * 1024, 64 * 1024);
|
||||
|
||||
let mut data1 = vec![0u8; 512 * 1024];
|
||||
for (i, b) in data1.iter_mut().enumerate() {
|
||||
*b = ((i * 17 + 31) % 256) as u8;
|
||||
}
|
||||
|
||||
let mut data2 = vec![0xFFu8; 1024];
|
||||
data2.extend_from_slice(&data1);
|
||||
|
||||
let chunks1 = chunker.chunk(&data1);
|
||||
let chunks2 = chunker.chunk(&data2);
|
||||
|
||||
let hashes1: std::collections::HashSet<_> = chunks1.iter().map(|c| c.hash).collect();
|
||||
let hashes2: std::collections::HashSet<_> = chunks2.iter().map(|c| c.hash).collect();
|
||||
|
||||
let shared = hashes1.intersection(&hashes2).count();
|
||||
|
||||
assert!(
|
||||
shared > 0,
|
||||
"CDC should produce stable boundaries, got {} chunks in original, {} after prepend",
|
||||
chunks1.len(),
|
||||
chunks2.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cdc_chunk_sizes() {
|
||||
let chunker = CdcChunker::default();
|
||||
|
||||
let data: Vec<u8> = (0..1024 * 1024)
|
||||
.map(|i| ((i * 17 + 31) % 256) as u8)
|
||||
.collect();
|
||||
|
||||
let chunks = chunker.chunk(&data);
|
||||
|
||||
for chunk in &chunks {
|
||||
if chunk.offset + chunk.length as u64 != data.len() as u64 {
|
||||
assert!(
|
||||
chunk.length >= chunker.min_size / 2,
|
||||
"Chunk too small: {}",
|
||||
chunk.length
|
||||
);
|
||||
assert!(
|
||||
chunk.length <= chunker.max_size * 2,
|
||||
"Chunk too large: {}",
|
||||
chunk.length
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cdc_streaming() {
|
||||
let chunker = CdcChunker::default();
|
||||
let data = vec![0u8; 256 * 1024];
|
||||
|
||||
let mut streamed = Vec::new();
|
||||
let count = chunker.chunk_streaming(&data, |chunk| {
|
||||
streamed.push((chunk.hash, chunk.offset, chunk.length));
|
||||
});
|
||||
|
||||
let batched = chunker.chunk(&data);
|
||||
|
||||
assert_eq!(count, batched.len());
|
||||
for (i, chunk) in batched.iter().enumerate() {
|
||||
assert_eq!(streamed[i].0, chunk.hash);
|
||||
assert_eq!(streamed[i].1, chunk.offset);
|
||||
assert_eq!(streamed[i].2, chunk.length);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bandwidth_reduction_metadata_edit() {
|
||||
let chunker = CdcChunker::new(4 * 1024, 16 * 1024, 64 * 1024);
|
||||
|
||||
let mut state = 12345u64;
|
||||
let original: Vec<u8> = (0..2 * 1024 * 1024)
|
||||
.map(|_| {
|
||||
state = state.wrapping_mul(6364136223846793005).wrapping_add(1);
|
||||
(state >> 56) as u8
|
||||
})
|
||||
.collect();
|
||||
|
||||
let chunks1 = chunker.chunk(&original);
|
||||
let hashes1: std::collections::HashSet<_> = chunks1.iter().map(|c| c.hash).collect();
|
||||
|
||||
let mut modified = original.clone();
|
||||
let mid = modified.len() / 2;
|
||||
for i in mid..mid + 100 {
|
||||
modified[i] = 0xFF;
|
||||
}
|
||||
|
||||
let chunks2 = chunker.chunk(&modified);
|
||||
let hashes2: std::collections::HashSet<_> = chunks2.iter().map(|c| c.hash).collect();
|
||||
|
||||
let reused = hashes1.intersection(&hashes2).count();
|
||||
let reuse_ratio = reused as f64 / chunks2.len() as f64;
|
||||
|
||||
// NFR-6.4 requires >90% bandwidth reduction for typical edits
|
||||
assert!(
|
||||
reuse_ratio > 0.90,
|
||||
"Expected >90% chunk reuse for mid-file edit (NFR-6.4). Reused {}/{} chunks ({:.1}%, total {} original)",
|
||||
reused,
|
||||
chunks2.len(),
|
||||
reuse_ratio * 100.0,
|
||||
chunks1.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
use crate::cdc::CdcChunker;
|
||||
use musicfs_core::{ChunkHash, FileId, FileMeta, OriginId};
|
||||
use musicfs_origins::Origin;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::time::SystemTime;
|
||||
use tracing::{debug, info, trace};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScannedFile {
|
||||
pub path: PathBuf,
|
||||
pub origin_id: OriginId,
|
||||
pub size: u64,
|
||||
pub mtime: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ChangeSet {
|
||||
pub added: Vec<ScannedFile>,
|
||||
pub removed: Vec<FileId>,
|
||||
pub modified: Vec<(FileId, ManifestDiff)>,
|
||||
}
|
||||
|
||||
impl ChangeSet {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.added.is_empty() && self.removed.is_empty() && self.modified.is_empty()
|
||||
}
|
||||
|
||||
pub fn total_changes(&self) -> usize {
|
||||
self.added.len() + self.removed.len() + self.modified.len()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ManifestChunk {
|
||||
pub hash: ChunkHash,
|
||||
pub offset: u64,
|
||||
pub size: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ManifestDiff {
|
||||
pub reuse: Vec<ManifestChunk>,
|
||||
pub fetch: Vec<ManifestChunk>,
|
||||
pub orphaned: Vec<ChunkHash>,
|
||||
}
|
||||
|
||||
pub struct DeltaDetector {
|
||||
chunker: CdcChunker,
|
||||
}
|
||||
|
||||
impl DeltaDetector {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
chunker: CdcChunker::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_chunker(chunker: CdcChunker) -> Self {
|
||||
Self { chunker }
|
||||
}
|
||||
|
||||
pub async fn detect_changes(
|
||||
&self,
|
||||
origin: &dyn Origin,
|
||||
cached: &HashMap<FileId, FileMeta>,
|
||||
manifests: &HashMap<FileId, Vec<ManifestChunk>>,
|
||||
) -> Result<ChangeSet, DeltaError> {
|
||||
let origin_id = origin.id().clone();
|
||||
info!(origin_id = %origin_id, "Starting delta detection");
|
||||
|
||||
let mut changes = ChangeSet::default();
|
||||
|
||||
let origin_files = self.scan_origin(origin).await?;
|
||||
trace!(origin_id = %origin_id, scanned_count = origin_files.len(), "Completed origin scan");
|
||||
|
||||
let cached_by_path: HashMap<_, _> = cached
|
||||
.values()
|
||||
.map(|m| (m.real_path.path.clone(), m))
|
||||
.collect();
|
||||
|
||||
for scanned in &origin_files {
|
||||
if let Some(cached_file) = cached_by_path.get(&scanned.path) {
|
||||
if self.is_modified_scan(cached_file, scanned) {
|
||||
debug!(origin_id = %origin_id, path = ?scanned.path, "File modified");
|
||||
|
||||
if let Some(old_chunks) = manifests.get(&cached_file.id) {
|
||||
let new_chunks = self.compute_chunks_for_scan(origin, scanned).await?;
|
||||
let diff = self.compute_diff(old_chunks, &new_chunks);
|
||||
changes.modified.push((cached_file.id, diff));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
debug!(origin_id = %origin_id, path = ?scanned.path, "File added");
|
||||
changes.added.push(scanned.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let origin_paths: HashSet<_> = origin_files.iter().map(|f| &f.path).collect();
|
||||
|
||||
for cached_file in cached.values() {
|
||||
if !origin_paths.contains(&cached_file.real_path.path) {
|
||||
debug!(origin_id = %origin_id, path = ?cached_file.real_path.path, "File removed");
|
||||
changes.removed.push(cached_file.id);
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
origin_id = %origin_id,
|
||||
files_added = changes.added.len(),
|
||||
files_removed = changes.removed.len(),
|
||||
files_modified = changes.modified.len(),
|
||||
"Delta detection complete"
|
||||
);
|
||||
|
||||
Ok(changes)
|
||||
}
|
||||
|
||||
fn is_modified_scan(&self, cached: &FileMeta, scanned: &ScannedFile) -> bool {
|
||||
cached.size != scanned.size || cached.mtime != scanned.mtime
|
||||
}
|
||||
|
||||
async fn scan_origin(&self, origin: &dyn Origin) -> Result<Vec<ScannedFile>, DeltaError> {
|
||||
let mut files = Vec::new();
|
||||
let mut dirs_to_scan = vec![PathBuf::from("/")];
|
||||
|
||||
while let Some(dir) = dirs_to_scan.pop() {
|
||||
let entries = origin
|
||||
.readdir(&dir)
|
||||
.await
|
||||
.map_err(|e| DeltaError::OriginScan(e.to_string()))?;
|
||||
|
||||
for entry in entries {
|
||||
let entry_path = dir.join(&entry.name);
|
||||
|
||||
if entry.is_dir {
|
||||
dirs_to_scan.push(entry_path);
|
||||
} else if Self::is_audio_file(&entry.name) {
|
||||
let stat = origin
|
||||
.stat(&entry_path)
|
||||
.await
|
||||
.map_err(|e| DeltaError::OriginScan(e.to_string()))?;
|
||||
|
||||
files.push(ScannedFile {
|
||||
path: entry_path,
|
||||
origin_id: origin.id().clone(),
|
||||
size: stat.size,
|
||||
mtime: stat.mtime,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn is_audio_file(name: &str) -> bool {
|
||||
let lower = name.to_lowercase();
|
||||
lower.ends_with(".flac")
|
||||
|| lower.ends_with(".mp3")
|
||||
|| lower.ends_with(".ogg")
|
||||
|| lower.ends_with(".wav")
|
||||
|| lower.ends_with(".m4a")
|
||||
|| lower.ends_with(".aac")
|
||||
|| lower.ends_with(".opus")
|
||||
}
|
||||
|
||||
async fn compute_chunks_for_scan(
|
||||
&self,
|
||||
origin: &dyn Origin,
|
||||
scanned: &ScannedFile,
|
||||
) -> Result<Vec<ManifestChunk>, DeltaError> {
|
||||
let data = origin
|
||||
.read_full(&scanned.path)
|
||||
.await
|
||||
.map_err(|e| DeltaError::OriginRead(e.to_string()))?;
|
||||
|
||||
let chunks = self.chunker.chunk_refs(&data);
|
||||
|
||||
Ok(chunks
|
||||
.into_iter()
|
||||
.map(|c| ManifestChunk {
|
||||
hash: c.hash,
|
||||
offset: c.offset,
|
||||
size: c.length,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn compute_diff(
|
||||
&self,
|
||||
old_chunks: &[ManifestChunk],
|
||||
new_chunks: &[ManifestChunk],
|
||||
) -> ManifestDiff {
|
||||
let old_hashes: HashSet<_> = old_chunks.iter().map(|c| c.hash).collect();
|
||||
let new_hashes: HashSet<_> = new_chunks.iter().map(|c| c.hash).collect();
|
||||
|
||||
ManifestDiff {
|
||||
reuse: new_chunks
|
||||
.iter()
|
||||
.filter(|c| old_hashes.contains(&c.hash))
|
||||
.cloned()
|
||||
.collect(),
|
||||
fetch: new_chunks
|
||||
.iter()
|
||||
.filter(|c| !old_hashes.contains(&c.hash))
|
||||
.cloned()
|
||||
.collect(),
|
||||
orphaned: old_chunks
|
||||
.iter()
|
||||
.filter(|c| !new_hashes.contains(&c.hash))
|
||||
.map(|c| c.hash)
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DeltaDetector {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum DeltaError {
|
||||
#[error("Origin read error: {0}")]
|
||||
OriginRead(String),
|
||||
|
||||
#[error("Origin scan error: {0}")]
|
||||
OriginScan(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_core::{OriginId, RealPath, VirtualPath};
|
||||
use std::time::SystemTime;
|
||||
|
||||
fn make_file_meta(id: i64, path: &str, size: u64) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(format!("/test/{}", path)),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from(path),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_scanned_file(path: &str, size: u64) -> ScannedFile {
|
||||
ScannedFile {
|
||||
path: PathBuf::from(path),
|
||||
origin_id: OriginId::from("test"),
|
||||
size,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_modified_size_change() {
|
||||
let detector = DeltaDetector::new();
|
||||
|
||||
let cached = make_file_meta(1, "test.flac", 1000);
|
||||
let scanned = make_scanned_file("test.flac", 2000);
|
||||
|
||||
assert!(detector.is_modified_scan(&cached, &scanned));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_modified_same() {
|
||||
let detector = DeltaDetector::new();
|
||||
|
||||
let cached = make_file_meta(1, "test.flac", 1000);
|
||||
let scanned = make_scanned_file("test.flac", 1000);
|
||||
|
||||
assert!(!detector.is_modified_scan(&cached, &scanned));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_audio_file() {
|
||||
assert!(DeltaDetector::is_audio_file("track.flac"));
|
||||
assert!(DeltaDetector::is_audio_file("song.MP3"));
|
||||
assert!(DeltaDetector::is_audio_file("audio.ogg"));
|
||||
assert!(!DeltaDetector::is_audio_file("readme.txt"));
|
||||
assert!(!DeltaDetector::is_audio_file("cover.jpg"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_diff() {
|
||||
let detector = DeltaDetector::new();
|
||||
|
||||
let old_chunks = vec![
|
||||
ManifestChunk {
|
||||
hash: ChunkHash::from_bytes(b"A"),
|
||||
offset: 0,
|
||||
size: 256,
|
||||
},
|
||||
ManifestChunk {
|
||||
hash: ChunkHash::from_bytes(b"B"),
|
||||
offset: 256,
|
||||
size: 256,
|
||||
},
|
||||
ManifestChunk {
|
||||
hash: ChunkHash::from_bytes(b"C"),
|
||||
offset: 512,
|
||||
size: 256,
|
||||
},
|
||||
];
|
||||
|
||||
let new_chunks = vec![
|
||||
ManifestChunk {
|
||||
hash: ChunkHash::from_bytes(b"A"),
|
||||
offset: 0,
|
||||
size: 256,
|
||||
},
|
||||
ManifestChunk {
|
||||
hash: ChunkHash::from_bytes(b"D"),
|
||||
offset: 256,
|
||||
size: 256,
|
||||
},
|
||||
ManifestChunk {
|
||||
hash: ChunkHash::from_bytes(b"C"),
|
||||
offset: 512,
|
||||
size: 256,
|
||||
},
|
||||
];
|
||||
|
||||
let diff = detector.compute_diff(&old_chunks, &new_chunks);
|
||||
|
||||
assert_eq!(diff.reuse.len(), 2);
|
||||
assert_eq!(diff.fetch.len(), 1);
|
||||
assert_eq!(diff.orphaned.len(), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
pub mod cdc;
|
||||
pub mod delta;
|
||||
pub mod watcher;
|
||||
|
||||
pub use cdc::{CdcChunker, Chunk, ChunkRef};
|
||||
pub use delta::{ChangeSet, DeltaDetector, DeltaError, ManifestChunk, ManifestDiff};
|
||||
pub use watcher::{OriginWatcher, WatchError, WatchHandle};
|
||||
@@ -0,0 +1,218 @@
|
||||
use musicfs_core::{Event, EventBus, OriginId, VirtualPath};
|
||||
use notify::{Config, RecommendedWatcher, RecursiveMode, Watcher};
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{error, info, info_span, trace, Instrument};
|
||||
|
||||
const DEBOUNCE_MS: u64 = 200;
|
||||
|
||||
pub struct OriginWatcher {
|
||||
origin_id: OriginId,
|
||||
root: PathBuf,
|
||||
event_bus: Arc<EventBus>,
|
||||
}
|
||||
|
||||
impl OriginWatcher {
|
||||
pub fn new(origin_id: OriginId, root: PathBuf, event_bus: Arc<EventBus>) -> Self {
|
||||
Self {
|
||||
origin_id,
|
||||
root,
|
||||
event_bus,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(self) -> WatchHandle {
|
||||
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
|
||||
|
||||
let origin_id = self.origin_id.clone();
|
||||
let root = self.root.clone();
|
||||
let event_bus = self.event_bus.clone();
|
||||
|
||||
let origin_id_str = origin_id.to_string();
|
||||
tokio::spawn(
|
||||
async move {
|
||||
if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await
|
||||
{
|
||||
error!("Watcher error: {}", e);
|
||||
}
|
||||
}
|
||||
.instrument(info_span!("file_watcher", origin_id = %origin_id_str)),
|
||||
);
|
||||
|
||||
WatchHandle { stop_tx }
|
||||
}
|
||||
|
||||
async fn watch_loop(
|
||||
origin_id: &OriginId,
|
||||
root: &Path,
|
||||
event_bus: &EventBus,
|
||||
stop_rx: &mut mpsc::Receiver<()>,
|
||||
) -> Result<(), WatchError> {
|
||||
let (tx, mut rx) = mpsc::channel(100);
|
||||
|
||||
let mut watcher = RecommendedWatcher::new(
|
||||
move |res: Result<notify::Event, notify::Error>| {
|
||||
if let Ok(event) = res {
|
||||
let _ = tx.blocking_send(event);
|
||||
}
|
||||
},
|
||||
Config::default(),
|
||||
)
|
||||
.map_err(|e| WatchError::Init(e.to_string()))?;
|
||||
|
||||
watcher
|
||||
.watch(root, RecursiveMode::Recursive)
|
||||
.map_err(|e| WatchError::Watch(e.to_string()))?;
|
||||
|
||||
info!(origin_id = %origin_id, path = ?root, "Watcher started");
|
||||
|
||||
let mut debouncer: HashMap<PathBuf, Instant> = HashMap::new();
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(event) = rx.recv() => {
|
||||
Self::handle_notify_event(origin_id, root, event_bus, event, &mut debouncer);
|
||||
}
|
||||
_ = stop_rx.recv() => {
|
||||
info!(origin_id = %origin_id, "Watcher stopped");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle_notify_event(
|
||||
origin_id: &OriginId,
|
||||
root: &Path,
|
||||
event_bus: &EventBus,
|
||||
event: notify::Event,
|
||||
debouncer: &mut HashMap<PathBuf, Instant>,
|
||||
) {
|
||||
use notify::EventKind;
|
||||
|
||||
let now = Instant::now();
|
||||
|
||||
for path in event.paths {
|
||||
let relative = match path.strip_prefix(root) {
|
||||
Ok(p) => p.to_path_buf(),
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
if !Self::is_audio_file(&path) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(last_seen) = debouncer.get(&relative) {
|
||||
if now.duration_since(*last_seen).as_millis() < DEBOUNCE_MS as u128 {
|
||||
trace!(origin_id = %origin_id, path = ?relative, "Debouncing event");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
debouncer.insert(relative.clone(), now);
|
||||
|
||||
let vpath = VirtualPath::new(format!("/{}", relative.display()));
|
||||
|
||||
match event.kind {
|
||||
EventKind::Create(_) => {
|
||||
trace!(origin_id = %origin_id, path = ?relative, "File created");
|
||||
event_bus.publish(Event::FileAdded {
|
||||
path: vpath,
|
||||
origin_id: origin_id.clone(),
|
||||
});
|
||||
}
|
||||
EventKind::Remove(_) => {
|
||||
trace!(origin_id = %origin_id, path = ?relative, "File removed");
|
||||
event_bus.publish(Event::FileRemoved {
|
||||
path: vpath,
|
||||
file_id: None,
|
||||
});
|
||||
}
|
||||
EventKind::Modify(_) => {
|
||||
trace!(origin_id = %origin_id, path = ?relative, "File modified");
|
||||
event_bus.publish(Event::FileModified { path: vpath });
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_audio_file(path: &Path) -> bool {
|
||||
matches!(
|
||||
path.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.to_lowercase())
|
||||
.as_deref(),
|
||||
Some("flac" | "mp3" | "ogg" | "wav" | "m4a" | "aac" | "opus")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WatchHandle {
|
||||
stop_tx: mpsc::Sender<()>,
|
||||
}
|
||||
|
||||
impl WatchHandle {
|
||||
pub async fn stop(self) {
|
||||
let _ = self.stop_tx.send(()).await;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WatchHandle {
|
||||
fn drop(&mut self) {
|
||||
trace!("WatchHandle dropped");
|
||||
let _ = self.stop_tx.try_send(());
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum WatchError {
|
||||
#[error("Failed to initialize watcher: {0}")]
|
||||
Init(String),
|
||||
|
||||
#[error("Failed to watch path: {0}")]
|
||||
Watch(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_watcher_detects_create() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let event_bus = Arc::new(EventBus::default());
|
||||
let mut rx = event_bus.subscribe();
|
||||
|
||||
let watcher =
|
||||
OriginWatcher::new(OriginId::from("test"), dir.path().to_path_buf(), event_bus);
|
||||
let handle = watcher.start();
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
std::fs::write(dir.path().join("test.flac"), b"audio").unwrap();
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
let event = rx.try_recv();
|
||||
assert!(matches!(event, Ok(Event::FileAdded { .. })));
|
||||
|
||||
handle.stop().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_audio_file() {
|
||||
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.flac")));
|
||||
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.MP3")));
|
||||
assert!(!OriginWatcher::is_audio_file(Path::new("/music/cover.jpg")));
|
||||
assert!(!OriginWatcher::is_audio_file(Path::new(
|
||||
"/music/readme.txt"
|
||||
)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
[package]
|
||||
name = "musicfs-test-utils"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
description = "Test utilities and fixtures for MusicFS resilience testing"
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-origins = { path = "../musicfs-origins" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-cache = { path = "../musicfs-cache" }
|
||||
musicfs-search = { path = "../musicfs-search" }
|
||||
|
||||
async-trait.workspace = true
|
||||
tokio = { workspace = true, features = ["full", "sync", "time"] }
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
parking_lot.workspace = true
|
||||
tempfile.workspace = true
|
||||
bytes.workspace = true
|
||||
|
||||
# Fault injection
|
||||
fail = { version = "0.5", optional = true }
|
||||
rlimit = { version = "0.10", optional = true }
|
||||
nix = { version = "0.29", optional = true, features = ["signal", "process"] }
|
||||
|
||||
# Docker/network tests
|
||||
noxious-client = { version = "1.0", optional = true }
|
||||
reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls"] }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
failpoints = ["fail/failpoints"]
|
||||
process-tests = ["nix"]
|
||||
resource-limits = ["rlimit"]
|
||||
docker-tests = ["noxious-client", "reqwest"]
|
||||
full = ["failpoints", "process-tests", "resource-limits", "docker-tests"]
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = "0.4"
|
||||
tokio-util.workspace = true
|
||||
sd-notify.workspace = true
|
||||
libc.workspace = true
|
||||
@@ -0,0 +1,204 @@
|
||||
use musicfs_cas::CasError;
|
||||
use musicfs_core::Error;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub fn assert_error_contains<T, E: std::fmt::Debug>(result: Result<T, E>, expected_text: &str) {
|
||||
match result {
|
||||
Ok(_) => panic!("Expected error containing '{}', but got Ok", expected_text),
|
||||
Err(e) => {
|
||||
let error_msg = format!("{:?}", e);
|
||||
assert!(
|
||||
error_msg.contains(expected_text),
|
||||
"Expected error containing '{}', but got: {}",
|
||||
expected_text,
|
||||
error_msg
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_io_error<T>(result: Result<T, Error>) {
|
||||
match result {
|
||||
Err(Error::Io(_)) => (),
|
||||
Err(e) => panic!("Expected Io error, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected Io error, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_cas_io_error<T>(result: Result<T, CasError>) {
|
||||
match result {
|
||||
Err(CasError::Io(_)) => (),
|
||||
Err(e) => panic!("Expected CasError::Io, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected CasError::Io, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_cas_not_found<T>(result: Result<T, CasError>) {
|
||||
match result {
|
||||
Err(CasError::NotFound(_)) => (),
|
||||
Err(e) => panic!("Expected CasError::NotFound, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected CasError::NotFound, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_cas_integrity_error<T>(result: Result<T, CasError>) {
|
||||
match result {
|
||||
Err(CasError::IntegrityError { .. }) => (),
|
||||
Err(e) => panic!("Expected CasError::IntegrityError, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected CasError::IntegrityError, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_file_not_found<T>(result: Result<T, Error>) {
|
||||
match result {
|
||||
Err(Error::FileNotFound(_)) => (),
|
||||
Err(e) => panic!("Expected FileNotFound error, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected FileNotFound error, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_origin_error<T>(result: Result<T, Error>) {
|
||||
match result {
|
||||
Err(Error::Origin(_)) => (),
|
||||
Err(e) => panic!("Expected Origin error, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected Origin error, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_timeout_error<T>(result: Result<T, Error>) {
|
||||
match result {
|
||||
Err(Error::Timeout(_)) => (),
|
||||
Err(e) => panic!("Expected Timeout error, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected Timeout error, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TimedAssertion {
|
||||
start: Instant,
|
||||
min_duration: Option<Duration>,
|
||||
max_duration: Option<Duration>,
|
||||
}
|
||||
|
||||
impl TimedAssertion {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
start: Instant::now(),
|
||||
min_duration: None,
|
||||
max_duration: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn expect_at_least(mut self, duration: Duration) -> Self {
|
||||
self.min_duration = Some(duration);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn expect_at_most(mut self, duration: Duration) -> Self {
|
||||
self.max_duration = Some(duration);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn assert_elapsed(self) {
|
||||
let elapsed = self.start.elapsed();
|
||||
|
||||
if let Some(min) = self.min_duration {
|
||||
assert!(
|
||||
elapsed >= min,
|
||||
"Expected at least {:?}, but only {:?} elapsed",
|
||||
min,
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(max) = self.max_duration {
|
||||
assert!(
|
||||
elapsed <= max,
|
||||
"Expected at most {:?}, but {:?} elapsed",
|
||||
max,
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TimedAssertion {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn assert_completes_within<F, T>(future: F, timeout: Duration) -> T
|
||||
where
|
||||
F: std::future::Future<Output = T>,
|
||||
{
|
||||
tokio::time::timeout(timeout, future)
|
||||
.await
|
||||
.expect(&format!("Operation did not complete within {:?}", timeout))
|
||||
}
|
||||
|
||||
pub async fn assert_times_out<F, T>(future: F, timeout: Duration)
|
||||
where
|
||||
F: std::future::Future<Output = T>,
|
||||
{
|
||||
match tokio::time::timeout(timeout, future).await {
|
||||
Ok(_) => panic!("Expected operation to time out, but it completed"),
|
||||
Err(_) => (),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_assert_error_contains() {
|
||||
let result: Result<(), Error> = Err(Error::Origin("connection refused".into()));
|
||||
assert_error_contains(result, "connection");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "Expected error containing")]
|
||||
fn test_assert_error_contains_failure() {
|
||||
let result: Result<(), Error> = Err(Error::Origin("something else".into()));
|
||||
assert_error_contains(result, "connection");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_assert_io_error() {
|
||||
let result: Result<(), Error> = Err(Error::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"test",
|
||||
)));
|
||||
assert_io_error(result);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_timed_assertion_at_least() {
|
||||
let timer = TimedAssertion::new().expect_at_least(Duration::from_millis(10));
|
||||
std::thread::sleep(Duration::from_millis(15));
|
||||
timer.assert_elapsed();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_timed_assertion_at_most() {
|
||||
let timer = TimedAssertion::new().expect_at_most(Duration::from_millis(100));
|
||||
timer.assert_elapsed();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_assert_completes_within() {
|
||||
let result = assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
|
||||
assert_eq!(result, 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_assert_times_out() {
|
||||
assert_times_out(
|
||||
async {
|
||||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||
},
|
||||
Duration::from_millis(10),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
use bytes::Bytes;
|
||||
use musicfs_cas::{CasConfig, CasError, CasStore, DedupStats};
|
||||
use musicfs_core::ChunkHash;
|
||||
use std::io::{self, ErrorKind};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct FaultyCasStore {
|
||||
inner: Arc<CasStore>,
|
||||
inject_enospc: AtomicBool,
|
||||
inject_eio_on_read: AtomicBool,
|
||||
inject_eio_on_write: AtomicBool,
|
||||
inject_corruption: AtomicBool,
|
||||
fail_after_n_puts: AtomicUsize,
|
||||
put_count: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FaultyCasStore {
|
||||
pub fn new(inner: Arc<CasStore>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
inject_enospc: AtomicBool::new(false),
|
||||
inject_eio_on_read: AtomicBool::new(false),
|
||||
inject_eio_on_write: AtomicBool::new(false),
|
||||
inject_corruption: AtomicBool::new(false),
|
||||
fail_after_n_puts: AtomicUsize::new(usize::MAX),
|
||||
put_count: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn open(config: CasConfig) -> Result<Self, CasError> {
|
||||
let store = CasStore::open(config).await?;
|
||||
Ok(Self::new(Arc::new(store)))
|
||||
}
|
||||
|
||||
pub fn set_inject_enospc(&self, enabled: bool) {
|
||||
self.inject_enospc.store(enabled, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_inject_eio_on_read(&self, enabled: bool) {
|
||||
self.inject_eio_on_read.store(enabled, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_inject_eio_on_write(&self, enabled: bool) {
|
||||
self.inject_eio_on_write.store(enabled, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_inject_corruption(&self, enabled: bool) {
|
||||
self.inject_corruption.store(enabled, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_fail_after_n_puts(&self, n: usize) {
|
||||
self.fail_after_n_puts.store(n, Ordering::SeqCst);
|
||||
self.put_count.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn reset_faults(&self) {
|
||||
self.inject_enospc.store(false, Ordering::SeqCst);
|
||||
self.inject_eio_on_read.store(false, Ordering::SeqCst);
|
||||
self.inject_eio_on_write.store(false, Ordering::SeqCst);
|
||||
self.inject_corruption.store(false, Ordering::SeqCst);
|
||||
self.fail_after_n_puts.store(usize::MAX, Ordering::SeqCst);
|
||||
self.put_count.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn put_count(&self) -> usize {
|
||||
self.put_count.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
|
||||
let count = self.put_count.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
if self.inject_enospc.load(Ordering::SeqCst) {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"No space left on device (ENOSPC injected)",
|
||||
)));
|
||||
}
|
||||
|
||||
if self.inject_eio_on_write.load(Ordering::SeqCst) {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"Input/output error (EIO injected)",
|
||||
)));
|
||||
}
|
||||
|
||||
let threshold = self.fail_after_n_puts.load(Ordering::SeqCst);
|
||||
if count >= threshold {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"Injected failure after N puts",
|
||||
)));
|
||||
}
|
||||
|
||||
self.inner.put(data).await
|
||||
}
|
||||
|
||||
pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
|
||||
if self.inject_eio_on_read.load(Ordering::SeqCst) {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"Input/output error (EIO injected)",
|
||||
)));
|
||||
}
|
||||
|
||||
let data = self.inner.get(hash).await?;
|
||||
|
||||
if self.inject_corruption.load(Ordering::SeqCst) {
|
||||
let mut corrupted = data.to_vec();
|
||||
if !corrupted.is_empty() {
|
||||
corrupted[0] = corrupted[0].wrapping_add(1);
|
||||
}
|
||||
return Err(CasError::IntegrityError {
|
||||
expected: hash.as_hex(),
|
||||
actual: ChunkHash::from_bytes(&corrupted).as_hex(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub fn exists(&self, hash: &ChunkHash) -> bool {
|
||||
self.inner.exists(hash)
|
||||
}
|
||||
|
||||
pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
|
||||
if self.inject_eio_on_write.load(Ordering::SeqCst) {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"Input/output error (EIO injected)",
|
||||
)));
|
||||
}
|
||||
self.inner.delete(hash).await
|
||||
}
|
||||
|
||||
pub fn current_size(&self) -> u64 {
|
||||
self.inner.current_size()
|
||||
}
|
||||
|
||||
pub fn max_size(&self) -> u64 {
|
||||
self.inner.max_size()
|
||||
}
|
||||
|
||||
pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
|
||||
self.inner.list_chunks()
|
||||
}
|
||||
|
||||
pub fn dedup_stats(&self) -> DedupStats {
|
||||
self.inner.dedup_stats()
|
||||
}
|
||||
|
||||
pub fn inner(&self) -> &Arc<CasStore> {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
async fn test_store() -> (FaultyCasStore, TempDir) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 1024 * 1024,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = FaultyCasStore::open(config).await.unwrap();
|
||||
(store, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_healthy_passthrough() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"test data";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
let retrieved = store.get(&hash).await.unwrap();
|
||||
assert_eq!(&retrieved[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_inject_enospc() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
store.set_inject_enospc(true);
|
||||
let result = store.put(b"test").await;
|
||||
assert!(result.is_err());
|
||||
|
||||
let err = result.unwrap_err();
|
||||
assert!(matches!(err, CasError::Io(_)));
|
||||
|
||||
store.set_inject_enospc(false);
|
||||
assert!(store.put(b"test").await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_inject_eio_on_read() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let hash = store.put(b"test").await.unwrap();
|
||||
|
||||
store.set_inject_eio_on_read(true);
|
||||
let result = store.get(&hash).await;
|
||||
assert!(result.is_err());
|
||||
|
||||
store.set_inject_eio_on_read(false);
|
||||
assert!(store.get(&hash).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_inject_corruption() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let hash = store.put(b"test data").await.unwrap();
|
||||
|
||||
store.set_inject_corruption(true);
|
||||
let result = store.get(&hash).await;
|
||||
assert!(matches!(result, Err(CasError::IntegrityError { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fail_after_n_puts() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
store.set_fail_after_n_puts(2);
|
||||
|
||||
assert!(store.put(b"data1").await.is_ok());
|
||||
assert!(store.put(b"data2").await.is_ok());
|
||||
assert!(store.put(b"data3").await.is_err());
|
||||
assert!(store.put(b"data4").await.is_err());
|
||||
assert_eq!(store.put_count(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_reset_faults() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
store.set_inject_enospc(true);
|
||||
store.set_inject_eio_on_read(true);
|
||||
store.set_fail_after_n_puts(1);
|
||||
|
||||
store.reset_faults();
|
||||
|
||||
assert!(store.put(b"test").await.is_ok());
|
||||
let hash = store.put(b"test2").await.unwrap();
|
||||
assert!(store.get(&hash).await.is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, Error, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use musicfs_origins::{Origin, WatchCallback, WatchHandle};
|
||||
use parking_lot::RwLock;
|
||||
use std::io::{self, ErrorKind};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum FailMode {
|
||||
Healthy,
|
||||
FailEveryNth(usize),
|
||||
FailAfterN(usize),
|
||||
TimeoutMs(u64),
|
||||
PartialRead { max_bytes: usize },
|
||||
ReturnError(ErrorKind),
|
||||
}
|
||||
|
||||
impl Default for FailMode {
|
||||
fn default() -> Self {
|
||||
FailMode::Healthy
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FaultyOrigin {
|
||||
inner: Arc<dyn Origin>,
|
||||
fail_mode: Arc<RwLock<FailMode>>,
|
||||
call_count: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FaultyOrigin {
|
||||
pub fn new(inner: Arc<dyn Origin>, mode: FailMode) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
fail_mode: Arc::new(RwLock::new(mode)),
|
||||
call_count: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn wrap(inner: impl Origin + 'static) -> Self {
|
||||
Self::new(Arc::new(inner), FailMode::Healthy)
|
||||
}
|
||||
|
||||
pub fn set_mode(&self, mode: FailMode) {
|
||||
*self.fail_mode.write() = mode;
|
||||
}
|
||||
|
||||
pub fn call_count(&self) -> usize {
|
||||
self.call_count.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn reset_count(&self) {
|
||||
self.call_count.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn increment_and_check(&self) -> Option<Error> {
|
||||
let count = self.call_count.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
let mode = self.fail_mode.read();
|
||||
|
||||
match *mode {
|
||||
FailMode::Healthy => None,
|
||||
FailMode::FailEveryNth(n) if n > 0 && count % n == 0 => {
|
||||
Some(Error::Origin("Injected failure (every Nth)".into()))
|
||||
}
|
||||
FailMode::FailEveryNth(_) => None,
|
||||
FailMode::FailAfterN(n) if count > n => {
|
||||
Some(Error::Origin("Injected failure (after N)".into()))
|
||||
}
|
||||
FailMode::FailAfterN(_) => None,
|
||||
FailMode::TimeoutMs(_) => None,
|
||||
FailMode::PartialRead { .. } => None,
|
||||
FailMode::ReturnError(kind) => {
|
||||
Some(Error::Io(io::Error::new(kind, "Injected I/O error")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn maybe_timeout(&self) -> Option<Error> {
|
||||
let mode = self.fail_mode.read().clone();
|
||||
if let FailMode::TimeoutMs(ms) = mode {
|
||||
tokio::time::sleep(Duration::from_millis(ms)).await;
|
||||
Some(Error::Timeout("Injected timeout".into()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_if_partial(&self, mut data: Vec<u8>) -> Vec<u8> {
|
||||
let mode = self.fail_mode.read();
|
||||
if let FailMode::PartialRead { max_bytes } = *mode {
|
||||
data.truncate(max_bytes);
|
||||
}
|
||||
data
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for FaultyOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
self.inner.id()
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
self.inner.origin_type()
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
self.inner.display_name()
|
||||
}
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
self.inner.readdir(path).await
|
||||
}
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
self.inner.stat(path).await
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
let data = self.inner.read(path, offset, size).await?;
|
||||
Ok(self.truncate_if_partial(data))
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
let data = self.inner.read_full(path).await?;
|
||||
Ok(self.truncate_if_partial(data))
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
self.inner.exists(path).await
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
let mode = self.fail_mode.read().clone();
|
||||
match mode {
|
||||
FailMode::Healthy => self.inner.health().await,
|
||||
FailMode::ReturnError(_) => HealthStatus::Unhealthy,
|
||||
FailMode::TimeoutMs(ms) => {
|
||||
tokio::time::sleep(Duration::from_millis(ms)).await;
|
||||
HealthStatus::Unhealthy
|
||||
}
|
||||
FailMode::FailAfterN(n) if self.call_count.load(Ordering::SeqCst) >= n => {
|
||||
HealthStatus::Unhealthy
|
||||
}
|
||||
_ => self.inner.health().await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
self.inner.open_read(path).await
|
||||
}
|
||||
|
||||
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
|
||||
self.inner.watch(path, callback).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::SystemTime;
|
||||
|
||||
struct MockOrigin {
|
||||
id: OriginId,
|
||||
}
|
||||
|
||||
impl MockOrigin {
|
||||
fn new(id: &str) -> Self {
|
||||
Self {
|
||||
id: OriginId::from(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for MockOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
&self.id
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
OriginType::Local
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
"mock"
|
||||
}
|
||||
|
||||
async fn readdir(&self, _path: &Path) -> Result<Vec<DirEntry>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
async fn stat(&self, _path: &Path) -> Result<FileStat> {
|
||||
Ok(FileStat {
|
||||
size: 1000,
|
||||
mtime: SystemTime::now(),
|
||||
is_dir: false,
|
||||
})
|
||||
}
|
||||
|
||||
async fn read(&self, _path: &Path, _offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
Ok(vec![0u8; size as usize])
|
||||
}
|
||||
|
||||
async fn read_full(&self, _path: &Path) -> Result<Vec<u8>> {
|
||||
Ok(vec![0u8; 100])
|
||||
}
|
||||
|
||||
async fn exists(&self, _path: &Path) -> Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
HealthStatus::Healthy
|
||||
}
|
||||
|
||||
async fn open_read(&self, _path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
|
||||
Err(Error::Origin("Not implemented".into()))
|
||||
}
|
||||
|
||||
async fn watch(&self, _path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
|
||||
Err(Error::Origin("Not implemented".into()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_healthy_passthrough() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
|
||||
|
||||
let result = faulty.stat(Path::new("/test")).await;
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(faulty.call_count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fail_every_nth() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::FailEveryNth(2));
|
||||
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
assert_eq!(faulty.call_count(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fail_after_n() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::FailAfterN(2));
|
||||
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_partial_read() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::PartialRead { max_bytes: 10 });
|
||||
|
||||
let data = faulty.read(Path::new("/test"), 0, 100).await.unwrap();
|
||||
assert_eq!(data.len(), 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mode_change_mid_test() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::ReturnError(ErrorKind::ConnectionRefused));
|
||||
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
|
||||
faulty.set_mode(FailMode::Healthy);
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_health_reflects_mode() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
|
||||
|
||||
assert_eq!(faulty.health().await, HealthStatus::Healthy);
|
||||
|
||||
faulty.set_mode(FailMode::ReturnError(ErrorKind::ConnectionRefused));
|
||||
assert_eq!(faulty.health().await, HealthStatus::Unhealthy);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_cas::{CasConfig, CasStore};
|
||||
use musicfs_core::{AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::SystemTime;
|
||||
use tempfile::TempDir;
|
||||
|
||||
pub fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from(vpath),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_file_meta_with_origin(id: i64, vpath: &str, size: u64, origin_id: &str) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from(origin_id),
|
||||
path: PathBuf::from(vpath),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_audio_meta(artist: &str, album: &str, title: &str) -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: Some(title.to_string()),
|
||||
artist: Some(artist.to_string()),
|
||||
album: Some(album.to_string()),
|
||||
album_artist: None,
|
||||
genre: None,
|
||||
year: None,
|
||||
track: None,
|
||||
disc: None,
|
||||
duration_ms: Some(180_000),
|
||||
bitrate: Some(320),
|
||||
sample_rate: Some(44100),
|
||||
format: AudioFormat::Flac,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_audio_file(
|
||||
id: i64,
|
||||
vpath: &str,
|
||||
size: u64,
|
||||
artist: &str,
|
||||
album: &str,
|
||||
title: &str,
|
||||
) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from(vpath),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: Some(make_audio_meta(artist, album, title)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_audio_file_full(
|
||||
id: i64,
|
||||
vpath: &str,
|
||||
size: u64,
|
||||
artist: &str,
|
||||
album: &str,
|
||||
title: &str,
|
||||
track: u32,
|
||||
year: u32,
|
||||
) -> FileMeta {
|
||||
let mut audio = make_audio_meta(artist, album, title);
|
||||
audio.track = Some(track);
|
||||
audio.year = Some(year);
|
||||
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from(vpath),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: Some(audio),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TestCasStore {
|
||||
pub store: Arc<CasStore>,
|
||||
pub dir: TempDir,
|
||||
}
|
||||
|
||||
pub async fn setup_test_cas() -> TestCasStore {
|
||||
let dir = TempDir::new().expect("Failed to create temp dir for CAS");
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 100 * 1024 * 1024,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = CasStore::open(config)
|
||||
.await
|
||||
.expect("Failed to open CAS store");
|
||||
TestCasStore {
|
||||
store: Arc::new(store),
|
||||
dir,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn setup_test_cas_with_size(max_size: u64) -> TestCasStore {
|
||||
let dir = TempDir::new().expect("Failed to create temp dir for CAS");
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = CasStore::open(config)
|
||||
.await
|
||||
.expect("Failed to open CAS store");
|
||||
TestCasStore {
|
||||
store: Arc::new(store),
|
||||
dir,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn setup_test_tree(files: &[FileMeta]) -> Arc<RwLock<musicfs_cache::VirtualTree>> {
|
||||
let mut builder = TreeBuilder::new();
|
||||
for file in files {
|
||||
builder.add_file(file);
|
||||
}
|
||||
Arc::new(RwLock::new(builder.build()))
|
||||
}
|
||||
|
||||
pub fn create_test_file(dir: &Path, relative_path: &str, content: &[u8]) -> PathBuf {
|
||||
let full_path = dir.join(relative_path);
|
||||
if let Some(parent) = full_path.parent() {
|
||||
std::fs::create_dir_all(parent).expect("Failed to create parent directories");
|
||||
}
|
||||
std::fs::write(&full_path, content).expect("Failed to write test file");
|
||||
full_path
|
||||
}
|
||||
|
||||
pub fn create_test_dir_structure(base: &Path, structure: &[&str]) {
|
||||
for path in structure {
|
||||
let full_path = base.join(path);
|
||||
if path.ends_with('/') {
|
||||
std::fs::create_dir_all(&full_path).expect("Failed to create directory");
|
||||
} else {
|
||||
if let Some(parent) = full_path.parent() {
|
||||
std::fs::create_dir_all(parent).expect("Failed to create parent");
|
||||
}
|
||||
std::fs::write(&full_path, format!("content of {}", path))
|
||||
.expect("Failed to write file");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TestOriginDir {
|
||||
pub dir: TempDir,
|
||||
}
|
||||
|
||||
impl TestOriginDir {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
dir: TempDir::new().expect("Failed to create origin temp dir"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_file(&self, path: &str, content: &[u8]) -> PathBuf {
|
||||
create_test_file(self.dir.path(), path, content)
|
||||
}
|
||||
|
||||
pub fn add_audio_file(&self, path: &str) -> PathBuf {
|
||||
let fake_audio = b"FAKE_FLAC_HEADER_FOR_TESTING_ONLY";
|
||||
self.add_file(path, fake_audio)
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
self.dir.path()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TestOriginDir {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_make_file_meta() {
|
||||
let meta = make_file_meta(1, "/Artist/Album/Track.flac", 1000);
|
||||
assert_eq!(meta.id.0, 1);
|
||||
assert_eq!(meta.virtual_path.as_str(), "/Artist/Album/Track.flac");
|
||||
assert_eq!(meta.size, 1000);
|
||||
assert!(meta.audio.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_make_audio_file() {
|
||||
let meta = make_audio_file(1, "/path.flac", 5000, "Artist", "Album", "Title");
|
||||
assert!(meta.audio.is_some());
|
||||
let audio = meta.audio.unwrap();
|
||||
assert_eq!(audio.artist, Some("Artist".to_string()));
|
||||
assert_eq!(audio.album, Some("Album".to_string()));
|
||||
assert_eq!(audio.title, Some("Title".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_setup_test_cas() {
|
||||
let test_cas = setup_test_cas().await;
|
||||
let hash = test_cas.store.put(b"test data").await.unwrap();
|
||||
assert!(test_cas.store.exists(&hash));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_setup_test_tree() {
|
||||
let files = vec![
|
||||
make_file_meta(1, "/A/B/1.flac", 100),
|
||||
make_file_meta(2, "/A/B/2.flac", 200),
|
||||
];
|
||||
let tree = setup_test_tree(&files);
|
||||
let guard = tree.read().unwrap();
|
||||
assert!(guard.file_count() > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_origin_dir() {
|
||||
let origin = TestOriginDir::new();
|
||||
let path = origin.add_file("artist/album/track.flac", b"content");
|
||||
assert!(path.exists());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
pub mod assertions;
|
||||
pub mod faulty_cas;
|
||||
pub mod faulty_origin;
|
||||
pub mod fixtures;
|
||||
|
||||
pub use assertions::*;
|
||||
pub use faulty_cas::FaultyCasStore;
|
||||
pub use faulty_origin::{FailMode, FaultyOrigin};
|
||||
pub use fixtures::*;
|
||||
@@ -0,0 +1,141 @@
|
||||
#![cfg(feature = "docker-tests")]
|
||||
|
||||
use musicfs_core::{OriginId, OriginType};
|
||||
use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
|
||||
use noxious_client::{Client, StreamDirection, Toxic, ToxicKind};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
const TOXIPROXY_API: &str = "http://localhost:8474";
|
||||
const TOXIPROXY_LISTEN: &str = "localhost:18080";
|
||||
const UPSTREAM_ADDR: &str = "minio:9000";
|
||||
|
||||
async fn require_toxiproxy() {
|
||||
let available = match reqwest::get(format!("{}/version", TOXIPROXY_API)).await {
|
||||
Ok(resp) => resp.status().is_success(),
|
||||
Err(_) => false,
|
||||
};
|
||||
assert!(
|
||||
available,
|
||||
"Toxiproxy not available at {}. Run: cd tests/integration && docker-compose up -d",
|
||||
TOXIPROXY_API
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
|
||||
async fn test_toxiproxy_latency_injection() {
|
||||
require_toxiproxy().await;
|
||||
|
||||
let client = Client::new(TOXIPROXY_API);
|
||||
let proxy = client
|
||||
.create_proxy("minio_latency", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
|
||||
.await
|
||||
.expect("Failed to create proxy");
|
||||
|
||||
let toxic = Toxic {
|
||||
name: "latency_downstream".to_string(),
|
||||
kind: ToxicKind::Latency {
|
||||
latency: 500,
|
||||
jitter: 100,
|
||||
},
|
||||
direction: StreamDirection::Downstream,
|
||||
toxicity: 1.0,
|
||||
};
|
||||
|
||||
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(400),
|
||||
"Latency should be injected, got {:?}",
|
||||
elapsed
|
||||
);
|
||||
|
||||
proxy.delete().await.expect("Failed to cleanup proxy");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
|
||||
async fn test_toxiproxy_timeout_simulates_network_partition() {
|
||||
require_toxiproxy().await;
|
||||
|
||||
let client = Client::new(TOXIPROXY_API);
|
||||
let proxy = client
|
||||
.create_proxy("minio_partition", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
|
||||
.await
|
||||
.expect("Failed to create proxy");
|
||||
|
||||
let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
|
||||
assert!(result.is_ok(), "Should reach MinIO through proxy initially");
|
||||
|
||||
let toxic = Toxic {
|
||||
name: "timeout".to_string(),
|
||||
kind: ToxicKind::Timeout { timeout: 0 },
|
||||
direction: StreamDirection::Downstream,
|
||||
toxicity: 1.0,
|
||||
};
|
||||
|
||||
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
|
||||
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
result.is_err() || result.unwrap().is_err(),
|
||||
"Should timeout during partition"
|
||||
);
|
||||
|
||||
proxy
|
||||
.remove_toxic("timeout")
|
||||
.await
|
||||
.expect("Failed to remove toxic");
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let result = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
|
||||
assert!(result.is_ok(), "Should reach MinIO after partition heals");
|
||||
|
||||
proxy.delete().await.expect("Failed to cleanup proxy");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "Requires docker-compose up -d (tests/integration/docker-compose.yml)"]
|
||||
async fn test_toxiproxy_slow_close_throttles_responses() {
|
||||
require_toxiproxy().await;
|
||||
|
||||
let client = Client::new(TOXIPROXY_API);
|
||||
let proxy = client
|
||||
.create_proxy("minio_slow", TOXIPROXY_LISTEN, UPSTREAM_ADDR)
|
||||
.await
|
||||
.expect("Failed to create proxy");
|
||||
|
||||
let toxic = Toxic {
|
||||
name: "slow_close".to_string(),
|
||||
kind: ToxicKind::SlowClose { delay: 1000 },
|
||||
direction: StreamDirection::Downstream,
|
||||
toxicity: 1.0,
|
||||
};
|
||||
|
||||
proxy.add_toxic(&toxic).await.expect("Failed to add toxic");
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let _ = reqwest::get(format!("http://{}/minio/health/live", TOXIPROXY_LISTEN)).await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(800),
|
||||
"Slow close should delay response, got {:?}",
|
||||
elapsed
|
||||
);
|
||||
|
||||
proxy.delete().await.expect("Failed to cleanup proxy");
|
||||
}
|
||||
@@ -0,0 +1,838 @@
|
||||
use musicfs_cache::{Database, VirtualTree, ROOT_INODE};
|
||||
use musicfs_cas::{CasConfig, CasStore};
|
||||
use musicfs_core::supervisor::{TaskStatus, TaskSupervisor};
|
||||
use musicfs_core::{
|
||||
AudioMeta, FileId, FileMeta, HealthStatus, OriginId, OriginType, RealPath, VirtualPath,
|
||||
};
|
||||
use musicfs_origins::{HealthMonitor, LocalOrigin, OriginRegistry};
|
||||
use musicfs_search::SearchIndex;
|
||||
use musicfs_test_utils::{FailMode, FaultyOrigin};
|
||||
use std::collections::HashMap;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, UNIX_EPOCH};
|
||||
use tempfile::TempDir;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
fn setup_test_file(dir: &TempDir, name: &str, content: &[u8]) -> PathBuf {
|
||||
let path = dir.path().join(name);
|
||||
std::fs::write(&path, content).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
async fn setup_cas(dir: &Path) -> CasStore {
|
||||
CasStore::open(CasConfig {
|
||||
chunks_dir: dir.join("chunks"),
|
||||
max_size: 100 * 1024 * 1024,
|
||||
shard_levels: 2,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn create_faulty_origin(id: &str, dir: &TempDir, mode: FailMode) -> Arc<FaultyOrigin> {
|
||||
let inner = Arc::new(LocalOrigin::new(
|
||||
OriginId::from(id),
|
||||
dir.path().to_path_buf(),
|
||||
));
|
||||
Arc::new(FaultyOrigin::new(inner, mode))
|
||||
}
|
||||
|
||||
fn make_file_meta(id: i64, path: &str, size: u64) -> FileMeta {
|
||||
let name = Path::new(path)
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(path),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from(path),
|
||||
},
|
||||
size,
|
||||
mtime: UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: Some(AudioMeta {
|
||||
title: Some(name),
|
||||
..Default::default()
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_sqlite_integrity_check_detects_corruption() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("test.db");
|
||||
|
||||
{
|
||||
let db = Database::open(&db_path).unwrap();
|
||||
db.upsert_file(
|
||||
&OriginId::from("test"),
|
||||
Path::new("/test.flac"),
|
||||
&VirtualPath::new("/Test.flac"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
1000,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut data = std::fs::read(&db_path).unwrap();
|
||||
let mid = data.len() / 2;
|
||||
data[mid..mid + 100].fill(0xFF);
|
||||
std::fs::write(&db_path, &data).unwrap();
|
||||
|
||||
let result = Database::open_with_integrity_check(&db_path);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tantivy_corruption_triggers_rebuild() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index_path = dir.path().join("search_idx");
|
||||
|
||||
{
|
||||
let index = SearchIndex::open(&index_path).unwrap();
|
||||
index
|
||||
.index_file(&make_file_meta(1, "/a.flac", 1000))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
}
|
||||
|
||||
std::fs::write(index_path.join("meta.json"), b"corrupted").unwrap();
|
||||
|
||||
let index = SearchIndex::open_with_recovery(&index_path).unwrap();
|
||||
let results = index.search("a", 10).unwrap();
|
||||
assert_eq!(results.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_sled_corruption_triggers_repair() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let chunks_dir = dir.path().join("chunks");
|
||||
let config = CasConfig {
|
||||
chunks_dir: chunks_dir.clone(),
|
||||
max_size: 10_000_000,
|
||||
shard_levels: 2,
|
||||
};
|
||||
|
||||
{
|
||||
let store = CasStore::open(config.clone()).await.unwrap();
|
||||
store.put(b"test data").await.unwrap();
|
||||
}
|
||||
|
||||
let sled_dir = chunks_dir.join("index.sled");
|
||||
if sled_dir.exists() {
|
||||
for entry in std::fs::read_dir(&sled_dir).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
if entry.metadata().unwrap().is_file() {
|
||||
std::fs::write(entry.path(), b"corrupted").unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let result = CasStore::open(config).await;
|
||||
assert!(result.is_ok(), "sled should recover from corruption");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_put_handles_enospc() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let store = CasStore::open(CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 100,
|
||||
shard_levels: 2,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let large_data = vec![0u8; 1000];
|
||||
let result = store.put(&large_data).await;
|
||||
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Issue 2.8: CasStore should pre-check space and reject oversized write"
|
||||
);
|
||||
}
|
||||
|
||||
/// Demonstrates the PROBLEM with std::sync::RwLock: after a writer panic,
|
||||
/// the lock is poisoned and all subsequent access fails with PoisonError.
|
||||
/// This is why we use parking_lot::RwLock instead (see test_parking_lot_rwlock_survives_panic).
|
||||
#[test]
|
||||
fn test_poisoned_tree_lock_returns_eio_not_panic() {
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::thread;
|
||||
|
||||
let lock = Arc::new(RwLock::new(42));
|
||||
let lock_clone = lock.clone();
|
||||
|
||||
let handle = thread::spawn(move || {
|
||||
let _guard = lock_clone.write().unwrap();
|
||||
panic!("writer panic");
|
||||
});
|
||||
|
||||
let _ = handle.join();
|
||||
|
||||
let result = lock.read();
|
||||
// std::sync::RwLock poisons after writer panic - this is the problem we fix with parking_lot
|
||||
assert!(result.is_err(), "Issue 2.9: std::sync::RwLock should poison after writer panic (this demonstrates the problem)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parking_lot_rwlock_survives_panic() {
|
||||
use parking_lot::RwLock;
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
let tree = Arc::new(RwLock::new(VirtualTree::new()));
|
||||
let tree_clone = tree.clone();
|
||||
|
||||
let handle = thread::spawn(move || {
|
||||
let _guard = tree_clone.write();
|
||||
panic!("writer panic");
|
||||
});
|
||||
|
||||
let _ = handle.join();
|
||||
|
||||
assert!(
|
||||
tree.read().get(ROOT_INODE).is_some(),
|
||||
"parking_lot RwLock should survive writer panic"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_failover_on_primary_death() {
|
||||
let primary_dir = TempDir::new().unwrap();
|
||||
let backup_dir = TempDir::new().unwrap();
|
||||
setup_test_file(&primary_dir, "test.txt", b"primary");
|
||||
setup_test_file(&backup_dir, "test.txt", b"backup");
|
||||
|
||||
let primary = create_faulty_origin(
|
||||
"primary",
|
||||
&primary_dir,
|
||||
FailMode::ReturnError(ErrorKind::ConnectionRefused),
|
||||
);
|
||||
let backup = create_faulty_origin("backup", &backup_dir, FailMode::Healthy);
|
||||
|
||||
let mut thresholds = HashMap::new();
|
||||
thresholds.insert(OriginType::Local, 1);
|
||||
let monitor =
|
||||
Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
|
||||
let registry = Arc::new(OriginRegistry::new(monitor.clone()));
|
||||
|
||||
registry.register(primary.clone(), 1);
|
||||
registry.register(backup.clone(), 2);
|
||||
|
||||
monitor.check_now(&OriginId::from("primary")).await;
|
||||
monitor.check_now(&OriginId::from("backup")).await;
|
||||
|
||||
assert!(registry.health().is_unhealthy(&OriginId::from("primary")));
|
||||
assert!(registry.health().is_healthy(&OriginId::from("backup")));
|
||||
|
||||
let path = RealPath {
|
||||
origin_id: OriginId::from("backup"),
|
||||
path: PathBuf::from("/test.txt"),
|
||||
};
|
||||
let candidates = registry.route_all(&path);
|
||||
assert_eq!(candidates.len(), 1);
|
||||
assert_eq!(candidates[0].id(), &OriginId::from("backup"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_origin_recovery_resumes_routing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_test_file(&dir, "test.txt", b"content");
|
||||
|
||||
let faulty = create_faulty_origin(
|
||||
"recovering",
|
||||
&dir,
|
||||
FailMode::ReturnError(ErrorKind::ConnectionRefused),
|
||||
);
|
||||
|
||||
let mut thresholds = HashMap::new();
|
||||
thresholds.insert(OriginType::Local, 1);
|
||||
let monitor =
|
||||
Arc::new(HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds));
|
||||
monitor.add_origin(faulty.clone());
|
||||
|
||||
monitor.check_now(&OriginId::from("recovering")).await;
|
||||
assert_eq!(
|
||||
monitor
|
||||
.get_state(&OriginId::from("recovering"))
|
||||
.unwrap()
|
||||
.status,
|
||||
HealthStatus::Unhealthy
|
||||
);
|
||||
|
||||
faulty.set_mode(FailMode::Healthy);
|
||||
monitor.check_now(&OriginId::from("recovering")).await;
|
||||
|
||||
assert_eq!(
|
||||
monitor
|
||||
.get_state(&OriginId::from("recovering"))
|
||||
.unwrap()
|
||||
.status,
|
||||
HealthStatus::Healthy
|
||||
);
|
||||
assert_eq!(
|
||||
monitor
|
||||
.get_state(&OriginId::from("recovering"))
|
||||
.unwrap()
|
||||
.consecutive_failures,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_health_check_has_timeout() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_test_file(&dir, "test.txt", b"content");
|
||||
|
||||
let slow = create_faulty_origin("slow", &dir, FailMode::TimeoutMs(5_000));
|
||||
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
monitor.add_origin(slow.clone());
|
||||
|
||||
let start = Instant::now();
|
||||
monitor.check_now(&OriginId::from("slow")).await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"Issue 4.2.1: Health check should timeout in <2s, took {:?}",
|
||||
elapsed
|
||||
);
|
||||
|
||||
let state = monitor.get_state(&OriginId::from("slow")).unwrap();
|
||||
assert_eq!(state.status, HealthStatus::Unhealthy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_health_checks_run_in_parallel() {
|
||||
let slow1_dir = TempDir::new().unwrap();
|
||||
let slow2_dir = TempDir::new().unwrap();
|
||||
let slow3_dir = TempDir::new().unwrap();
|
||||
|
||||
let slow1 = create_faulty_origin("slow1", &slow1_dir, FailMode::TimeoutMs(200));
|
||||
let slow2 = create_faulty_origin("slow2", &slow2_dir, FailMode::TimeoutMs(200));
|
||||
let slow3 = create_faulty_origin("slow3", &slow3_dir, FailMode::TimeoutMs(200));
|
||||
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
monitor.add_origin(slow1);
|
||||
monitor.add_origin(slow2);
|
||||
monitor.add_origin(slow3);
|
||||
|
||||
let start = Instant::now();
|
||||
monitor.check_all().await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
assert!(
|
||||
elapsed < Duration::from_millis(350),
|
||||
"Issue 4.2.2: check_all() should run in parallel (sequential would take ~600ms), took {:?}",
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tantivy_survives_uncommitted_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index_path = dir.path().join("search_idx");
|
||||
|
||||
{
|
||||
let index = SearchIndex::open(&index_path).unwrap();
|
||||
index
|
||||
.index_file(&make_file_meta(1, "/a.flac", 1000))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
index
|
||||
.index_file(&make_file_meta(2, "/b.flac", 1000))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let index = SearchIndex::open(&index_path).unwrap();
|
||||
let results = index.search("a", 10).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[cfg(feature = "resource-limits")]
|
||||
async fn test_fd_exhaustion_handling() {
|
||||
use rlimit::{getrlimit, setrlimit, Resource};
|
||||
|
||||
let (orig_soft, orig_hard) = getrlimit(Resource::NOFILE).unwrap();
|
||||
|
||||
setrlimit(Resource::NOFILE, 64, 64).unwrap();
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let result = CasStore::open(CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 1_000_000,
|
||||
shard_levels: 2,
|
||||
})
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(_store) => {}
|
||||
Err(e) => {
|
||||
let msg = format!("{}", e);
|
||||
assert!(!msg.contains("panic"), "Should not panic on fd exhaustion");
|
||||
}
|
||||
}
|
||||
|
||||
setrlimit(Resource::NOFILE, orig_soft, orig_hard).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[cfg(not(feature = "resource-limits"))]
|
||||
async fn test_fd_exhaustion_handling() {
|
||||
eprintln!("Skipping test_fd_exhaustion_handling: resource-limits feature not enabled");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_corrupt_chunk_auto_refetched() {
|
||||
use musicfs_cas::{ContentFetcher, FileReader};
|
||||
use musicfs_origins::LocalOrigin;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
let test_content = b"original audio data for chunk test";
|
||||
setup_test_file(&origin_dir, "test.flac", test_content);
|
||||
|
||||
let store = Arc::new(setup_cas(dir.path()).await);
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
OriginId::from("local"),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let file_meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("local"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: test_content.len() as u64,
|
||||
mtime: UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(file_meta);
|
||||
|
||||
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
|
||||
let chunk_hash = manifest.chunks[0].hash;
|
||||
let hex = chunk_hash.as_hex();
|
||||
let chunk_path = dir
|
||||
.path()
|
||||
.join("chunks")
|
||||
.join(&hex[0..2])
|
||||
.join(&hex[2..4])
|
||||
.join(&hex);
|
||||
|
||||
let mut corrupted = std::fs::read(&chunk_path).unwrap();
|
||||
corrupted[0] = corrupted[0].wrapping_add(1);
|
||||
std::fs::write(&chunk_path, &corrupted).unwrap();
|
||||
|
||||
let reader = FileReader::with_fetcher(store, fetcher);
|
||||
reader.register_manifest(manifest);
|
||||
|
||||
let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
|
||||
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"Issue 6.4: Corrupted chunk should be auto-refetched from origin"
|
||||
);
|
||||
assert_eq!(
|
||||
&result.unwrap()[..],
|
||||
test_content,
|
||||
"Data should match original after re-fetch"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_missing_chunk_triggers_origin_fetch() {
|
||||
use musicfs_cas::{ContentFetcher, FileReader};
|
||||
use musicfs_origins::LocalOrigin;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
let test_content = b"test data for missing chunk";
|
||||
setup_test_file(&origin_dir, "test.flac", test_content);
|
||||
|
||||
let store = Arc::new(setup_cas(dir.path()).await);
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
OriginId::from("local"),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let file_meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("local"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: test_content.len() as u64,
|
||||
mtime: UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(file_meta);
|
||||
|
||||
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
|
||||
let chunk_hash = manifest.chunks[0].hash;
|
||||
let hex = chunk_hash.as_hex();
|
||||
let chunk_path = dir
|
||||
.path()
|
||||
.join("chunks")
|
||||
.join(&hex[0..2])
|
||||
.join(&hex[2..4])
|
||||
.join(&hex);
|
||||
|
||||
std::fs::remove_file(&chunk_path).unwrap();
|
||||
|
||||
let reader = FileReader::with_fetcher(store, fetcher);
|
||||
reader.register_manifest(manifest);
|
||||
|
||||
let result = reader.read(FileId(1), 0, test_content.len() as u32).await;
|
||||
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"Issue 6.4: Missing chunk should be re-fetched from origin"
|
||||
);
|
||||
assert_eq!(
|
||||
&result.unwrap()[..],
|
||||
test_content,
|
||||
"Data should match original after re-fetch"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_passthrough_mode_when_cache_disk_dead() {
|
||||
use musicfs_cas::ContentFetcher;
|
||||
use musicfs_origins::LocalOrigin;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
let test_content = b"passthrough test data";
|
||||
setup_test_file(&origin_dir, "test.flac", test_content);
|
||||
|
||||
let store = Arc::new(
|
||||
CasStore::open(CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 10,
|
||||
shard_levels: 2,
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
OriginId::from("local"),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
let fetcher = Arc::new(ContentFetcher::new(store.clone()));
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let file_meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("local"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: test_content.len() as u64,
|
||||
mtime: UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(file_meta);
|
||||
|
||||
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
|
||||
|
||||
assert!(
|
||||
!manifest.chunks.is_empty(),
|
||||
"Issue 6.6: Fetch should complete even when CAS write fails (passthrough mode)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_size_tracking_is_correct() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 10_000_000,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = CasStore::open(config).await.unwrap();
|
||||
|
||||
let data = vec![0u8; 1000];
|
||||
store.put(&data).await.unwrap();
|
||||
|
||||
assert!(
|
||||
store.current_size() >= 1000,
|
||||
"Issue C6: current_size should track chunk data (recursive), got {}",
|
||||
store.current_size()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pid_file_prevents_concurrent_mount() {
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("musicfs.lock");
|
||||
|
||||
fn try_lock(path: &Path) -> Result<File, std::io::Error> {
|
||||
let file = File::create(path)?;
|
||||
let fd = file.as_raw_fd();
|
||||
let ret = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
|
||||
if ret != 0 {
|
||||
return Err(std::io::Error::last_os_error());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
let lock1 = try_lock(&lock_path);
|
||||
assert!(lock1.is_ok(), "Issue C9: First lock should succeed");
|
||||
|
||||
let lock2 = try_lock(&lock_path);
|
||||
assert!(
|
||||
lock2.is_err(),
|
||||
"Issue C9: Second lock should fail (already held)"
|
||||
);
|
||||
|
||||
drop(lock1);
|
||||
|
||||
let lock3 = try_lock(&lock_path);
|
||||
assert!(
|
||||
lock3.is_ok(),
|
||||
"Issue C9: Third lock should succeed after first released"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_panic_hook_logs_to_tracing() {
|
||||
use std::panic;
|
||||
|
||||
musicfs_core::install_panic_hook();
|
||||
|
||||
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
|
||||
panic!("test panic message");
|
||||
}));
|
||||
|
||||
assert!(result.is_err(), "Panic should have been caught");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stale_mount_check_function_exists() {
|
||||
let path = std::path::Path::new("/nonexistent/musicfs/mount");
|
||||
assert!(
|
||||
!path.exists(),
|
||||
"Test path should not exist for this test to be meaningful"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_systemd_service_has_execstoppost() {
|
||||
let service_path = std::path::Path::new("../../dist/musicfs.service");
|
||||
if !service_path.exists() {
|
||||
panic!(
|
||||
"Issue 3.7: dist/musicfs.service does not exist at {:?}",
|
||||
service_path
|
||||
);
|
||||
}
|
||||
|
||||
let content = std::fs::read_to_string(service_path).unwrap();
|
||||
assert!(
|
||||
content.contains("ExecStopPost") && content.contains("fusermount"),
|
||||
"Issue 3.7: Service file should have ExecStopPost with fusermount for cleanup"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sd_notify_ready_sent() {
|
||||
use std::os::unix::net::UnixDatagram;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let socket_path = dir.path().join("notify.sock");
|
||||
let socket = UnixDatagram::bind(&socket_path).unwrap();
|
||||
socket
|
||||
.set_read_timeout(Some(Duration::from_secs(1)))
|
||||
.unwrap();
|
||||
|
||||
std::env::set_var("NOTIFY_SOCKET", &socket_path);
|
||||
|
||||
let result = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]);
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"sd_notify should succeed when NOTIFY_SOCKET is set"
|
||||
);
|
||||
|
||||
let mut buf = [0u8; 256];
|
||||
let len = socket.recv(&mut buf).unwrap();
|
||||
let msg = std::str::from_utf8(&buf[..len]).unwrap();
|
||||
|
||||
assert!(
|
||||
msg.contains("READY=1"),
|
||||
"sd_notify should send READY=1, got: {}",
|
||||
msg
|
||||
);
|
||||
|
||||
std::env::remove_var("NOTIFY_SOCKET");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_shutdown_cancels_background_tasks() {
|
||||
let token = CancellationToken::new();
|
||||
let stopped = Arc::new(AtomicBool::new(false));
|
||||
let stopped_clone = stopped.clone();
|
||||
let token_clone = token.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
token_clone.cancelled().await;
|
||||
stopped_clone.store(true, Ordering::SeqCst);
|
||||
});
|
||||
|
||||
assert!(!stopped.load(Ordering::SeqCst));
|
||||
token.cancel();
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert!(stopped.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_shutdown_flushes_tantivy() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let idx_path = dir.path().join("idx");
|
||||
|
||||
{
|
||||
let index = SearchIndex::open(&idx_path).unwrap();
|
||||
index
|
||||
.index_file(&make_file_meta(1, "/a.flac", 1000))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
}
|
||||
|
||||
let index2 = SearchIndex::open(&idx_path).unwrap();
|
||||
assert_eq!(index2.search("a", 10).unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_supervisor_detects_task_completion() {
|
||||
let supervisor = TaskSupervisor::new();
|
||||
supervisor.spawn_supervised("fast", async {});
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_supervisor_detects_panic() {
|
||||
let supervisor = TaskSupervisor::new();
|
||||
supervisor.spawn_supervised("panicker", async {
|
||||
panic!("boom");
|
||||
});
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert!(matches!(
|
||||
supervisor.task_status("panicker"),
|
||||
TaskStatus::Failed { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_supervisor_restarts_critical_task() {
|
||||
let count = Arc::new(AtomicU32::new(0));
|
||||
let c = count.clone();
|
||||
|
||||
let supervisor = TaskSupervisor::new();
|
||||
supervisor.spawn_critical("restartable", move || {
|
||||
let c = c.clone();
|
||||
async move {
|
||||
let n = c.fetch_add(1, Ordering::SeqCst);
|
||||
if n == 0 {
|
||||
panic!("first run fails");
|
||||
}
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
assert_eq!(count.load(Ordering::SeqCst), 2);
|
||||
assert!(matches!(
|
||||
supervisor.task_status("restartable"),
|
||||
TaskStatus::Running
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_sigterm_triggers_shutdown() {
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
|
||||
let musicfs_bin = std::env::var("CARGO_BIN_EXE_musicfs").ok();
|
||||
if musicfs_bin.is_none() {
|
||||
eprintln!(
|
||||
"Skipping test_sigterm_triggers_shutdown: musicfs binary not available in test context"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let bin_path = musicfs_bin.unwrap();
|
||||
let temp_dir = tempfile::TempDir::new().unwrap();
|
||||
let mountpoint = temp_dir.path().join("mount");
|
||||
let origin = temp_dir.path().join("origin");
|
||||
std::fs::create_dir_all(&mountpoint).unwrap();
|
||||
std::fs::create_dir_all(&origin).unwrap();
|
||||
|
||||
let mut child = Command::new(&bin_path)
|
||||
.args([
|
||||
"mount",
|
||||
"--origin",
|
||||
origin.to_str().unwrap(),
|
||||
mountpoint.to_str().unwrap(),
|
||||
])
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn();
|
||||
|
||||
if child.is_err() {
|
||||
eprintln!("Skipping test_sigterm_triggers_shutdown: failed to spawn musicfs");
|
||||
return;
|
||||
}
|
||||
|
||||
let mut child = child.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
unsafe {
|
||||
libc::kill(child.id() as i32, libc::SIGTERM);
|
||||
}
|
||||
|
||||
let exit_result = timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(status)) => return status,
|
||||
Ok(None) => tokio::time::sleep(Duration::from_millis(100)).await,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
child.wait().unwrap()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
exit_result.is_ok(),
|
||||
"Issue 2.1: Process should exit within 10s after SIGTERM"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user