Refactor to split client from server

This commit is contained in:
Alexander
2026-07-01 16:31:55 +02:00
parent d35aa2aecb
commit 8dba0c097d
56 changed files with 441 additions and 326 deletions
+36
View File
@@ -0,0 +1,36 @@
[package]
name = "musicfs-client"
version.workspace = true
edition.workspace = true
default-run = "musicfs"
[lib]
name = "musicfs"
path = "src/lib.rs"
[[bin]]
name = "musicfs"
path = "src/main.rs"
[dependencies]
musicfs-core.workspace = true
musicfs-proto.workspace = true
fuser.workspace = true
libc.workspace = true
sea-orm.workspace = true
tokio.workspace = true
anyhow.workspace = true
async-trait.workspace = true
clap.workspace = true
notify.workspace = true
tokio-stream.workspace = true
tonic.workspace = true
bytes.workspace = true
http.workspace = true
tracing.workspace = true
log.workspace = true
twox-hash.workspace = true
chrono.workspace = true
[dev-dependencies]
tempfile.workspace = true
+115
View File
@@ -0,0 +1,115 @@
use sea_orm::{
ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter,
sea_query::OnConflict,
};
use tracing::{debug, error};
use crate::db::entities::cached_file_bytes::{ActiveModel, Column, Entity, Model};
/// Return the cached bytes for `inode` if present. `None` means "not cached";
/// the caller should fetch from the server and call [`put_cached_bytes`].
pub async fn get_cached_bytes(inode: i64, db: &DatabaseConnection) -> Option<Vec<u8>> {
let row: Option<Model> = match Entity::find_by_id(inode).one(db).await {
Ok(m) => m,
Err(e) => {
error!(%inode, error = %e, "get_cached_bytes: DB read failed");
None
}
};
return row.map(|m| m.data);
}
/// Insert or replace the cached bytes for `inode`. Called after a successful
/// `GetFile` round-trip so subsequent reads of the same range hit the cache.
pub async fn put_cached_bytes(inode: i64, data: Vec<u8>, db: &DatabaseConnection) {
let active = ActiveModel {
inode: Set(inode),
data: Set(data),
fetched_at: Set(chrono::Utc::now()),
};
if let Err(e) = Entity::insert(active)
.on_conflict(
OnConflict::column(Column::Inode)
.update_columns([Column::Data, Column::FetchedAt])
.to_owned(),
)
.exec(db)
.await
{
error!(%inode, error = %e, "put_cached_bytes: DB upsert failed");
}
}
/// Remove cache rows for the given inodes. Called by reconcile when a file's
/// hash has changed (the cached bytes are now stale).
pub async fn delete_cached_bytes_for(inodes: &[i64], db: &DatabaseConnection) {
if inodes.is_empty() {
return;
}
debug!(count = inodes.len(), "deleting cached bytes");
if let Err(e) = Entity::delete_many()
.filter(Column::Inode.is_in(inodes.to_vec()))
.exec(db)
.await
{
error!(error = %e, "delete_cached_bytes_for: DB delete failed");
}
}
#[cfg(test)]
mod tests {
use super::*;
use sea_orm::ConnectionTrait;
async fn connect() -> Option<DatabaseConnection> {
let pg_host = std::env::var("PGHOST").ok()?;
let url = format!("postgresql://fujin@localhost/musicfs?host={pg_host}");
let db = sea_orm::Database::connect(&url).await.ok()?;
db.execute_unprepared("SELECT 1 FROM cached_file_bytes LIMIT 0")
.await
.ok()?;
Some(db)
}
async fn setup_parent_item(db: &DatabaseConnection, inode: i64) {
db.execute_unprepared(&format!(
"INSERT INTO items (inode, name, original_path, local_path, file_type, hash) \
VALUES ({inode}, 'test', '/test', '/test', 'file', 0) \
ON CONFLICT (inode) DO NOTHING"
))
.await
.unwrap();
}
async fn teardown(db: &DatabaseConnection, inode: i64) {
let _ = db
.execute_unprepared(&format!("DELETE FROM items WHERE inode = {inode}"))
.await;
}
#[tokio::test]
async fn cache_round_trip_with_timestamptz() {
let db = match connect().await {
Some(db) => db,
None => {
eprintln!("skip: no database (PGHOST unset or unreachable)");
return;
}
};
let inode = -999_999_i64;
let test_data = b"regression-test-payload".to_vec();
setup_parent_item(&db, inode).await;
put_cached_bytes(inode, test_data.clone(), &db).await;
let cached = get_cached_bytes(inode, &db).await;
assert_eq!(
cached,
Some(test_data),
"get_cached_bytes must decode fetched_at (TIMESTAMPTZ), not fail with type mismatch"
);
teardown(&db, inode).await;
}
}
+155
View File
@@ -0,0 +1,155 @@
use sea_orm::entity::prelude::*;
use crate::item::{FileType, Item};
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "items")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub inode: i64,
pub name: String,
pub original_path: String,
pub local_path: String,
pub file_type: String,
pub hash: i64,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
impl From<&Item> for ActiveModel {
fn from(item: &Item) -> Self {
use sea_orm::ActiveValue::Set;
ActiveModel {
inode: Set(item.inode.0 as i64),
name: Set(item.name.clone()),
original_path: Set(item.original_path.to_string_lossy().into_owned()),
local_path: Set(item.local_path.to_string_lossy().into_owned()),
file_type: Set(match item.file_type {
FileType::Directory => "directory".to_string(),
FileType::File => "file".to_string(),
}),
hash: Set(item.hash as i64),
}
}
}
pub mod cached_file_bytes {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "cached_file_bytes")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub inode: i64,
#[sea_orm(column_type = "Blob")]
pub data: Vec<u8>,
pub fetched_at: DateTimeUtc,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::item::{FileType, Item};
use crate::origins::attrs::FileAttrs;
use fuser::INodeNo;
use std::path::PathBuf;
fn attrs_for_tempdir() -> FileAttrs {
let tmp = tempfile::tempdir().unwrap();
let metadata = std::fs::metadata(tmp.path()).unwrap();
return FileAttrs::from(&metadata);
}
#[test]
fn item_to_active_model_fields_match() {
let item = Item::new(
INodeNo(42),
INodeNo::ROOT,
"test".to_string(),
PathBuf::from("/original/path"),
PathBuf::from("local/path"),
FileType::Directory,
attrs_for_tempdir(),
None,
);
let am = ActiveModel::from(&item);
if let sea_orm::ActiveValue::Set(v) = &am.inode {
assert_eq!(*v, 42i64);
} else {
panic!("inode field not set");
}
if let sea_orm::ActiveValue::Set(v) = &am.name {
assert_eq!(v, "test");
} else {
panic!("name field not set");
}
if let sea_orm::ActiveValue::Set(v) = &am.original_path {
assert_eq!(v, "/original/path");
} else {
panic!("original_path field not set");
}
if let sea_orm::ActiveValue::Set(v) = &am.local_path {
assert_eq!(v, "local/path");
} else {
panic!("local_path field not set");
}
if let sea_orm::ActiveValue::Set(v) = &am.hash {
assert_eq!(*v, item.hash as i64);
} else {
panic!("hash field not set");
}
}
#[test]
fn file_type_string_mapping() {
let item_dir = Item::new(
INodeNo(1),
INodeNo::ROOT,
"dir".to_string(),
PathBuf::from("/original/dir"),
PathBuf::from("local/dir"),
FileType::Directory,
attrs_for_tempdir(),
None,
);
let am_dir = ActiveModel::from(&item_dir);
if let sea_orm::ActiveValue::Set(v) = &am_dir.file_type {
assert_eq!(v, "directory");
} else {
panic!("file_type field not set for directory");
}
let item_file = Item::new(
INodeNo(2),
INodeNo::ROOT,
"file".to_string(),
PathBuf::from("/original/file"),
PathBuf::from("local/file"),
FileType::File,
attrs_for_tempdir(),
None,
);
let am_file = ActiveModel::from(&item_file);
if let sea_orm::ActiveValue::Set(v) = &am_file.file_type {
assert_eq!(v, "file");
} else {
panic!("file_type field not set for file");
}
}
}
+3
View File
@@ -0,0 +1,3 @@
pub mod cache;
pub mod entities;
pub mod sync;
+92
View File
@@ -0,0 +1,92 @@
use std::collections::{BTreeMap, HashMap, HashSet};
use fuser::INodeNo;
use sea_orm::entity::prelude::*;
use tracing::{debug, error};
use crate::db::entities::{ActiveModel, Entity, Model};
use crate::item::Item;
use crate::music::db::save_music_metadata;
pub async fn sync_items_to_db(
snapshot: &BTreeMap<INodeNo, Item>,
db_items: &HashMap<i64, Model>,
client: &sea_orm::DatabaseConnection,
) {
let mut to_insert: Vec<ActiveModel> = vec![];
let mut to_update: Vec<ActiveModel> = vec![];
let mut to_delete: Vec<i64> = vec![];
let mut to_save_music: Vec<i64> = vec![];
for (ino, item) in snapshot {
let ino_i64 = ino.0 as i64;
match db_items.get(&ino_i64) {
None => {
to_insert.push(ActiveModel::from(item));
if item.music_metadata.is_some() {
to_save_music.push(ino_i64);
}
}
Some(db_item) if db_item.hash != item.hash as i64 => {
to_update.push(ActiveModel::from(item));
if item.music_metadata.is_some() {
to_save_music.push(ino_i64);
}
}
_ => {}
}
}
let fresh_inodes: HashSet<i64> = snapshot.keys().map(|i| i.0 as i64).collect();
for ino in db_items.keys().filter(|i| !fresh_inodes.contains(i)) {
to_delete.push(*ino);
}
debug!(
insert = to_insert.len(),
update = to_update.len(),
delete = to_delete.len(),
save_music = to_save_music.len(),
"sync_items_to_db"
);
if !to_insert.is_empty() {
if let Err(e) = Entity::insert_many(to_insert).exec(client).await {
error!(error = %e, "sync: insert_many failed");
}
}
for model in to_update {
if let Err(e) = model.update(client).await {
error!(error = %e, "sync: update failed");
}
}
for ino in to_delete {
if let Err(e) = Entity::delete_by_id(ino).exec(client).await {
error!(%ino, error = %e, "sync: delete failed");
}
}
for ino_i64 in &to_save_music {
let ino = INodeNo(*ino_i64 as u64);
if let Some(music_metadata) = snapshot
.get(&ino)
.and_then(|item| item.music_metadata.as_ref())
{
if let Err(e) = save_music_metadata(*ino_i64, music_metadata, client).await {
error!(ino = *ino_i64, error = %e, "sync: save_music_metadata failed");
}
}
}
}
pub fn run_db_blocking<F: std::future::Future>(future: F) -> F::Output {
return tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.unwrap_or_else(|e| {
error!(error = %e, "run_db_blocking: failed to build runtime");
panic!("run_db_blocking: failed to build runtime: {e}");
})
.block_on(future);
}
+146
View File
@@ -0,0 +1,146 @@
use std::{
path::PathBuf,
time::{SystemTime, UNIX_EPOCH},
};
use std::os::unix::ffi::OsStrExt;
use fuser::INodeNo;
use crate::music::metadata::MusicMetadata;
use crate::origins::attrs::FileAttrs;
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
pub enum FileType {
Directory,
File,
}
#[derive(Debug, Clone)]
pub struct Item {
pub inode: INodeNo,
pub parent_inode: INodeNo,
pub name: String,
pub original_path: PathBuf,
pub local_path: PathBuf,
pub file_type: FileType,
pub attrs: FileAttrs,
pub music_metadata: Option<MusicMetadata>,
pub hash: u64,
}
fn secs_since_epoch(time: SystemTime) -> u64 {
return time
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
}
impl Item {
#[allow(clippy::too_many_arguments)]
pub fn new(
inode: INodeNo,
parent_inode: INodeNo,
name: String,
original_path: PathBuf,
local_path: PathBuf,
file_type: FileType,
attrs: FileAttrs,
music_metadata: Option<MusicMetadata>,
) -> Item {
let mut item = Item {
inode,
parent_inode,
name,
original_path,
local_path,
file_type,
attrs,
music_metadata,
hash: 0,
};
item.hash = item.compute_hash();
return item;
}
pub fn compute_hash(&self) -> u64 {
musicfs_core::compute_item_hash(
self.inode.0,
self.original_path.as_os_str().as_bytes(),
secs_since_epoch(self.attrs.ctime),
secs_since_epoch(self.attrs.mtime),
secs_since_epoch(self.attrs.crtime),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::origins::attrs::FileAttrs;
#[test]
fn compute_hash_deterministic() {
let attrs = file_attrs_for_tempdir();
let item1 = Item::new(
INodeNo(42),
INodeNo::ROOT,
"test".to_string(),
PathBuf::from("/some/path"),
PathBuf::from("test"),
FileType::File,
attrs.clone(),
None,
);
let item2 = Item::new(
INodeNo(42),
INodeNo::ROOT,
"test".to_string(),
PathBuf::from("/some/path"),
PathBuf::from("test"),
FileType::File,
attrs,
None,
);
assert_eq!(item1.hash, item2.hash);
}
#[test]
fn compute_hash_changes_on_path_change() {
let attrs = file_attrs_for_tempdir();
let item1 = Item::new(
INodeNo(42),
INodeNo::ROOT,
"test".to_string(),
PathBuf::from("/some/path"),
PathBuf::from("test"),
FileType::File,
attrs.clone(),
None,
);
let item2 = Item::new(
INodeNo(42),
INodeNo::ROOT,
"test".to_string(),
PathBuf::from("/different/path"),
PathBuf::from("test"),
FileType::File,
attrs,
None,
);
assert_ne!(item1.hash, item2.hash);
}
fn file_attrs_for_tempdir() -> FileAttrs {
let tmp = tempfile::tempdir().unwrap();
let metadata = std::fs::metadata(tmp.path()).unwrap();
return FileAttrs::from(&metadata);
}
}
+8
View File
@@ -0,0 +1,8 @@
pub mod db;
pub mod item;
pub mod music;
pub mod origins;
pub mod virtual_dirs;
pub use musicfs_core::logging;
pub use musicfs_proto as proto;
+151
View File
@@ -0,0 +1,151 @@
use std::{collections::HashMap, path::Path, sync::Arc};
use clap::Parser;
use fuser::INodeNo;
use musicfs::db::entities::{Entity, Model};
use musicfs::db::sync::sync_items_to_db;
use musicfs::item::Item;
use musicfs::logging::{LogConfig, init};
use musicfs::music::db::restore_music_metadata_from_db;
use musicfs::origins::local::LocalOrigin;
use musicfs::origins::network::NetworkOrigin;
use musicfs::origins::{FuseFs, Origin};
use musicfs::virtual_dirs::restore_virtual_paths;
use sea_orm::entity::prelude::*;
use tokio::signal::unix::{SignalKind, signal};
use tracing::{debug, error, info};
#[derive(Parser, Debug)]
#[command(version, about, long_about = None)]
struct Args {
#[arg(short, long, required = true)]
mountpoint: String,
/// Local directory path (→ LocalOrigin) OR `http://host:port` URL of a
/// musicfs-server (→ NetworkOrigin).
#[arg(short, long, required = true)]
source: String,
#[arg(short, long, required = true)]
database: String,
/// Directory for daily-rotated log files.
#[arg(long, default_value = "./logs")]
log_dir: std::path::PathBuf,
}
#[tokio::main]
async fn main() {
let args = Args::parse();
// Bind the guard for the whole process so the non-blocking file writer
// flushes on exit. Initialized before anything else so even early
// failures land in the log.
let _guard = init(LogConfig {
log_dir: args.log_dir.clone(),
file_prefix: "musicfs".to_string(),
max_files: 7,
});
let mountpoint = args.mountpoint.clone();
info!(
mountpoint = %mountpoint,
source = %args.source,
database = %args.database,
log_dir = %args.log_dir.display(),
"musicfs starting"
);
// sea-orm logs every statement at INFO by default, which floods normal
// output; demote to DEBUG so queries stay hidden under RUST_LOG=info.
let mut db_opts = sea_orm::ConnectOptions::new(args.database.clone());
db_opts.sqlx_logging_level(log::LevelFilter::Debug);
let db = sea_orm::Database::connect(db_opts)
.await
.unwrap_or_else(|e| {
error!(database = %args.database, error = %e, "database connect failed");
panic!("database connect: {e}");
});
info!(database = %args.database, "database connected");
let (snapshot, byte_source, watcher) = if looks_like_url(&args.source) {
debug!(source = %args.source, "using NetworkOrigin");
let origin = NetworkOrigin::new(args.source.clone(), mountpoint.clone(), db.clone())
.unwrap_or_else(|e| {
error!(source = %args.source, error = %e, "network origin init failed");
panic!("network origin init: {e}");
});
// Await directly on this runtime; the sync `snapshot()` wrapper would
// try to build a nested runtime and panic (we're inside #[tokio::main]).
let snapshot = origin.snapshot_async().await.unwrap_or_else(|e| {
error!(source = %args.source, error = %e, "network initial snapshot failed");
panic!("network initial snapshot: {e}");
});
info!(files = snapshot.len(), "network snapshot complete");
(snapshot, origin.byte_source(), origin.watcher())
} else {
debug!(source = %args.source, "using LocalOrigin");
let origin = LocalOrigin::new(args.source.clone(), mountpoint.clone());
let mut snapshot = origin.snapshot().unwrap_or_else(|e| {
error!(source = %args.source, error = %e, "local initial snapshot failed");
panic!("local initial snapshot: {e}");
});
let db_items: HashMap<i64, Model> = Entity::find()
.all(&db)
.await
.unwrap_or_else(|e| {
error!(error = %e, "loading db items failed");
panic!("loading db items: {e}");
})
.into_iter()
.map(|e| (e.inode, e))
.collect();
sync_items_to_db(&snapshot, &db_items, &db).await;
restore_music_metadata_from_db(&mut snapshot, &db_items, &db).await;
restore_virtual_paths(&mut snapshot, &db_items, Path::new(&args.source));
info!(files = snapshot.len(), "local snapshot complete");
(snapshot, origin.byte_source(), origin.watcher())
};
let files: Arc<std::sync::Mutex<std::collections::BTreeMap<INodeNo, Item>>> =
Arc::new(std::sync::Mutex::new(snapshot));
let watcher_handle = watcher.watch(files.clone());
let fs = FuseFs {
files,
bytes: byte_source,
client: db,
runtime_handle: tokio::runtime::Handle::current(),
};
let cfg = fuser::Config::default();
let session = fuser::spawn_mount2(fs, &mountpoint, &cfg).unwrap_or_else(|e| {
error!(mountpoint = %mountpoint, error = %e, "failed to mount FUSE filesystem");
panic!("failed to mount FUSE filesystem: {e}");
});
info!(mountpoint = %mountpoint, "FUSE mounted");
let mut sigint = signal(SignalKind::interrupt()).unwrap_or_else(|e| {
error!(error = %e, "failed to register SIGINT handler");
panic!("register SIGINT handler: {e}");
});
let mut sigterm = signal(SignalKind::terminate()).unwrap_or_else(|e| {
error!(error = %e, "failed to register SIGTERM handler");
panic!("register SIGTERM handler: {e}");
});
tokio::select! {
_ = sigint.recv() => info!("received SIGINT, shutting down"),
_ = sigterm.recv() => info!("received SIGTERM, shutting down"),
}
info!(mountpoint = %mountpoint, "unmounting");
watcher_handle.stop();
drop(session);
}
fn looks_like_url(s: &str) -> bool {
return s.starts_with("http://") || s.starts_with("https://");
}
+292
View File
@@ -0,0 +1,292 @@
use std::collections::{BTreeMap, HashMap};
use fuser::INodeNo;
use sea_orm::entity::prelude::*;
use crate::db::entities::Model;
use crate::item::Item;
use crate::music::metadata::MusicMetadata;
use tracing::{debug, error};
mod entities {
pub mod music_metadata {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "music_metadata")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub inode: i64,
pub track_title: String,
pub album: String,
pub track_number: i32,
#[sea_orm(column_type = "Blob")]
pub header: Vec<u8>,
pub real_audio_start: i64,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
pub mod artists {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "music_metadata_artists")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub inode: i64,
#[sea_orm(primary_key, auto_increment = false)]
pub artist: String,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
pub mod other_tags {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "music_metadata_other_tags")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub inode: i64,
#[sea_orm(primary_key, auto_increment = false)]
pub position: i32,
pub tag: String,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
pub mod pictures {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "music_metadata_pictures")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub inode: i64,
#[sea_orm(primary_key, auto_increment = false)]
pub position: i32,
#[sea_orm(column_type = "Blob")]
pub block_header: Vec<u8>,
pub data_offset: i64,
pub data_length: i64,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
}
use entities::{artists, music_metadata as music_metadata_entity, other_tags, pictures};
pub async fn save_music_metadata(
inode: i64,
music_metadata: &MusicMetadata,
client: &sea_orm::DatabaseConnection,
) -> Result<(), sea_orm::DbErr> {
use sea_orm::ActiveValue::Set;
music_metadata_entity::Entity::delete_by_id(inode)
.exec(client)
.await?;
music_metadata_entity::Entity::insert(music_metadata_entity::ActiveModel {
inode: Set(inode),
track_title: Set(music_metadata.track_title.clone()),
album: Set(music_metadata.album.clone()),
track_number: Set(music_metadata.track_number),
header: Set(music_metadata.header.clone()),
real_audio_start: Set(music_metadata.real_audio_start as i64),
})
.exec(client)
.await?;
if !music_metadata.artist.is_empty() {
artists::Entity::insert_many(music_metadata.artist.iter().map(|a| artists::ActiveModel {
inode: Set(inode),
artist: Set(a.clone()),
}))
.exec(client)
.await?;
}
if !music_metadata.other_tags.is_empty() {
other_tags::Entity::insert_many(music_metadata.other_tags.iter().enumerate().map(
|(pos, tag)| other_tags::ActiveModel {
inode: Set(inode),
position: Set(pos as i32),
tag: Set(tag.clone()),
},
))
.exec(client)
.await?;
}
if !music_metadata.picture_block_headers.is_empty() {
pictures::Entity::insert_many(
music_metadata
.picture_block_headers
.iter()
.zip(music_metadata.picture_data_ranges.iter())
.enumerate()
.map(|(pos, (hdr, (offset, len)))| pictures::ActiveModel {
inode: Set(inode),
position: Set(pos as i32),
block_header: Set(hdr.clone()),
data_offset: Set(*offset as i64),
data_length: Set(*len as i64),
}),
)
.exec(client)
.await?;
}
return Ok(());
}
pub async fn restore_music_metadata_from_db(
snapshot: &mut BTreeMap<INodeNo, Item>,
db_items: &HashMap<i64, Model>,
client: &sea_orm::DatabaseConnection,
) {
use artists::Column as ArtCol;
use music_metadata_entity::Column as MmCol;
use other_tags::Column as OtCol;
use pictures::Column as PicCol;
let unchanged_music_inodes: Vec<i64> = db_items
.values()
.filter_map(|db_item| {
let ino = INodeNo(db_item.inode as u64);
snapshot
.get(&ino)
.filter(|item| item.hash as i64 == db_item.hash && item.music_metadata.is_some())
.map(|_| db_item.inode)
})
.collect();
if unchanged_music_inodes.is_empty() {
return;
}
let mm_rows: HashMap<i64, music_metadata_entity::Model> =
match music_metadata_entity::Entity::find()
.filter(MmCol::Inode.is_in(unchanged_music_inodes.clone()))
.all(client)
.await
{
Ok(rows) => rows.into_iter().map(|m| (m.inode, m)).collect(),
Err(e) => {
error!(error = %e, "restore_music_metadata: mm rows query failed");
return;
}
};
let mut artists_by_inode: HashMap<i64, Vec<String>> = HashMap::new();
let artist_rows = match artists::Entity::find()
.filter(ArtCol::Inode.is_in(unchanged_music_inodes.clone()))
.all(client)
.await
{
Ok(r) => r,
Err(e) => {
error!(error = %e, "restore_music_metadata: artists query failed");
vec![]
}
};
for row in artist_rows {
artists_by_inode
.entry(row.inode)
.or_default()
.push(row.artist);
}
let mut other_tags_by_inode: HashMap<i64, Vec<(i32, String)>> = HashMap::new();
let other_tag_rows = match other_tags::Entity::find()
.filter(OtCol::Inode.is_in(unchanged_music_inodes.clone()))
.all(client)
.await
{
Ok(r) => r,
Err(e) => {
error!(error = %e, "restore_music_metadata: other_tags query failed");
vec![]
}
};
for row in other_tag_rows {
other_tags_by_inode
.entry(row.inode)
.or_default()
.push((row.position, row.tag));
}
let mut pictures_by_inode: HashMap<i64, Vec<pictures::Model>> = HashMap::new();
let picture_rows = match pictures::Entity::find()
.filter(PicCol::Inode.is_in(unchanged_music_inodes))
.all(client)
.await
{
Ok(r) => r,
Err(e) => {
error!(error = %e, "restore_music_metadata: pictures query failed");
vec![]
}
};
for row in picture_rows {
pictures_by_inode.entry(row.inode).or_default().push(row);
}
debug!(restored = mm_rows.len(), "restored music metadata from db");
for (inode, mm_row) in mm_rows {
let ino = INodeNo(inode as u64);
let Some(item) = snapshot.get_mut(&ino) else {
continue;
};
let mut sorted_tags = other_tags_by_inode.remove(&inode).unwrap_or_default();
sorted_tags.sort_by_key(|(pos, _)| *pos);
let mut sorted_pics = pictures_by_inode.remove(&inode).unwrap_or_default();
sorted_pics.sort_by_key(|p| p.position);
let picture_block_headers: Vec<Vec<u8>> =
sorted_pics.iter().map(|p| p.block_header.clone()).collect();
let picture_data_ranges: Vec<(u64, u64)> = sorted_pics
.iter()
.map(|p| (p.data_offset as u64, p.data_length as u64))
.collect();
let mut music_metadata = MusicMetadata {
artist: artists_by_inode.remove(&inode).unwrap_or_default(),
album_artist: None,
album: mm_row.album,
track_number: mm_row.track_number,
track_title: mm_row.track_title,
other_tags: sorted_tags.into_iter().map(|(_, tag)| tag).collect(),
header: mm_row.header,
picture_block_headers,
picture_data_ranges,
real_audio_start: mm_row.real_audio_start as u64,
vorbis_comment_offset: 0,
vorbis_comment_length: 0,
};
music_metadata.find_vorbis_offsets();
item.music_metadata = Some(music_metadata);
}
}
+2
View File
@@ -0,0 +1,2 @@
pub mod db;
pub use musicfs_core::music::{metadata, parse};
@@ -0,0 +1,215 @@
use std::{
fs,
io::{self, Read, Seek, SeekFrom},
path::Path,
};
pub fn read_bytes_at(path: &Path, offset: u64, len: usize) -> io::Result<Vec<u8>> {
let mut f = fs::File::open(path)?;
f.seek(SeekFrom::Start(offset))?;
let mut buf = vec![0u8; len];
let n = f.read(&mut buf)?;
buf.truncate(n);
return Ok(buf);
}
fn assemble_virtual_read(
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
header: &[u8],
pic_hdrs: &[Vec<u8>],
pic_ranges: &[(u64, u64)],
real_audio_start: u64,
offset: u64,
size: u32,
) -> io::Result<Vec<u8>> {
let end = offset + size as u64;
let header_end = header.len() as u64;
if end <= header_end {
return Ok(header[offset as usize..end as usize].to_vec());
}
let mut buf = Vec::with_capacity(size as usize);
if offset < header_end {
buf.extend_from_slice(&header[offset as usize..]);
}
let mut virt_pos = header_end;
for (pic_hdr, (data_real_offset, data_len)) in pic_hdrs.iter().zip(pic_ranges.iter()) {
let pic_hdr_len = pic_hdr.len() as u64;
let pic_hdr_end = virt_pos + pic_hdr_len;
let pic_end = pic_hdr_end + data_len;
if end <= virt_pos {
break;
}
if offset >= pic_end {
virt_pos = pic_end;
continue;
}
let hdr_from = (offset.max(virt_pos) - virt_pos) as usize;
let hdr_to = ((end.min(pic_hdr_end)) - virt_pos) as usize;
if hdr_from < hdr_to {
buf.extend_from_slice(&pic_hdr[hdr_from..hdr_to.min(pic_hdr.len())]);
}
let data_start = offset.max(pic_hdr_end);
let data_end = end.min(pic_end);
if data_start < data_end {
let real_off = data_real_offset + (data_start - pic_hdr_end);
let bytes = reader(real_off, (data_end - data_start) as usize)?;
buf.extend_from_slice(&bytes);
}
virt_pos = pic_end;
}
let audio_start = offset.max(virt_pos);
if audio_start < end {
let real_off = real_audio_start + (audio_start - virt_pos);
let bytes = reader(real_off, (end - audio_start) as usize)?;
buf.extend_from_slice(&bytes);
}
return Ok(buf);
}
pub fn assemble_flac_read(
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
header: &[u8],
pic_hdrs: &[Vec<u8>],
pic_ranges: &[(u64, u64)],
real_audio_start: u64,
offset: u64,
size: u32,
) -> io::Result<Vec<u8>> {
assemble_virtual_read(
reader,
header,
pic_hdrs,
pic_ranges,
real_audio_start,
offset,
size,
)
}
pub fn assemble_mp3_read(
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
header: &[u8],
pic_hdrs: &[Vec<u8>],
pic_ranges: &[(u64, u64)],
real_audio_start: u64,
offset: u64,
size: u32,
) -> io::Result<Vec<u8>> {
assemble_virtual_read(
reader,
header,
pic_hdrs,
pic_ranges,
real_audio_start,
offset,
size,
)
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use std::path::PathBuf;
fn file_reader(path: PathBuf) -> impl Fn(u64, usize) -> io::Result<Vec<u8>> {
return move |offset, len| read_bytes_at(&path, offset, len);
}
#[test]
fn read_bytes_at_full_file() {
let mut f = tempfile::NamedTempFile::new().unwrap();
f.write_all(b"hello world").unwrap();
f.flush().unwrap();
let path = f.path();
let result = read_bytes_at(path, 0, 11).unwrap();
assert_eq!(result, b"hello world");
}
#[test]
fn read_bytes_at_offset() {
let mut f = tempfile::NamedTempFile::new().unwrap();
f.write_all(b"abcdefghij").unwrap();
f.flush().unwrap();
let path = f.path();
let result = read_bytes_at(path, 3, 4).unwrap();
assert_eq!(result, b"defg");
}
#[test]
fn assemble_flac_read_header_only() {
let header = b"HEADERDATA";
let noop = |_, _| Ok(vec![]);
let result = assemble_flac_read(&noop, header, &[], &[], 0, 2, 4).unwrap();
assert_eq!(result, b"ADER");
}
#[test]
fn assemble_flac_read_audio_region() {
let mut f = tempfile::NamedTempFile::new().unwrap();
let content: Vec<u8> = (0..20).collect();
f.write_all(&content).unwrap();
f.flush().unwrap();
let path = f.path().to_path_buf();
let reader = file_reader(path);
let header = b"HDR";
let result = assemble_flac_read(&reader, header, &[], &[], 10, 5, 4).unwrap();
// offset=5, header_len=3, so we read from header[3..] (0 bytes) + audio at real_off=10+(5-3)=12
// content[12..16] = [12, 13, 14, 15]
assert_eq!(result, vec![12, 13, 14, 15]);
}
#[test]
fn assemble_flac_read_header_audio_boundary() {
let mut f = tempfile::NamedTempFile::new().unwrap();
let content: Vec<u8> = (0..110).collect();
f.write_all(&content).unwrap();
f.flush().unwrap();
let path = f.path().to_path_buf();
let reader = file_reader(path);
let header = b"ABCD";
let result = assemble_flac_read(&reader, header, &[], &[], 100, 2, 6).unwrap();
// offset=2, size=6, header_len=4
// First 2 bytes from header[2..4] = "CD"
// Remaining 4 bytes from audio at real_off=100+(2+2-4)=100
// content[100..104] = [100, 101, 102, 103]
let expected = [b'C', b'D', 100, 101, 102, 103];
assert_eq!(result, expected);
}
#[test]
fn assemble_mp3_read_header_only() {
let header = b"ID3\x04\x00DATAHERE";
let noop = |_, _| Ok(vec![]);
let result = assemble_mp3_read(&noop, header, &[], &[], 0, 4, 4).unwrap();
assert_eq!(result, &header[4..8]);
}
#[test]
fn assemble_mp3_read_audio_region() {
let mut f = tempfile::NamedTempFile::new().unwrap();
let content: Vec<u8> = (0..20).collect();
f.write_all(&content).unwrap();
f.flush().unwrap();
let path = f.path().to_path_buf();
let reader = file_reader(path);
let header = b"ID3";
let result = assemble_mp3_read(&reader, header, &[], &[], 10, 5, 4).unwrap();
assert_eq!(result, vec![12, 13, 14, 15]);
}
}
@@ -0,0 +1,73 @@
pub mod file_io;
pub mod snapshot;
pub mod watcher;
use std::{
collections::BTreeMap,
io,
path::{Path, PathBuf},
sync::Arc,
};
use fuser::INodeNo;
use crate::item::Item;
use crate::origins::{ByteSource, FileWatcher, Origin};
pub struct LocalOrigin {
pub(crate) source: PathBuf,
pub(crate) destination: PathBuf,
}
impl std::fmt::Debug for LocalOrigin {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return f
.debug_struct("LocalOrigin")
.field("source", &self.source)
.field("destination", &self.destination)
.finish();
}
}
impl LocalOrigin {
pub fn new(source: String, destination: String) -> LocalOrigin {
return LocalOrigin {
source: source.into(),
destination: destination.into(),
};
}
pub fn source(&self) -> &Path {
return &self.source;
}
}
impl Origin for LocalOrigin {
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
return snapshot::build_snapshot(&self.source, &self.destination);
}
fn byte_source(&self) -> Arc<dyn ByteSource> {
return Arc::new(LocalByteSource);
}
fn watcher(&self) -> Box<dyn FileWatcher> {
return Box::new(watcher::LocalOriginFileWatcher::new(
self.source.clone(),
self.destination.clone(),
));
}
}
pub struct LocalByteSource;
impl ByteSource for LocalByteSource {
fn read_at(
&self,
_inode: INodeNo,
locator: &Path,
offset: u64,
len: usize,
) -> io::Result<Vec<u8>> {
return file_io::read_bytes_at(locator, offset, len);
}
}
@@ -0,0 +1,110 @@
use std::{
collections::BTreeMap,
fs, io,
path::{Path, PathBuf},
sync::{Arc, Mutex},
};
use fuser::INodeNo;
use std::os::unix::fs::MetadataExt;
use crate::item::{FileType, Item};
use crate::music::parse::parse_music_metadata_for_path;
use crate::origins::attrs::FileAttrs;
use crate::virtual_dirs::ensure_virtual_dirs;
use tracing::{error, info};
pub fn fill_fileset(map: &Arc<Mutex<BTreeMap<INodeNo, Item>>>, source: &Path, destination: &Path) {
match build_snapshot(source, destination) {
Ok(new_snapshot) => {
let count = new_snapshot.len();
let mut files = map.lock().unwrap();
files.retain(|ino, _| new_snapshot.contains_key(ino));
for (ino, new_item) in new_snapshot {
match files.get(&ino) {
Some(existing) if existing.hash == new_item.hash => {}
_ => {
files.insert(ino, new_item);
}
}
}
info!(count, "snapshot rebuilt");
}
Err(e) => error!(source = %source.display(), error = %e, "error while reading source"),
}
}
pub fn build_snapshot(
source: &Path,
destination: &Path,
) -> Result<BTreeMap<INodeNo, Item>, io::Error> {
let mut map = BTreeMap::new();
let local_root = Item::new(
INodeNo::ROOT,
INodeNo::ROOT,
"/".to_string(),
source.to_path_buf(),
destination.to_path_buf(),
FileType::Directory,
FileAttrs::from(&fs::metadata(source)?),
None,
);
map.insert(INodeNo::ROOT, local_root);
read_into_map(source, destination, &mut map)?;
return Ok(map);
}
pub fn read_into_map(
source: &Path,
destination: &Path,
map: &mut BTreeMap<INodeNo, Item>,
) -> Result<(), io::Error> {
for item in fs::read_dir(source)? {
let entry = item?;
let item_path = entry.path();
if entry.file_type()?.is_dir() {
read_into_map(&item_path, destination, map)?;
continue;
}
let name = entry.file_name().to_string_lossy().into_owned();
let metadata = entry.metadata()?;
let music_metadata = parse_music_metadata_for_path(&item_path);
let mut local_path = PathBuf::new();
if let Some(ref mm) = music_metadata {
let joined;
let artist_dir = match mm.album_artist.as_deref() {
Some(a) => a,
None => {
joined = mm.artist.join("-");
&joined
}
};
local_path.push(artist_dir);
local_path.push(&mm.album);
}
local_path.push(&name);
let inode = INodeNo(metadata.ino());
let parent_inode = ensure_virtual_dirs(&local_path, source, map);
let local_item = Item::new(
inode,
parent_inode,
name,
item_path,
local_path,
FileType::File,
FileAttrs::from(&metadata),
music_metadata,
);
map.insert(inode, local_item);
}
return Ok(());
}
@@ -0,0 +1,80 @@
use std::{
collections::BTreeMap,
path::PathBuf,
sync::{Arc, Mutex, mpsc},
thread,
};
use fuser::INodeNo;
use notify::{Event, EventKind, RecursiveMode, Watcher};
use crate::item::Item;
use crate::origins::{FileWatcher, WatcherHandle};
use tracing::{debug, error, info, trace};
pub struct LocalOriginFileWatcher {
source: PathBuf,
destination: PathBuf,
}
impl LocalOriginFileWatcher {
pub fn new(source: PathBuf, destination: PathBuf) -> Self {
return LocalOriginFileWatcher {
source,
destination,
};
}
}
impl FileWatcher for LocalOriginFileWatcher {
fn watch(&self, files: Arc<Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle {
let source = self.source.clone();
let destination = self.destination.clone();
info!(source = %source.display(), "starting file watcher");
thread::spawn(move || {
let (tx, rx): (
mpsc::Sender<Result<Event, notify::Error>>,
mpsc::Receiver<Result<Event, notify::Error>>,
) = mpsc::channel();
let mut watcher: notify::INotifyWatcher = match notify::recommended_watcher(tx) {
Ok(w) => w,
Err(e) => {
error!(error = %e, "watcher: failed to create notify watcher");
return;
}
};
if let Err(e) = watcher.watch(&source, RecursiveMode::Recursive) {
error!(source = %source.display(), error = %e, "watcher: failed to watch source");
return;
}
for res in rx {
match res {
Ok(event) => {
trace!(?event, "watcher event");
match event.kind {
EventKind::Any => {
debug!("watcher: EventKind::Any, ignoring");
}
EventKind::Access(_access_kind) => {
debug!("watcher: item accessed");
}
EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) => {
debug!("watcher: create/modify/remove; rebuilding snapshot");
super::snapshot::fill_fileset(&files, &source, &destination);
}
EventKind::Other => {
debug!("watcher: other event, ignoring");
}
}
}
Err(e) => error!(error = %e, "watcher: notify error"),
}
}
});
return WatcherHandle::detached();
}
}
+631
View File
@@ -0,0 +1,631 @@
pub use musicfs_core::attrs;
pub mod local;
pub mod network;
use std::{
collections::BTreeMap,
io,
path::Path,
sync::{Arc, Mutex},
time::Duration,
};
use fuser::{Errno, FileAttr, Filesystem, Generation, INodeNo, Request};
use sea_orm::{ActiveModelTrait, DatabaseConnection};
use tracing::{debug, error, trace};
use crate::item::{FileType, Item};
use crate::music::db::save_music_metadata;
use crate::origins::local::file_io;
use crate::virtual_dirs::parent_inode_from_path;
/// Transport-agnostic byte-range reader. `locator` is whatever string the
/// origin treats as a file key: a filesystem path for `LocalOrigin`, a remote
/// key for the future network origin. Both produce bytes at `(offset, len)`.
///
/// `inode` is the FUSE-level inode; NetworkOrigin uses it as the server-side
/// file id (and as the Postgres cache key). LocalOrigin ignores it.
pub trait ByteSource: Send + Sync {
fn read_at(
&self,
inode: INodeNo,
locator: &Path,
offset: u64,
len: usize,
) -> io::Result<Vec<u8>>;
}
pub trait FileWatcher: Send + Sync {
fn watch(&self, files: Arc<Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle;
}
/// Returned by [`FileWatcher::watch`] so the caller can stop a background
/// watcher before the process (and its tokio runtime) shuts down. Without
/// this, an async watcher blocked on a timer/stream panics when the runtime is
/// torn down out from under it.
pub struct WatcherHandle {
stop: Option<Box<dyn FnOnce() + Send>>,
}
impl WatcherHandle {
pub fn new(stop: impl FnOnce() + Send + 'static) -> Self {
return WatcherHandle {
stop: Some(Box::new(stop)),
};
}
/// A watcher with no shutdown work — its thread exits on its own.
pub fn detached() -> Self {
return WatcherHandle { stop: None };
}
/// Signal the watcher to stop and wait for it to finish.
pub fn stop(mut self) {
if let Some(stop) = self.stop.take() {
stop();
}
}
}
pub trait Origin: Send + Sync {
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>>;
fn byte_source(&self) -> Arc<dyn ByteSource>;
fn watcher(&self) -> Box<dyn FileWatcher>;
}
pub struct FuseFs {
pub files: Arc<Mutex<BTreeMap<INodeNo, Item>>>,
pub bytes: Arc<dyn ByteSource>,
pub client: DatabaseConnection,
pub runtime_handle: tokio::runtime::Handle,
}
pub(crate) fn file_to_attr(item: &Item) -> FileAttr {
let attrs = &item.attrs;
let kind = match item.file_type {
FileType::Directory => fuser::FileType::Directory,
FileType::File => fuser::FileType::RegularFile,
};
let size = match &item.music_metadata {
Some(mm) if !mm.header.is_empty() => mm.virtual_size(attrs.size),
_ => attrs.size,
};
return FileAttr {
ino: item.inode,
size,
blocks: attrs.blocks,
atime: attrs.atime,
mtime: attrs.mtime,
ctime: attrs.ctime,
crtime: attrs.crtime,
kind,
perm: attrs.perm,
nlink: attrs.nlink,
uid: attrs.uid,
gid: attrs.gid,
rdev: attrs.rdev,
blksize: attrs.blksize,
flags: 0,
};
}
impl Filesystem for FuseFs {
fn open(
&self,
_req: &Request,
ino: INodeNo,
_flags: fuser::OpenFlags,
reply: fuser::ReplyOpen,
) {
trace!(%ino, "open");
if self.files.lock().unwrap().contains_key(&ino) {
reply.opened(fuser::FileHandle(ino.0), fuser::FopenFlags::empty());
} else {
debug!(%ino, "open: not found");
reply.error(Errno::ENOENT);
}
}
fn setattr(
&self,
_req: &Request,
ino: INodeNo,
_mode: Option<u32>,
_uid: Option<u32>,
_gid: Option<u32>,
_size: Option<u64>,
_atime: Option<fuser::TimeOrNow>,
_mtime: Option<fuser::TimeOrNow>,
_ctime: Option<std::time::SystemTime>,
_fh: Option<fuser::FileHandle>,
_crtime: Option<std::time::SystemTime>,
_chgtime: Option<std::time::SystemTime>,
_bkuptime: Option<std::time::SystemTime>,
_flags: Option<fuser::BsdFileFlags>,
reply: fuser::ReplyAttr,
) {
trace!(%ino, "setattr");
match self.files.lock().unwrap().get(&ino) {
Some(file) => reply.attr(&Duration::new(1, 0), &file_to_attr(file)),
None => {
debug!(%ino, "setattr: not found");
reply.error(Errno::ENOENT);
}
}
}
fn write(
&self,
_req: &Request,
ino: INodeNo,
_fh: fuser::FileHandle,
offset: u64,
data: &[u8],
_write_flags: fuser::WriteFlags,
_flags: fuser::OpenFlags,
_lock_owner: Option<fuser::LockOwner>,
reply: fuser::ReplyWrite,
) {
trace!(%ino, offset, len = data.len(), "write");
let written = data.len() as u32;
let write_start = offset;
let write_end = write_start + data.len() as u64;
let updated_music_metadata = {
let mut files = self.files.lock().unwrap();
let item = match files.get_mut(&ino) {
Some(item) => item,
None => {
debug!(%ino, "write: not found");
reply.written(written);
return;
}
};
match &mut item.music_metadata {
Some(mm) if mm.vorbis_comment_length > 0 => {
let vc_data_offset = mm.vorbis_comment_offset;
let vc_hdr_offset = vc_data_offset - 4;
if write_start <= vc_hdr_offset && write_end >= vc_data_offset {
let hdr_from = (vc_hdr_offset - write_start) as usize;
let new_length = u32::from_be_bytes([
0,
data[hdr_from + 1],
data[hdr_from + 2],
data[hdr_from + 3],
]) as u64;
let vc_data_end = vc_data_offset + new_length;
if write_end >= vc_data_end {
let from = (vc_data_offset - write_start) as usize;
let to = (vc_data_end - write_start) as usize;
mm.update_from_vorbis_comment_data(&data[from..to]);
debug!(%ino, "write: vorbis comment tag update detected");
Some(mm.clone())
} else {
None
}
} else {
None
}
}
Some(mm)
if !mm.header.is_empty()
&& write_start == 0
&& data.len() >= 3
&& &data[0..3] == b"ID3" =>
{
mm.update_from_id3_data(data);
debug!(%ino, "write: ID3v2 tag update detected");
Some(mm.clone())
}
Some(mm) if mm.header.is_empty() && data.len() == 128 && &data[0..3] == b"TAG" => {
mm.update_from_id3v1_data(data);
debug!(%ino, "write: ID3v1 tag update detected");
Some(mm.clone())
}
_ => None,
}
};
if let Some(music_metadata) = updated_music_metadata {
let client = self.client.clone();
let ino_i64 = ino.0 as i64;
self.runtime_handle.block_on(async move {
if let Err(e) = save_music_metadata(ino_i64, &music_metadata, &client).await {
error!(ino = ino_i64, error = %e, "write: save_music_metadata failed");
} else {
debug!(ino = ino_i64, "write: persisted updated music metadata");
}
});
}
reply.written(written);
}
fn getattr(
&self,
_req: &fuser::Request,
ino: INodeNo,
_fh: Option<fuser::FileHandle>,
reply: fuser::ReplyAttr,
) {
trace!(%ino, "getattr");
match self.files.lock().unwrap().get(&ino) {
Some(file) => {
let ttl = Duration::new(1, 0);
let attr = file_to_attr(file);
reply.attr(&ttl, &attr);
}
None => {
debug!(%ino, "getattr: not found");
reply.error(Errno::ENOENT);
}
}
}
fn readdir(
&self,
_req: &fuser::Request,
ino: INodeNo,
fh: fuser::FileHandle,
offset: u64,
mut reply: fuser::ReplyDirectory,
) {
trace!(%ino, %fh, offset, "readdir");
let files = self.files.lock().unwrap();
let parent_inode = match files.get(&ino) {
Some(dir) => dir.parent_inode,
None => {
debug!(%ino, "readdir: not found");
reply.error(Errno::ENOENT);
return;
}
};
if offset < 1 {
if reply.add(ino, 1, fuser::FileType::Directory, ".") {
reply.ok();
return;
}
}
if offset < 2 {
if reply.add(parent_inode, 2, fuser::FileType::Directory, "..") {
reply.ok();
return;
}
}
let skip_count = if offset <= 2 {
0
} else {
(offset - 2) as usize
};
for (i, (key, value)) in files
.iter()
.filter(|(_, v)| v.parent_inode == ino && v.inode != ino)
.skip(skip_count)
.enumerate()
{
let entry_offset = (skip_count + i + 3) as u64;
let file_type = match value.file_type {
FileType::Directory => fuser::FileType::Directory,
FileType::File => fuser::FileType::RegularFile,
};
if reply.add(*key, entry_offset, file_type, &value.name) {
break;
}
}
reply.ok();
}
fn rename(
&self,
_req: &Request,
parent: INodeNo,
name: &std::ffi::OsStr,
newparent: INodeNo,
newname: &std::ffi::OsStr,
_flags: fuser::RenameFlags,
reply: fuser::ReplyEmpty,
) {
trace!(
%parent,
name = %name.display(),
%newparent,
newname = %newname.display(),
"rename"
);
let name_str = match name.to_str() {
Some(s) => s,
None => {
error!(%parent, "rename: source name is not valid UTF-8");
reply.error(Errno::EINVAL);
return;
}
};
let newname_str = match newname.to_str() {
Some(s) => s,
None => {
error!(%newparent, "rename: target name is not valid UTF-8");
reply.error(Errno::EINVAL);
return;
}
};
let mut files = self.files.lock().unwrap();
let item_ino = match files
.iter()
.find(|(_, v)| v.parent_inode == parent && v.name == name_str)
{
Some((ino, _)) => *ino,
None => {
debug!(%parent, name = %name_str, "rename: source not found");
reply.error(Errno::ENOENT);
return;
}
};
let new_local_path = if newparent == INodeNo::ROOT {
std::path::PathBuf::from(newname_str)
} else {
match files.get(&newparent) {
Some(dir) => dir.local_path.join(newname_str),
None => {
debug!(%newparent, "rename: target parent not found");
reply.error(Errno::ENOENT);
return;
}
}
};
debug!(ino = %item_ino, from = %name_str, to = %newname_str, "rename");
let new_parent_inode = parent_inode_from_path(&new_local_path);
let item = files.get_mut(&item_ino).unwrap();
item.name = newname_str.to_string();
item.local_path = new_local_path.clone();
item.parent_inode = new_parent_inode;
let inode_i64 = item_ino.0 as i64;
let new_name_owned = newname_str.to_string();
let new_local_path_str = new_local_path.to_string_lossy().into_owned();
let client = self.client.clone();
drop(files);
self.runtime_handle.block_on(async move {
use sea_orm::ActiveValue::Set;
if let Err(e) = (crate::db::entities::ActiveModel {
inode: Set(inode_i64),
name: Set(new_name_owned),
local_path: Set(new_local_path_str),
..Default::default()
}
.update(&client)
.await)
{
error!(ino = inode_i64, error = %e, "rename: db update failed");
}
});
reply.ok();
}
fn read(
&self,
_req: &Request,
ino: INodeNo,
_fh: fuser::FileHandle,
offset: u64,
size: u32,
_flags: fuser::OpenFlags,
_lock_owner: Option<fuser::LockOwner>,
reply: fuser::ReplyData,
) {
trace!(%ino, offset, size, "read");
let (inode, locator, virtual_layout) = {
let files = self.files.lock().unwrap();
let item = match files.get(&ino) {
Some(f) => f,
None => {
debug!(%ino, "read: not found");
reply.error(Errno::ENOENT);
return;
}
};
let inode = item.inode;
let locator = item.original_path.clone();
let virtual_layout = item
.music_metadata
.as_ref()
.filter(|mm| !mm.header.is_empty())
.map(|mm| {
(
mm.header.starts_with(b"ID3"),
mm.header.clone(),
mm.picture_block_headers.clone(),
mm.picture_data_ranges.clone(),
mm.real_audio_start,
)
});
(inode, locator, virtual_layout)
};
let Some((is_mp3, header, pic_hdrs, pic_ranges, real_audio_start)) = virtual_layout else {
match self.bytes.read_at(inode, &locator, offset, size as usize) {
Ok(bytes) => reply.data(&bytes),
Err(e) => {
error!(%inode, offset, size, error = %e, "read: read_at failed; returning EIO");
reply.error(Errno::EIO);
}
}
return;
};
let bytes = &self.bytes;
let reader = |off: u64, len: usize| bytes.read_at(inode, &locator, off, len);
let result = if is_mp3 {
file_io::assemble_mp3_read(
&reader,
&header,
&pic_hdrs,
&pic_ranges,
real_audio_start,
offset,
size,
)
} else {
file_io::assemble_flac_read(
&reader,
&header,
&pic_hdrs,
&pic_ranges,
real_audio_start,
offset,
size,
)
};
match result {
Ok(bytes) => reply.data(&bytes),
Err(e) => {
error!(%inode, offset, size, error = %e, "read: assembly failed; returning EIO");
reply.error(Errno::EIO);
}
}
}
fn lookup(
&self,
_req: &Request,
parent: INodeNo,
name: &std::ffi::OsStr,
reply: fuser::ReplyEntry,
) {
trace!(%parent, name = %name.display(), "lookup");
match self
.files
.lock()
.unwrap()
.iter()
.find(|item| item.1.parent_inode == parent && item.1.name == name.to_str().unwrap())
{
Some(item) => {
let ttl = Duration::new(1, 0);
let attr = file_to_attr(item.1);
reply.entry(&ttl, &attr, Generation(0));
}
None => {
debug!(%parent, name = %name.display(), "lookup: not found");
reply.error(Errno::ENOENT);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::item::FileType;
use crate::music::metadata::MusicMetadata;
use crate::origins::attrs::FileAttrs;
use fuser::INodeNo;
use std::os::unix::fs::MetadataExt;
use std::path::PathBuf;
#[test]
fn file_to_attr_uses_real_size_for_non_flac() {
let tmp = tempfile::tempdir().unwrap();
let metadata = std::fs::metadata(tmp.path()).unwrap();
let attrs = FileAttrs::from(&metadata);
let item = Item::new(
INodeNo(42),
INodeNo::ROOT,
"test".to_string(),
tmp.path().to_path_buf(),
PathBuf::from("test"),
FileType::File,
attrs.clone(),
None,
);
let attr = file_to_attr(&item);
assert_eq!(attr.size, metadata.size());
}
#[test]
fn file_to_attr_uses_virtual_size_for_flac() {
let tmp = tempfile::tempdir().unwrap();
let test_file = tmp.path().join("test.flac");
std::fs::write(&test_file, vec![0u8; 1000]).unwrap();
let metadata = std::fs::metadata(&test_file).unwrap();
let attrs = FileAttrs::from(&metadata);
let mm = MusicMetadata {
real_audio_start: 500,
header: vec![0u8; 100],
picture_data_ranges: vec![(0, 50)],
..MusicMetadata::default()
};
let item = Item::new(
INodeNo(43),
INodeNo::ROOT,
"test_flac".to_string(),
test_file.clone(),
PathBuf::from("test_flac"),
FileType::File,
attrs.clone(),
Some(mm.clone()),
);
let attr = file_to_attr(&item);
let expected_virtual_size = mm.virtual_size(attrs.size);
assert_eq!(attr.size, expected_virtual_size);
}
#[test]
fn file_to_attr_kind_directory() {
let tmp = tempfile::tempdir().unwrap();
let metadata = std::fs::metadata(tmp.path()).unwrap();
let item = Item::new(
INodeNo(44),
INodeNo::ROOT,
"test_dir".to_string(),
tmp.path().to_path_buf(),
PathBuf::from("test_dir"),
FileType::Directory,
FileAttrs::from(&metadata),
None,
);
let attr = file_to_attr(&item);
assert_eq!(attr.kind, fuser::FileType::Directory);
}
#[test]
fn file_to_attr_kind_file() {
let tmp = tempfile::tempdir().unwrap();
let metadata = std::fs::metadata(tmp.path()).unwrap();
let item = Item::new(
INodeNo(45),
INodeNo::ROOT,
"test_file".to_string(),
tmp.path().to_path_buf(),
PathBuf::from("test_file"),
FileType::File,
FileAttrs::from(&metadata),
None,
);
let attr = file_to_attr(&item);
assert_eq!(attr.kind, fuser::FileType::RegularFile);
}
}
@@ -0,0 +1,373 @@
pub mod transport;
pub mod watcher;
use std::{
collections::BTreeMap,
io,
path::{Path, PathBuf},
sync::{Arc, RwLock},
time::{Duration, SystemTime},
};
use fuser::INodeNo;
use sea_orm::EntityTrait;
use crate::db::cache::{delete_cached_bytes_for, get_cached_bytes, put_cached_bytes};
use crate::db::entities as item_entities;
use crate::db::sync::{run_db_blocking, sync_items_to_db};
use crate::item::{FileType, Item};
use crate::music::db::restore_music_metadata_from_db;
use crate::music::metadata::MusicMetadata;
use crate::origins::attrs::FileAttrs;
use crate::origins::{ByteSource, FileWatcher, Origin};
use crate::proto::ManifestEntry as ProtoManifestEntry;
use crate::virtual_dirs::{ensure_virtual_dirs, restore_virtual_paths};
use tracing::{debug, error, info, trace};
use self::transport::NetworkTransport;
pub struct NetworkOrigin {
pub(crate) endpoint: String,
pub(crate) destination: PathBuf,
/// Handle to the process-wide runtime (main's `#[tokio::main]`). Used by the
/// sync FUSE byte/watcher paths to drive async transport via `block_on`.
/// A Handle (not an owned Runtime) so dropping NetworkOrigin in async
/// context can't panic, and the runtime outlives any clones we hand out.
handle: tokio::runtime::Handle,
transport: NetworkTransport,
client: sea_orm::DatabaseConnection,
/// In-memory cache of the latest manifest, keyed by server id. Written by
/// `snapshot()`, read by the watcher to know what state to reconcile
/// against on the next pass without round-tripping through the DB.
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
}
impl std::fmt::Debug for NetworkOrigin {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return f
.debug_struct("NetworkOrigin")
.field("endpoint", &self.endpoint)
.field("destination", &self.destination)
.finish();
}
}
impl NetworkOrigin {
pub fn new(
endpoint: String,
destination: String,
client: sea_orm::DatabaseConnection,
) -> io::Result<Self> {
// Must be called from within the process runtime (main's #[tokio::main]).
let handle = tokio::runtime::Handle::current();
let transport = NetworkTransport::new(endpoint.clone()).map_err(io_err)?;
return Ok(NetworkOrigin {
endpoint,
destination: destination.into(),
handle,
transport,
client,
latest_manifest: Arc::new(RwLock::new(BTreeMap::new())),
});
}
pub fn runtime_handle(&self) -> tokio::runtime::Handle {
return self.handle.clone();
}
}
impl Origin for NetworkOrigin {
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
return run_db_blocking(self.snapshot_async()).map_err(io_err);
}
fn byte_source(&self) -> Arc<dyn ByteSource> {
return Arc::new(NetworkByteSource {
transport: self.transport.clone(),
runtime_handle: self.runtime_handle(),
client: self.client.clone(),
});
}
fn watcher(&self) -> Box<dyn FileWatcher> {
return Box::new(watcher::NetworkOriginFileWatcher::new(
self.transport.clone(),
self.runtime_handle(),
self.client.clone(),
self.destination.clone(),
self.latest_manifest.clone(),
));
}
}
impl NetworkOrigin {
/// Async snapshot driven directly on the caller's runtime. `main` (already
/// `#[tokio::main]`) awaits this; the sync `Origin::snapshot()` wrapper is
/// only for non-async callers and must not be invoked from within a runtime
/// (it builds and `block_on`s a throwaway one).
pub async fn snapshot_async(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
// 1. Pull client's current (inode, hash) pairs from the DB.
let client_entries: Vec<(u64, u64)> = item_entities::Entity::find()
.all(&self.client)
.await
.map_err(|e| {
error!(error = %e, "network snapshot: client entries DB read failed");
io_err(e)
})?
.into_iter()
.filter(|m| m.file_type == "file")
.map(|m| (m.inode as u64, m.hash as u64))
.collect();
// 2. Ask server what changed.
let response = self
.transport
.reconcile(client_entries)
.await
.map_err(|e| {
error!(error = %e, "network snapshot: reconcile with server failed");
io_err(e)
})?;
// 3. Invalidate cached bytes for changed + deleted inodes — they are
// stale by definition.
let mut changed_or_deleted: Vec<i64> =
response.changed.iter().map(|ih| ih.inode as i64).collect();
changed_or_deleted.extend(response.deleted.iter().map(|i| *i as i64));
delete_cached_bytes_for(&changed_or_deleted, &self.client).await;
// latest_manifest is in-memory; on restart it's empty so the reconcile
// delta misses unchanged files. Fetch full manifest then, delta otherwise.
let mut current_manifest = self.latest_manifest.read().unwrap().clone();
if current_manifest.is_empty() {
let entries = self.transport.get_manifest().await.map_err(|e| {
error!(error = %e, "network snapshot: get_manifest from server failed");
io_err(e)
})?;
current_manifest = entries.into_iter().map(|e| (e.id, e)).collect();
} else {
let wanted: Vec<u64> = response.changed.iter().map(|ih| ih.inode).collect();
if !wanted.is_empty() {
let wanted_count = wanted.len();
let entries = self.transport.get_metadata(wanted).await.map_err(|e| {
error!(
wanted = wanted_count,
error = %e,
"network snapshot: get_metadata from server failed"
);
io_err(e)
})?;
for entry in entries {
current_manifest.insert(entry.id, entry);
}
}
}
for inode in &response.deleted {
current_manifest.remove(inode);
}
*self.latest_manifest.write().unwrap() = current_manifest.clone();
let snapshot =
build_snapshot_from_manifest(&current_manifest, &self.destination, &self.client)
.await?;
info!(
changed = response.changed.len(),
deleted = response.deleted.len(),
files = snapshot.len(),
"network snapshot complete"
);
return Ok(snapshot);
}
}
/// Build a complete Items snapshot from the server manifest, sync it to the
/// DB, and restore music metadata + virtual paths from existing DB rows.
///
/// Shared between `snapshot_async` (initial mount) and the watcher's
/// `reconcile_once` (runtime updates) so both paths produce identical
/// snapshots and keep the DB in sync.
pub(crate) async fn build_snapshot_from_manifest(
manifest: &BTreeMap<u64, ProtoManifestEntry>,
destination: &Path,
client: &sea_orm::DatabaseConnection,
) -> io::Result<BTreeMap<INodeNo, Item>> {
let source_root = PathBuf::from("/");
let mut snapshot = BTreeMap::new();
let root_attrs = FileAttrs {
size: 0,
blocks: 0,
atime: SystemTime::UNIX_EPOCH,
mtime: SystemTime::UNIX_EPOCH,
ctime: SystemTime::UNIX_EPOCH,
crtime: SystemTime::UNIX_EPOCH,
perm: 0o755,
nlink: 2,
uid: 0,
gid: 0,
rdev: 0,
blksize: 4096,
};
snapshot.insert(
INodeNo::ROOT,
Item::new(
INodeNo::ROOT,
INodeNo::ROOT,
"/".to_string(),
destination.to_path_buf(),
destination.to_path_buf(),
FileType::Directory,
root_attrs,
None,
),
);
for (_id, entry) in manifest {
let item = manifest_entry_to_item(entry, &source_root, &mut snapshot);
snapshot.insert(item.inode, item);
}
let db_items: std::collections::HashMap<i64, item_entities::Model> =
item_entities::Entity::find()
.all(client)
.await
.map_err(|e| {
error!(error = %e, "build_snapshot_from_manifest: DB read failed");
io_err(e)
})?
.into_iter()
.map(|e| (e.inode, e))
.collect();
sync_items_to_db(&snapshot, &db_items, client).await;
restore_music_metadata_from_db(&mut snapshot, &db_items, client).await;
restore_virtual_paths(&mut snapshot, &db_items, &source_root);
return Ok(snapshot);
}
fn manifest_entry_to_item(
entry: &ProtoManifestEntry,
source_root: &Path,
snapshot: &mut BTreeMap<INodeNo, Item>,
) -> Item {
let inode = INodeNo(entry.id);
let original_path = PathBuf::from(&entry.rel_path);
let attrs = FileAttrs {
size: entry.size,
blocks: 0,
atime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.mtime),
mtime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.mtime),
ctime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.ctime),
crtime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.crtime),
perm: 0o644,
nlink: 1,
uid: 0,
gid: 0,
rdev: 0,
blksize: 4096,
};
let music_metadata: Option<MusicMetadata> =
entry.music_metadata.clone().map(music_metadata_from_proto);
let mut local_path = PathBuf::new();
if let Some(mm) = &music_metadata {
let joined;
let artist_dir = match mm.album_artist.as_deref() {
Some(a) => a,
None => {
joined = mm.artist.join("-");
&joined
}
};
local_path.push(artist_dir);
local_path.push(&mm.album);
}
let name = Path::new(&entry.rel_path)
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| entry.rel_path.clone());
local_path.push(&name);
let parent_inode = ensure_virtual_dirs(&local_path, source_root, snapshot);
return Item::new(
inode,
parent_inode,
name,
original_path,
local_path,
FileType::File,
attrs,
music_metadata,
);
}
fn music_metadata_from_proto(mm: crate::proto::MusicMetadata) -> MusicMetadata {
MusicMetadata {
artist: mm.artist,
album_artist: mm.album_artist,
album: mm.album,
track_number: mm.track_number,
track_title: mm.track_title,
other_tags: mm.other_tags,
header: mm.header,
picture_block_headers: mm.picture_block_headers,
picture_data_ranges: mm
.picture_data_ranges
.into_iter()
.map(|p| (p.offset, p.length))
.collect(),
real_audio_start: mm.real_audio_start,
vorbis_comment_offset: mm.vorbis_comment_offset,
vorbis_comment_length: mm.vorbis_comment_length,
}
}
fn io_err<E: std::fmt::Display>(e: E) -> io::Error {
return io::Error::new(io::ErrorKind::Other, e.to_string());
}
pub struct NetworkByteSource {
transport: NetworkTransport,
runtime_handle: tokio::runtime::Handle,
client: sea_orm::DatabaseConnection,
}
impl ByteSource for NetworkByteSource {
fn read_at(
&self,
inode: INodeNo,
_locator: &Path,
offset: u64,
len: usize,
) -> io::Result<Vec<u8>> {
let inode_i64 = inode.0 as i64;
trace!(%inode, offset, len, "network read_at");
let cached = self
.runtime_handle
.block_on(get_cached_bytes(inode_i64, &self.client));
if let Some(data) = cached {
debug!(%inode, "read_at cache hit");
return slice_range(&data, offset, len);
}
debug!(%inode, "read_at cache miss; fetching from server");
let (data, _total_size) = self
.runtime_handle
.block_on(self.transport.fetch_file_range(inode.0, 0, 0))
.map_err(|e| {
error!(%inode, error = %e, "read_at: fetch_file_range failed");
io_err(e)
})?;
self.runtime_handle
.block_on(put_cached_bytes(inode_i64, data.clone(), &self.client));
return slice_range(&data, offset, len);
}
}
fn slice_range(data: &[u8], offset: u64, len: usize) -> io::Result<Vec<u8>> {
let start = (offset as usize).min(data.len());
let end = (start + len).min(data.len());
return Ok(data[start..end].to_vec());
}
@@ -0,0 +1,134 @@
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result, anyhow};
use tokio_stream::StreamExt;
use tonic::codec::Streaming;
use crate::proto::{
ChangeEvent, GetFileRequest, GetManifestRequest, GetMetadataRequest, InodeHash, ManifestEntry,
MusicFsClient, ReconcileRequest, ReconcileResponse, SubscribeEventsRequest,
};
/// Thin wrapper over the generated tonic client. Owns the connection and
/// exposes four operations matching the four RPCs NetworkOrigin needs.
///
/// All methods are async and must be polled on the runtime whose Handle was
/// passed in at construction (or a child of it). The sync FUSE path uses
/// `runtime_handle.block_on(...)` to enter this runtime.
#[derive(Clone)]
pub struct NetworkTransport {
client: Arc<tokio::sync::Mutex<MusicFsClient<tonic::transport::Channel>>>,
endpoint: tonic::transport::Endpoint,
}
impl NetworkTransport {
/// Build a transport that connects lazily. The first RPC establishes the
/// connection; reconnects happen automatically if the channel drops.
pub fn new(url: String) -> Result<Self> {
let endpoint: tonic::transport::Endpoint = url
.try_into()
.map_err(|e| anyhow!("invalid server URL: {e}"))?;
let endpoint = endpoint
.timeout(Duration::from_secs(3))
.connect_timeout(Duration::from_secs(5))
.http2_keep_alive_interval(Duration::from_secs(10))
.keep_alive_timeout(Duration::from_secs(5));
let channel = endpoint.connect_lazy();
let client = Arc::new(tokio::sync::Mutex::new(MusicFsClient::new(channel)));
return Ok(NetworkTransport { client, endpoint });
}
pub async fn reconcile(&self, entries: Vec<(u64, u64)>) -> Result<ReconcileResponse> {
let request = ReconcileRequest {
entries: entries
.into_iter()
.map(|(inode, hash)| InodeHash { inode, hash })
.collect(),
};
let mut client = self.client.lock().await;
let response = client
.reconcile(request)
.await
.context("Reconcile RPC failed")?;
return Ok(response.into_inner());
}
pub async fn get_metadata(&self, inodes: Vec<u64>) -> Result<Vec<ManifestEntry>> {
let request = GetMetadataRequest { inodes };
let mut client = self.client.lock().await;
let mut stream: Streaming<ManifestEntry> = client
.get_metadata(request)
.await
.context("GetMetadata RPC failed")?
.into_inner();
let mut out = Vec::new();
while let Some(entry) = stream.next().await {
out.push(entry.context("GetMetadata stream error")?);
}
return Ok(out);
}
pub async fn get_manifest(&self) -> Result<Vec<ManifestEntry>> {
let mut client = self.client.lock().await;
let mut stream: Streaming<ManifestEntry> = client
.get_manifest(GetManifestRequest {})
.await
.context("GetManifest RPC failed")?
.into_inner();
let mut out = Vec::new();
while let Some(entry) = stream.next().await {
out.push(entry.context("GetManifest stream error")?);
}
return Ok(out);
}
pub async fn fetch_file_range(
&self,
id: u64,
start: u64,
length: u64,
) -> Result<(Vec<u8>, u64)> {
let request = GetFileRequest { id, start, length };
let mut client = self.client.lock().await;
let mut stream = client
.get_file(request)
.await
.context("GetFile RPC failed")?
.into_inner();
let first = stream
.next()
.await
.ok_or_else(|| anyhow!("GetFile returned empty stream for id {id}"))?
.context("GetFile stream error")?;
let total_size = first.total_size;
let mut data = first.data;
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("GetFile stream error")?;
data.extend_from_slice(&chunk.data);
}
return Ok((data, total_size));
}
/// Take a fresh receiver on the SubscribeEvents stream. Each call opens a
/// new server-streaming RPC; the caller owns the lifetime.
pub async fn subscribe_events(&self) -> Result<Streaming<ChangeEvent>> {
let mut client = self.client.lock().await;
let stream = client
.subscribe_events(SubscribeEventsRequest {})
.await
.context("SubscribeEvents RPC failed")?
.into_inner();
return Ok(stream);
}
/// Reconnect — used by the watcher when the channel has gone bad.
#[allow(dead_code)]
pub async fn reconnect(&self) -> Result<()> {
let channel = self.endpoint.connect().await.context("reconnect failed")?;
let new_client = MusicFsClient::new(channel);
let mut guard = self.client.lock().await;
*guard = new_client;
return Ok(());
}
}
@@ -0,0 +1,185 @@
use std::{
collections::BTreeMap,
path::PathBuf,
sync::{Arc, RwLock},
thread,
time::Duration,
};
use fuser::INodeNo;
use sea_orm::{DatabaseConnection, EntityTrait};
use tokio::sync::Notify;
use tokio_stream::StreamExt;
use crate::item::Item;
use crate::origins::network::transport::NetworkTransport;
use crate::origins::{FileWatcher, WatcherHandle};
use crate::proto::ManifestEntry as ProtoManifestEntry;
use tracing::{info, warn};
const INITIAL_RETRY_DELAY: Duration = Duration::from_secs(1);
const MAX_RETRY_DELAY: Duration = Duration::from_secs(30);
pub struct NetworkOriginFileWatcher {
transport: NetworkTransport,
runtime_handle: tokio::runtime::Handle,
client: DatabaseConnection,
destination: PathBuf,
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
}
impl NetworkOriginFileWatcher {
#[allow(clippy::too_many_arguments)]
pub fn new(
transport: NetworkTransport,
runtime_handle: tokio::runtime::Handle,
client: DatabaseConnection,
destination: PathBuf,
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
) -> Self {
return NetworkOriginFileWatcher {
transport,
runtime_handle,
client,
destination,
latest_manifest,
};
}
}
impl FileWatcher for NetworkOriginFileWatcher {
fn watch(&self, files: Arc<std::sync::Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle {
let runtime_handle = self.runtime_handle.clone();
let shutdown = Arc::new(Notify::new());
let state = WatcherState {
transport: self.transport.clone(),
client: self.client.clone(),
destination: self.destination.clone(),
latest_manifest: self.latest_manifest.clone(),
files,
};
let loop_shutdown = shutdown.clone();
let join = thread::spawn(move || {
runtime_handle.block_on(state.run_loop(loop_shutdown));
});
return WatcherHandle::new(move || {
shutdown.notify_one();
let _ = join.join();
});
}
}
#[derive(Clone)]
struct WatcherState {
transport: NetworkTransport,
client: DatabaseConnection,
destination: PathBuf,
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
files: Arc<std::sync::Mutex<BTreeMap<INodeNo, Item>>>,
}
impl WatcherState {
async fn run_loop(self, shutdown: Arc<Notify>) {
let mut retry_delay = INITIAL_RETRY_DELAY;
loop {
let subscribed = tokio::select! {
biased;
_ = shutdown.notified() => return,
s = self.transport.subscribe_events() => s,
};
match subscribed {
Ok(mut stream) => {
info!("network watcher: subscribed to /events");
retry_delay = INITIAL_RETRY_DELAY;
loop {
let item = tokio::select! {
biased;
_ = shutdown.notified() => return,
item = stream.next() => item,
};
match item {
Some(Ok(_event)) => {
if let Err(e) = self.reconcile_once().await {
warn!(error = %e, "network watcher: reconcile after event failed");
}
}
Some(Err(e)) => {
warn!(error = %e, "network watcher: stream error; reconnecting");
break;
}
None => break,
}
}
}
Err(e) => {
warn!(error = %e, "network watcher: subscribe failed; will retry");
}
}
tokio::select! {
biased;
_ = shutdown.notified() => return,
_ = tokio::time::sleep(retry_delay) => {}
}
retry_delay = (retry_delay * 2).min(MAX_RETRY_DELAY);
if let Err(e) = self.reconcile_once().await {
warn!(error = %e, "network watcher: poll reconcile failed");
}
}
}
async fn reconcile_once(&self) -> anyhow::Result<()> {
let client_entries: Vec<(u64, u64)> = crate::db::entities::Entity::find()
.all(&self.client)
.await?
.into_iter()
.filter(|m| m.file_type == "file")
.map(|m| (m.inode as u64, m.hash as u64))
.collect();
let response = self.transport.reconcile(client_entries).await?;
let mut changed_or_deleted: Vec<i64> =
response.changed.iter().map(|ih| ih.inode as i64).collect();
changed_or_deleted.extend(response.deleted.iter().map(|i| *i as i64));
crate::db::cache::delete_cached_bytes_for(&changed_or_deleted, &self.client).await;
let wanted: Vec<u64> = response.changed.iter().map(|ih| ih.inode).collect();
let mut current_manifest = self.latest_manifest.read().unwrap().clone();
if !wanted.is_empty() {
let entries = self.transport.get_metadata(wanted).await?;
for entry in entries {
current_manifest.insert(entry.id, entry);
}
}
for inode in &response.deleted {
current_manifest.remove(inode);
}
*self.latest_manifest.write().unwrap() = current_manifest.clone();
let new_snapshot =
super::build_snapshot_from_manifest(&current_manifest, &self.destination, &self.client)
.await?;
{
let mut files = self.files.lock().unwrap();
files.retain(|ino, _| new_snapshot.contains_key(ino));
for (ino, new_item) in &new_snapshot {
match files.get(ino) {
Some(existing) if existing.hash == new_item.hash => {}
_ => {
files.insert(*ino, new_item.clone());
}
}
}
}
info!(
changed = response.changed.len(),
deleted = response.deleted.len(),
total = current_manifest.len(),
"network watcher: reconcile applied"
);
return Ok(());
}
}
+267
View File
@@ -0,0 +1,267 @@
use std::{
collections::BTreeMap,
fs,
hash::Hasher,
path::{Path, PathBuf},
};
use fuser::INodeNo;
use twox_hash::XxHash64;
use crate::db::entities::Model;
use crate::item::{FileType, Item};
use crate::origins::attrs::FileAttrs;
pub fn virtual_inode(path: &str) -> INodeNo {
let mut hasher = XxHash64::with_seed(5678);
hasher.write(path.as_bytes());
return INodeNo(hasher.finish() | (1u64 << 63));
}
pub fn parent_inode_from_path(local_path: &Path) -> INodeNo {
let components: Vec<_> = local_path
.components()
.filter(|c| matches!(c, std::path::Component::Normal(_)))
.collect();
if components.len() <= 1 {
return INodeNo::ROOT;
}
let mut parent_path = String::new();
for (i, comp) in components[..components.len() - 1].iter().enumerate() {
if i > 0 {
parent_path.push('/');
}
parent_path.push_str(comp.as_os_str().to_str().unwrap_or_default());
}
return virtual_inode(&parent_path);
}
pub fn ensure_virtual_dirs(
local_path: &Path,
source: &Path,
map: &mut BTreeMap<INodeNo, Item>,
) -> INodeNo {
let components: Vec<_> = local_path
.components()
.filter(|c| matches!(c, std::path::Component::Normal(_)))
.collect();
if components.len() <= 1 {
return INodeNo::ROOT;
}
let mut current_parent = INodeNo::ROOT;
let mut current_path = String::new();
for component in &components[..components.len() - 1] {
let comp_str = component.as_os_str().to_str().unwrap_or_default();
if !current_path.is_empty() {
current_path.push('/');
}
current_path.push_str(comp_str);
let virt_ino = virtual_inode(&current_path);
if !map.contains_key(&virt_ino) {
let virt_item = Item::new(
virt_ino,
current_parent,
comp_str.to_string(),
source.to_path_buf(),
PathBuf::from(&current_path),
FileType::Directory,
FileAttrs::from(&fs::metadata(source).unwrap()),
None,
);
map.insert(virt_ino, virt_item);
}
current_parent = virt_ino;
}
return current_parent;
}
pub fn restore_virtual_paths(
snapshot: &mut BTreeMap<INodeNo, Item>,
db_items: &std::collections::HashMap<i64, Model>,
source: &Path,
) {
let restorations: Vec<(INodeNo, String, PathBuf)> = db_items
.values()
.filter_map(|db_item| {
let ino = INodeNo(db_item.inode as u64);
if ino == INodeNo::ROOT {
return None;
}
snapshot
.get(&ino)
.filter(|item| item.hash as i64 == db_item.hash)
.map(|_| {
(
ino,
db_item.name.clone(),
PathBuf::from(&db_item.local_path),
)
})
})
.collect();
for (_, _, local_path) in &restorations {
ensure_virtual_dirs(local_path, source, snapshot);
}
for (ino, name, local_path) in restorations {
if let Some(item) = snapshot.get_mut(&ino) {
item.name = name;
item.parent_inode = parent_inode_from_path(&local_path);
item.local_path = local_path;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
#[test]
fn virtual_inode_deterministic() {
let ino1 = virtual_inode("foo");
let ino2 = virtual_inode("foo");
assert_eq!(ino1, ino2);
}
#[test]
fn virtual_inode_high_bit_set() {
let ino = virtual_inode("test");
assert_eq!(ino.0 & (1u64 << 63), 1u64 << 63);
}
#[test]
fn virtual_inode_different_inputs() {
let ino1 = virtual_inode("foo");
let ino2 = virtual_inode("bar");
assert_ne!(ino1, ino2);
}
#[test]
fn parent_inode_single_component() {
let ino = parent_inode_from_path(Path::new("file.txt"));
assert_eq!(ino, INodeNo::ROOT);
}
#[test]
fn parent_inode_multi_component() {
let ino = parent_inode_from_path(Path::new("a/b/c"));
let expected = virtual_inode("a/b");
assert_eq!(ino, expected);
}
#[test]
fn parent_inode_absolute_path_filtered() {
let ino = parent_inode_from_path(Path::new("/file"));
assert_eq!(ino, INodeNo::ROOT);
}
#[test]
fn ensure_virtual_dirs_single_component() {
let tmp = tempfile::tempdir().unwrap();
let source = tmp.path();
let mut map = BTreeMap::new();
let ino = ensure_virtual_dirs(Path::new("file.txt"), source, &mut map);
assert_eq!(ino, INodeNo::ROOT);
assert!(map.is_empty());
}
#[test]
fn ensure_virtual_dirs_creates_hierarchy() {
let tmp = tempfile::tempdir().unwrap();
let source = tmp.path();
let mut map = BTreeMap::new();
let ino = ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
let a_ino = virtual_inode("a");
let ab_ino = virtual_inode("a/b");
assert_eq!(ino, ab_ino);
assert_eq!(map.len(), 2);
assert!(map.contains_key(&a_ino));
assert!(map.contains_key(&ab_ino));
let a_item = &map[&a_ino];
assert_eq!(a_item.parent_inode, INodeNo::ROOT);
let ab_item = &map[&ab_ino];
assert_eq!(ab_item.parent_inode, a_ino);
}
#[test]
fn ensure_virtual_dirs_idempotent() {
let tmp = tempfile::tempdir().unwrap();
let source = tmp.path();
let mut map = BTreeMap::new();
ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
let size_after_first = map.len();
ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
let size_after_second = map.len();
assert_eq!(size_after_first, size_after_second);
}
#[test]
fn restore_virtual_paths_skips_root() {
let tmp = tempfile::tempdir().unwrap();
let source = tmp.path();
let mut snapshot = BTreeMap::new();
let real_ino = INodeNo(42);
let real_item = Item::new(
real_ino,
INodeNo::ROOT,
"real_file".to_string(),
source.to_path_buf(),
PathBuf::from("real_file"),
FileType::File,
FileAttrs::from(&fs::metadata(source).unwrap()),
None,
);
snapshot.insert(real_ino, real_item);
let mut db_items = HashMap::new();
db_items.insert(
1i64,
Model {
inode: 1,
hash: 0,
name: "root".to_string(),
original_path: "/tmp/test".to_string(),
local_path: "/tmp/test".to_string(),
file_type: "directory".to_string(),
},
);
db_items.insert(
42i64,
Model {
inode: 42,
hash: 0,
name: "real_file".to_string(),
original_path: "real_file".to_string(),
local_path: "real_file".to_string(),
file_type: "file".to_string(),
},
);
restore_virtual_paths(&mut snapshot, &db_items, source);
for item in snapshot.values() {
assert_ne!(item.name, "/");
assert_ne!(item.name, "tmp");
}
}
}
@@ -0,0 +1,83 @@
use std::io::Write;
use std::path::PathBuf;
use musicfs::origins::local::file_io::{assemble_flac_read, read_bytes_at};
fn file_reader(path: PathBuf) -> impl Fn(u64, usize) -> std::io::Result<Vec<u8>> {
return move |offset, len| read_bytes_at(&path, offset, len);
}
#[test]
fn assemble_flac_read_picture_header_region() {
let mut f = tempfile::NamedTempFile::new().unwrap();
let data: Vec<u8> = (0..100).collect();
f.write_all(&data).unwrap();
f.flush().unwrap();
let path = f.path().to_path_buf();
let header = b"ABCD";
let pic_hdr = vec![0x86u8, 0x00, 0x00, 0x05];
let pic_ranges = [(10u64, 5u64)];
let real_audio_start = 50u64;
let reader = file_reader(path);
let result = assemble_flac_read(
&reader,
header,
&[pic_hdr.clone()],
&pic_ranges,
real_audio_start,
4,
4,
)
.unwrap();
assert_eq!(result, pic_hdr);
}
#[test]
fn assemble_flac_read_picture_data_region() {
let mut f = tempfile::NamedTempFile::new().unwrap();
let data: Vec<u8> = (0..100).collect();
f.write_all(&data).unwrap();
f.flush().unwrap();
let path = f.path().to_path_buf();
let header = b"ABCD";
let pic_hdr = vec![0x86u8, 0x00, 0x00, 0x05];
let pic_ranges = [(10u64, 5u64)];
let real_audio_start = 50u64;
let reader = file_reader(path);
let result = assemble_flac_read(
&reader,
header,
&[pic_hdr.clone()],
&pic_ranges,
real_audio_start,
8,
5,
)
.unwrap();
assert_eq!(result, vec![10, 11, 12, 13, 14]);
}
#[test]
fn assemble_flac_read_full_virtual_layout() {
let mut f = tempfile::NamedTempFile::new().unwrap();
let data: Vec<u8> = (0..200).map(|i| i as u8).collect();
f.write_all(&data).unwrap();
f.flush().unwrap();
let path = f.path().to_path_buf();
let header = vec![0xAA; 8];
let real_audio_start = 100u64;
let reader = file_reader(path);
let result = assemble_flac_read(&reader, &header, &[], &[], real_audio_start, 0, 18).unwrap();
let mut expected = vec![0xAA; 8];
expected.extend_from_slice(&(100..110).map(|i| i as u8).collect::<Vec<u8>>());
assert_eq!(result, expected);
}
@@ -0,0 +1,109 @@
use std::fs;
use std::io::Write;
use std::sync::{Arc, Mutex};
use fuser::INodeNo;
use musicfs::item::FileType;
use musicfs::origins::local::snapshot::{build_snapshot, fill_fileset};
#[test]
fn build_snapshot_empty_dir() {
let source = tempfile::tempdir().unwrap();
let dest = tempfile::tempdir().unwrap();
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
assert_eq!(snapshot.len(), 1);
assert!(snapshot.contains_key(&INodeNo::ROOT));
let root = &snapshot[&INodeNo::ROOT];
assert_eq!(root.inode, INodeNo::ROOT);
assert_eq!(root.parent_inode, INodeNo::ROOT);
assert_eq!(root.file_type, FileType::Directory);
}
#[test]
fn build_snapshot_flat_files() {
let source = tempfile::tempdir().unwrap();
let dest = tempfile::tempdir().unwrap();
let mut f = fs::File::create(source.path().join("a.txt")).unwrap();
f.write_all(b"content a").unwrap();
let mut f = fs::File::create(source.path().join("b.txt")).unwrap();
f.write_all(b"content b").unwrap();
let mut f = fs::File::create(source.path().join("c.txt")).unwrap();
f.write_all(b"content c").unwrap();
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
assert_eq!(snapshot.len(), 4);
assert!(snapshot.contains_key(&INodeNo::ROOT));
let file_entries: Vec<_> = snapshot
.values()
.filter(|item| item.file_type == FileType::File)
.collect();
assert_eq!(file_entries.len(), 3);
for file_entry in file_entries {
assert_eq!(file_entry.parent_inode, INodeNo::ROOT);
}
}
#[test]
fn build_snapshot_nested_dirs() {
let source = tempfile::tempdir().unwrap();
let dest = tempfile::tempdir().unwrap();
fs::create_dir(source.path().join("subdir")).unwrap();
let nested_path = source.path().join("subdir").join("file.txt");
let mut f = fs::File::create(&nested_path).unwrap();
f.write_all(b"nested content").unwrap();
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
assert!(snapshot.contains_key(&INodeNo::ROOT));
let file_entry = snapshot
.values()
.find(|item| item.name == "file.txt" && item.file_type == FileType::File)
.expect("file.txt not found");
assert_eq!(file_entry.parent_inode, INodeNo::ROOT);
assert_eq!(file_entry.original_path, nested_path);
}
#[test]
fn fill_fileset_removes_deleted() {
let source = tempfile::tempdir().unwrap();
let dest = tempfile::tempdir().unwrap();
let mut f = fs::File::create(source.path().join("file1.txt")).unwrap();
f.write_all(b"content 1").unwrap();
let mut f = fs::File::create(source.path().join("file2.txt")).unwrap();
f.write_all(b"content 2").unwrap();
let initial_snapshot = build_snapshot(source.path(), dest.path()).unwrap();
let map = Arc::new(Mutex::new(initial_snapshot));
assert_eq!(map.lock().unwrap().len(), 3);
fs::remove_file(source.path().join("file1.txt")).unwrap();
fill_fileset(&map, source.path(), dest.path());
let final_map = map.lock().unwrap();
assert_eq!(final_map.len(), 2);
assert!(final_map.contains_key(&INodeNo::ROOT));
let remaining_files: Vec<_> = final_map
.values()
.filter(|item| item.file_type == FileType::File)
.collect();
assert_eq!(remaining_files.len(), 1);
assert_eq!(remaining_files[0].name, "file2.txt");
}