Refactor to split client from server
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
[package]
|
||||
name = "musicfs-client"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
default-run = "musicfs"
|
||||
|
||||
[lib]
|
||||
name = "musicfs"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "musicfs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
musicfs-core.workspace = true
|
||||
musicfs-proto.workspace = true
|
||||
fuser.workspace = true
|
||||
libc.workspace = true
|
||||
sea-orm.workspace = true
|
||||
tokio.workspace = true
|
||||
anyhow.workspace = true
|
||||
async-trait.workspace = true
|
||||
clap.workspace = true
|
||||
notify.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
tonic.workspace = true
|
||||
bytes.workspace = true
|
||||
http.workspace = true
|
||||
tracing.workspace = true
|
||||
log.workspace = true
|
||||
twox-hash.workspace = true
|
||||
chrono.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -0,0 +1,115 @@
|
||||
use sea_orm::{
|
||||
ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter,
|
||||
sea_query::OnConflict,
|
||||
};
|
||||
use tracing::{debug, error};
|
||||
|
||||
use crate::db::entities::cached_file_bytes::{ActiveModel, Column, Entity, Model};
|
||||
|
||||
/// Return the cached bytes for `inode` if present. `None` means "not cached";
|
||||
/// the caller should fetch from the server and call [`put_cached_bytes`].
|
||||
pub async fn get_cached_bytes(inode: i64, db: &DatabaseConnection) -> Option<Vec<u8>> {
|
||||
let row: Option<Model> = match Entity::find_by_id(inode).one(db).await {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
error!(%inode, error = %e, "get_cached_bytes: DB read failed");
|
||||
None
|
||||
}
|
||||
};
|
||||
return row.map(|m| m.data);
|
||||
}
|
||||
|
||||
/// Insert or replace the cached bytes for `inode`. Called after a successful
|
||||
/// `GetFile` round-trip so subsequent reads of the same range hit the cache.
|
||||
pub async fn put_cached_bytes(inode: i64, data: Vec<u8>, db: &DatabaseConnection) {
|
||||
let active = ActiveModel {
|
||||
inode: Set(inode),
|
||||
data: Set(data),
|
||||
fetched_at: Set(chrono::Utc::now()),
|
||||
};
|
||||
if let Err(e) = Entity::insert(active)
|
||||
.on_conflict(
|
||||
OnConflict::column(Column::Inode)
|
||||
.update_columns([Column::Data, Column::FetchedAt])
|
||||
.to_owned(),
|
||||
)
|
||||
.exec(db)
|
||||
.await
|
||||
{
|
||||
error!(%inode, error = %e, "put_cached_bytes: DB upsert failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove cache rows for the given inodes. Called by reconcile when a file's
|
||||
/// hash has changed (the cached bytes are now stale).
|
||||
pub async fn delete_cached_bytes_for(inodes: &[i64], db: &DatabaseConnection) {
|
||||
if inodes.is_empty() {
|
||||
return;
|
||||
}
|
||||
debug!(count = inodes.len(), "deleting cached bytes");
|
||||
if let Err(e) = Entity::delete_many()
|
||||
.filter(Column::Inode.is_in(inodes.to_vec()))
|
||||
.exec(db)
|
||||
.await
|
||||
{
|
||||
error!(error = %e, "delete_cached_bytes_for: DB delete failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use sea_orm::ConnectionTrait;
|
||||
|
||||
async fn connect() -> Option<DatabaseConnection> {
|
||||
let pg_host = std::env::var("PGHOST").ok()?;
|
||||
let url = format!("postgresql://fujin@localhost/musicfs?host={pg_host}");
|
||||
let db = sea_orm::Database::connect(&url).await.ok()?;
|
||||
db.execute_unprepared("SELECT 1 FROM cached_file_bytes LIMIT 0")
|
||||
.await
|
||||
.ok()?;
|
||||
Some(db)
|
||||
}
|
||||
|
||||
async fn setup_parent_item(db: &DatabaseConnection, inode: i64) {
|
||||
db.execute_unprepared(&format!(
|
||||
"INSERT INTO items (inode, name, original_path, local_path, file_type, hash) \
|
||||
VALUES ({inode}, 'test', '/test', '/test', 'file', 0) \
|
||||
ON CONFLICT (inode) DO NOTHING"
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn teardown(db: &DatabaseConnection, inode: i64) {
|
||||
let _ = db
|
||||
.execute_unprepared(&format!("DELETE FROM items WHERE inode = {inode}"))
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_round_trip_with_timestamptz() {
|
||||
let db = match connect().await {
|
||||
Some(db) => db,
|
||||
None => {
|
||||
eprintln!("skip: no database (PGHOST unset or unreachable)");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let inode = -999_999_i64;
|
||||
let test_data = b"regression-test-payload".to_vec();
|
||||
|
||||
setup_parent_item(&db, inode).await;
|
||||
put_cached_bytes(inode, test_data.clone(), &db).await;
|
||||
|
||||
let cached = get_cached_bytes(inode, &db).await;
|
||||
assert_eq!(
|
||||
cached,
|
||||
Some(test_data),
|
||||
"get_cached_bytes must decode fetched_at (TIMESTAMPTZ), not fail with type mismatch"
|
||||
);
|
||||
|
||||
teardown(&db, inode).await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
use crate::item::{FileType, Item};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "items")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
pub name: String,
|
||||
pub original_path: String,
|
||||
pub local_path: String,
|
||||
pub file_type: String,
|
||||
pub hash: i64,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
|
||||
impl From<&Item> for ActiveModel {
|
||||
fn from(item: &Item) -> Self {
|
||||
use sea_orm::ActiveValue::Set;
|
||||
ActiveModel {
|
||||
inode: Set(item.inode.0 as i64),
|
||||
name: Set(item.name.clone()),
|
||||
original_path: Set(item.original_path.to_string_lossy().into_owned()),
|
||||
local_path: Set(item.local_path.to_string_lossy().into_owned()),
|
||||
file_type: Set(match item.file_type {
|
||||
FileType::Directory => "directory".to_string(),
|
||||
FileType::File => "file".to_string(),
|
||||
}),
|
||||
hash: Set(item.hash as i64),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub mod cached_file_bytes {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "cached_file_bytes")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
#[sea_orm(column_type = "Blob")]
|
||||
pub data: Vec<u8>,
|
||||
pub fetched_at: DateTimeUtc,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
use fuser::INodeNo;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn attrs_for_tempdir() -> FileAttrs {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
return FileAttrs::from(&metadata);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_to_active_model_fields_match() {
|
||||
let item = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/original/path"),
|
||||
PathBuf::from("local/path"),
|
||||
FileType::Directory,
|
||||
attrs_for_tempdir(),
|
||||
None,
|
||||
);
|
||||
let am = ActiveModel::from(&item);
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.inode {
|
||||
assert_eq!(*v, 42i64);
|
||||
} else {
|
||||
panic!("inode field not set");
|
||||
}
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.name {
|
||||
assert_eq!(v, "test");
|
||||
} else {
|
||||
panic!("name field not set");
|
||||
}
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.original_path {
|
||||
assert_eq!(v, "/original/path");
|
||||
} else {
|
||||
panic!("original_path field not set");
|
||||
}
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.local_path {
|
||||
assert_eq!(v, "local/path");
|
||||
} else {
|
||||
panic!("local_path field not set");
|
||||
}
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.hash {
|
||||
assert_eq!(*v, item.hash as i64);
|
||||
} else {
|
||||
panic!("hash field not set");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_type_string_mapping() {
|
||||
let item_dir = Item::new(
|
||||
INodeNo(1),
|
||||
INodeNo::ROOT,
|
||||
"dir".to_string(),
|
||||
PathBuf::from("/original/dir"),
|
||||
PathBuf::from("local/dir"),
|
||||
FileType::Directory,
|
||||
attrs_for_tempdir(),
|
||||
None,
|
||||
);
|
||||
let am_dir = ActiveModel::from(&item_dir);
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am_dir.file_type {
|
||||
assert_eq!(v, "directory");
|
||||
} else {
|
||||
panic!("file_type field not set for directory");
|
||||
}
|
||||
|
||||
let item_file = Item::new(
|
||||
INodeNo(2),
|
||||
INodeNo::ROOT,
|
||||
"file".to_string(),
|
||||
PathBuf::from("/original/file"),
|
||||
PathBuf::from("local/file"),
|
||||
FileType::File,
|
||||
attrs_for_tempdir(),
|
||||
None,
|
||||
);
|
||||
let am_file = ActiveModel::from(&item_file);
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am_file.file_type {
|
||||
assert_eq!(v, "file");
|
||||
} else {
|
||||
panic!("file_type field not set for file");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod cache;
|
||||
pub mod entities;
|
||||
pub mod sync;
|
||||
@@ -0,0 +1,92 @@
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use tracing::{debug, error};
|
||||
|
||||
use crate::db::entities::{ActiveModel, Entity, Model};
|
||||
use crate::item::Item;
|
||||
use crate::music::db::save_music_metadata;
|
||||
|
||||
pub async fn sync_items_to_db(
|
||||
snapshot: &BTreeMap<INodeNo, Item>,
|
||||
db_items: &HashMap<i64, Model>,
|
||||
client: &sea_orm::DatabaseConnection,
|
||||
) {
|
||||
let mut to_insert: Vec<ActiveModel> = vec![];
|
||||
let mut to_update: Vec<ActiveModel> = vec![];
|
||||
let mut to_delete: Vec<i64> = vec![];
|
||||
let mut to_save_music: Vec<i64> = vec![];
|
||||
|
||||
for (ino, item) in snapshot {
|
||||
let ino_i64 = ino.0 as i64;
|
||||
match db_items.get(&ino_i64) {
|
||||
None => {
|
||||
to_insert.push(ActiveModel::from(item));
|
||||
if item.music_metadata.is_some() {
|
||||
to_save_music.push(ino_i64);
|
||||
}
|
||||
}
|
||||
Some(db_item) if db_item.hash != item.hash as i64 => {
|
||||
to_update.push(ActiveModel::from(item));
|
||||
if item.music_metadata.is_some() {
|
||||
to_save_music.push(ino_i64);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let fresh_inodes: HashSet<i64> = snapshot.keys().map(|i| i.0 as i64).collect();
|
||||
for ino in db_items.keys().filter(|i| !fresh_inodes.contains(i)) {
|
||||
to_delete.push(*ino);
|
||||
}
|
||||
|
||||
debug!(
|
||||
insert = to_insert.len(),
|
||||
update = to_update.len(),
|
||||
delete = to_delete.len(),
|
||||
save_music = to_save_music.len(),
|
||||
"sync_items_to_db"
|
||||
);
|
||||
|
||||
if !to_insert.is_empty() {
|
||||
if let Err(e) = Entity::insert_many(to_insert).exec(client).await {
|
||||
error!(error = %e, "sync: insert_many failed");
|
||||
}
|
||||
}
|
||||
for model in to_update {
|
||||
if let Err(e) = model.update(client).await {
|
||||
error!(error = %e, "sync: update failed");
|
||||
}
|
||||
}
|
||||
for ino in to_delete {
|
||||
if let Err(e) = Entity::delete_by_id(ino).exec(client).await {
|
||||
error!(%ino, error = %e, "sync: delete failed");
|
||||
}
|
||||
}
|
||||
|
||||
for ino_i64 in &to_save_music {
|
||||
let ino = INodeNo(*ino_i64 as u64);
|
||||
if let Some(music_metadata) = snapshot
|
||||
.get(&ino)
|
||||
.and_then(|item| item.music_metadata.as_ref())
|
||||
{
|
||||
if let Err(e) = save_music_metadata(*ino_i64, music_metadata, client).await {
|
||||
error!(ino = *ino_i64, error = %e, "sync: save_music_metadata failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run_db_blocking<F: std::future::Future>(future: F) -> F::Output {
|
||||
return tokio::runtime::Builder::new_current_thread()
|
||||
.enable_io()
|
||||
.enable_time()
|
||||
.build()
|
||||
.unwrap_or_else(|e| {
|
||||
error!(error = %e, "run_db_blocking: failed to build runtime");
|
||||
panic!("run_db_blocking: failed to build runtime: {e}");
|
||||
})
|
||||
.block_on(future);
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
|
||||
use fuser::INodeNo;
|
||||
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
|
||||
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
|
||||
pub enum FileType {
|
||||
Directory,
|
||||
File,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Item {
|
||||
pub inode: INodeNo,
|
||||
pub parent_inode: INodeNo,
|
||||
pub name: String,
|
||||
pub original_path: PathBuf,
|
||||
pub local_path: PathBuf,
|
||||
pub file_type: FileType,
|
||||
pub attrs: FileAttrs,
|
||||
pub music_metadata: Option<MusicMetadata>,
|
||||
pub hash: u64,
|
||||
}
|
||||
|
||||
fn secs_since_epoch(time: SystemTime) -> u64 {
|
||||
return time
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
}
|
||||
|
||||
impl Item {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
inode: INodeNo,
|
||||
parent_inode: INodeNo,
|
||||
name: String,
|
||||
original_path: PathBuf,
|
||||
local_path: PathBuf,
|
||||
file_type: FileType,
|
||||
attrs: FileAttrs,
|
||||
music_metadata: Option<MusicMetadata>,
|
||||
) -> Item {
|
||||
let mut item = Item {
|
||||
inode,
|
||||
parent_inode,
|
||||
name,
|
||||
original_path,
|
||||
local_path,
|
||||
file_type,
|
||||
attrs,
|
||||
music_metadata,
|
||||
hash: 0,
|
||||
};
|
||||
item.hash = item.compute_hash();
|
||||
|
||||
return item;
|
||||
}
|
||||
|
||||
pub fn compute_hash(&self) -> u64 {
|
||||
musicfs_core::compute_item_hash(
|
||||
self.inode.0,
|
||||
self.original_path.as_os_str().as_bytes(),
|
||||
secs_since_epoch(self.attrs.ctime),
|
||||
secs_since_epoch(self.attrs.mtime),
|
||||
secs_since_epoch(self.attrs.crtime),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
|
||||
#[test]
|
||||
fn compute_hash_deterministic() {
|
||||
let attrs = file_attrs_for_tempdir();
|
||||
|
||||
let item1 = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/some/path"),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs.clone(),
|
||||
None,
|
||||
);
|
||||
|
||||
let item2 = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/some/path"),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(item1.hash, item2.hash);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_hash_changes_on_path_change() {
|
||||
let attrs = file_attrs_for_tempdir();
|
||||
|
||||
let item1 = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/some/path"),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs.clone(),
|
||||
None,
|
||||
);
|
||||
|
||||
let item2 = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/different/path"),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_ne!(item1.hash, item2.hash);
|
||||
}
|
||||
|
||||
fn file_attrs_for_tempdir() -> FileAttrs {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
return FileAttrs::from(&metadata);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
pub mod db;
|
||||
pub mod item;
|
||||
pub mod music;
|
||||
pub mod origins;
|
||||
pub mod virtual_dirs;
|
||||
|
||||
pub use musicfs_core::logging;
|
||||
pub use musicfs_proto as proto;
|
||||
@@ -0,0 +1,151 @@
|
||||
use std::{collections::HashMap, path::Path, sync::Arc};
|
||||
|
||||
use clap::Parser;
|
||||
use fuser::INodeNo;
|
||||
use musicfs::db::entities::{Entity, Model};
|
||||
use musicfs::db::sync::sync_items_to_db;
|
||||
use musicfs::item::Item;
|
||||
use musicfs::logging::{LogConfig, init};
|
||||
use musicfs::music::db::restore_music_metadata_from_db;
|
||||
use musicfs::origins::local::LocalOrigin;
|
||||
use musicfs::origins::network::NetworkOrigin;
|
||||
use musicfs::origins::{FuseFs, Origin};
|
||||
use musicfs::virtual_dirs::restore_virtual_paths;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about, long_about = None)]
|
||||
struct Args {
|
||||
#[arg(short, long, required = true)]
|
||||
mountpoint: String,
|
||||
|
||||
/// Local directory path (→ LocalOrigin) OR `http://host:port` URL of a
|
||||
/// musicfs-server (→ NetworkOrigin).
|
||||
#[arg(short, long, required = true)]
|
||||
source: String,
|
||||
|
||||
#[arg(short, long, required = true)]
|
||||
database: String,
|
||||
|
||||
/// Directory for daily-rotated log files.
|
||||
#[arg(long, default_value = "./logs")]
|
||||
log_dir: std::path::PathBuf,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let args = Args::parse();
|
||||
|
||||
// Bind the guard for the whole process so the non-blocking file writer
|
||||
// flushes on exit. Initialized before anything else so even early
|
||||
// failures land in the log.
|
||||
let _guard = init(LogConfig {
|
||||
log_dir: args.log_dir.clone(),
|
||||
file_prefix: "musicfs".to_string(),
|
||||
max_files: 7,
|
||||
});
|
||||
|
||||
let mountpoint = args.mountpoint.clone();
|
||||
info!(
|
||||
mountpoint = %mountpoint,
|
||||
source = %args.source,
|
||||
database = %args.database,
|
||||
log_dir = %args.log_dir.display(),
|
||||
"musicfs starting"
|
||||
);
|
||||
|
||||
// sea-orm logs every statement at INFO by default, which floods normal
|
||||
// output; demote to DEBUG so queries stay hidden under RUST_LOG=info.
|
||||
let mut db_opts = sea_orm::ConnectOptions::new(args.database.clone());
|
||||
db_opts.sqlx_logging_level(log::LevelFilter::Debug);
|
||||
let db = sea_orm::Database::connect(db_opts)
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
error!(database = %args.database, error = %e, "database connect failed");
|
||||
panic!("database connect: {e}");
|
||||
});
|
||||
info!(database = %args.database, "database connected");
|
||||
|
||||
let (snapshot, byte_source, watcher) = if looks_like_url(&args.source) {
|
||||
debug!(source = %args.source, "using NetworkOrigin");
|
||||
let origin = NetworkOrigin::new(args.source.clone(), mountpoint.clone(), db.clone())
|
||||
.unwrap_or_else(|e| {
|
||||
error!(source = %args.source, error = %e, "network origin init failed");
|
||||
panic!("network origin init: {e}");
|
||||
});
|
||||
// Await directly on this runtime; the sync `snapshot()` wrapper would
|
||||
// try to build a nested runtime and panic (we're inside #[tokio::main]).
|
||||
let snapshot = origin.snapshot_async().await.unwrap_or_else(|e| {
|
||||
error!(source = %args.source, error = %e, "network initial snapshot failed");
|
||||
panic!("network initial snapshot: {e}");
|
||||
});
|
||||
info!(files = snapshot.len(), "network snapshot complete");
|
||||
(snapshot, origin.byte_source(), origin.watcher())
|
||||
} else {
|
||||
debug!(source = %args.source, "using LocalOrigin");
|
||||
let origin = LocalOrigin::new(args.source.clone(), mountpoint.clone());
|
||||
let mut snapshot = origin.snapshot().unwrap_or_else(|e| {
|
||||
error!(source = %args.source, error = %e, "local initial snapshot failed");
|
||||
panic!("local initial snapshot: {e}");
|
||||
});
|
||||
|
||||
let db_items: HashMap<i64, Model> = Entity::find()
|
||||
.all(&db)
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
error!(error = %e, "loading db items failed");
|
||||
panic!("loading db items: {e}");
|
||||
})
|
||||
.into_iter()
|
||||
.map(|e| (e.inode, e))
|
||||
.collect();
|
||||
sync_items_to_db(&snapshot, &db_items, &db).await;
|
||||
restore_music_metadata_from_db(&mut snapshot, &db_items, &db).await;
|
||||
restore_virtual_paths(&mut snapshot, &db_items, Path::new(&args.source));
|
||||
|
||||
info!(files = snapshot.len(), "local snapshot complete");
|
||||
(snapshot, origin.byte_source(), origin.watcher())
|
||||
};
|
||||
|
||||
let files: Arc<std::sync::Mutex<std::collections::BTreeMap<INodeNo, Item>>> =
|
||||
Arc::new(std::sync::Mutex::new(snapshot));
|
||||
let watcher_handle = watcher.watch(files.clone());
|
||||
|
||||
let fs = FuseFs {
|
||||
files,
|
||||
bytes: byte_source,
|
||||
client: db,
|
||||
runtime_handle: tokio::runtime::Handle::current(),
|
||||
};
|
||||
|
||||
let cfg = fuser::Config::default();
|
||||
let session = fuser::spawn_mount2(fs, &mountpoint, &cfg).unwrap_or_else(|e| {
|
||||
error!(mountpoint = %mountpoint, error = %e, "failed to mount FUSE filesystem");
|
||||
panic!("failed to mount FUSE filesystem: {e}");
|
||||
});
|
||||
info!(mountpoint = %mountpoint, "FUSE mounted");
|
||||
|
||||
let mut sigint = signal(SignalKind::interrupt()).unwrap_or_else(|e| {
|
||||
error!(error = %e, "failed to register SIGINT handler");
|
||||
panic!("register SIGINT handler: {e}");
|
||||
});
|
||||
let mut sigterm = signal(SignalKind::terminate()).unwrap_or_else(|e| {
|
||||
error!(error = %e, "failed to register SIGTERM handler");
|
||||
panic!("register SIGTERM handler: {e}");
|
||||
});
|
||||
|
||||
tokio::select! {
|
||||
_ = sigint.recv() => info!("received SIGINT, shutting down"),
|
||||
_ = sigterm.recv() => info!("received SIGTERM, shutting down"),
|
||||
}
|
||||
|
||||
info!(mountpoint = %mountpoint, "unmounting");
|
||||
watcher_handle.stop();
|
||||
drop(session);
|
||||
}
|
||||
|
||||
fn looks_like_url(s: &str) -> bool {
|
||||
return s.starts_with("http://") || s.starts_with("https://");
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
use crate::db::entities::Model;
|
||||
use crate::item::Item;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use tracing::{debug, error};
|
||||
|
||||
mod entities {
|
||||
pub mod music_metadata {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "music_metadata")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
pub track_title: String,
|
||||
pub album: String,
|
||||
pub track_number: i32,
|
||||
#[sea_orm(column_type = "Blob")]
|
||||
pub header: Vec<u8>,
|
||||
pub real_audio_start: i64,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
pub mod artists {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "music_metadata_artists")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub artist: String,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
pub mod other_tags {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "music_metadata_other_tags")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub position: i32,
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
pub mod pictures {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "music_metadata_pictures")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub position: i32,
|
||||
#[sea_orm(column_type = "Blob")]
|
||||
pub block_header: Vec<u8>,
|
||||
pub data_offset: i64,
|
||||
pub data_length: i64,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
}
|
||||
|
||||
use entities::{artists, music_metadata as music_metadata_entity, other_tags, pictures};
|
||||
|
||||
pub async fn save_music_metadata(
|
||||
inode: i64,
|
||||
music_metadata: &MusicMetadata,
|
||||
client: &sea_orm::DatabaseConnection,
|
||||
) -> Result<(), sea_orm::DbErr> {
|
||||
use sea_orm::ActiveValue::Set;
|
||||
|
||||
music_metadata_entity::Entity::delete_by_id(inode)
|
||||
.exec(client)
|
||||
.await?;
|
||||
|
||||
music_metadata_entity::Entity::insert(music_metadata_entity::ActiveModel {
|
||||
inode: Set(inode),
|
||||
track_title: Set(music_metadata.track_title.clone()),
|
||||
album: Set(music_metadata.album.clone()),
|
||||
track_number: Set(music_metadata.track_number),
|
||||
header: Set(music_metadata.header.clone()),
|
||||
real_audio_start: Set(music_metadata.real_audio_start as i64),
|
||||
})
|
||||
.exec(client)
|
||||
.await?;
|
||||
|
||||
if !music_metadata.artist.is_empty() {
|
||||
artists::Entity::insert_many(music_metadata.artist.iter().map(|a| artists::ActiveModel {
|
||||
inode: Set(inode),
|
||||
artist: Set(a.clone()),
|
||||
}))
|
||||
.exec(client)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if !music_metadata.other_tags.is_empty() {
|
||||
other_tags::Entity::insert_many(music_metadata.other_tags.iter().enumerate().map(
|
||||
|(pos, tag)| other_tags::ActiveModel {
|
||||
inode: Set(inode),
|
||||
position: Set(pos as i32),
|
||||
tag: Set(tag.clone()),
|
||||
},
|
||||
))
|
||||
.exec(client)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if !music_metadata.picture_block_headers.is_empty() {
|
||||
pictures::Entity::insert_many(
|
||||
music_metadata
|
||||
.picture_block_headers
|
||||
.iter()
|
||||
.zip(music_metadata.picture_data_ranges.iter())
|
||||
.enumerate()
|
||||
.map(|(pos, (hdr, (offset, len)))| pictures::ActiveModel {
|
||||
inode: Set(inode),
|
||||
position: Set(pos as i32),
|
||||
block_header: Set(hdr.clone()),
|
||||
data_offset: Set(*offset as i64),
|
||||
data_length: Set(*len as i64),
|
||||
}),
|
||||
)
|
||||
.exec(client)
|
||||
.await?;
|
||||
}
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub async fn restore_music_metadata_from_db(
|
||||
snapshot: &mut BTreeMap<INodeNo, Item>,
|
||||
db_items: &HashMap<i64, Model>,
|
||||
client: &sea_orm::DatabaseConnection,
|
||||
) {
|
||||
use artists::Column as ArtCol;
|
||||
use music_metadata_entity::Column as MmCol;
|
||||
use other_tags::Column as OtCol;
|
||||
use pictures::Column as PicCol;
|
||||
|
||||
let unchanged_music_inodes: Vec<i64> = db_items
|
||||
.values()
|
||||
.filter_map(|db_item| {
|
||||
let ino = INodeNo(db_item.inode as u64);
|
||||
snapshot
|
||||
.get(&ino)
|
||||
.filter(|item| item.hash as i64 == db_item.hash && item.music_metadata.is_some())
|
||||
.map(|_| db_item.inode)
|
||||
})
|
||||
.collect();
|
||||
|
||||
if unchanged_music_inodes.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mm_rows: HashMap<i64, music_metadata_entity::Model> =
|
||||
match music_metadata_entity::Entity::find()
|
||||
.filter(MmCol::Inode.is_in(unchanged_music_inodes.clone()))
|
||||
.all(client)
|
||||
.await
|
||||
{
|
||||
Ok(rows) => rows.into_iter().map(|m| (m.inode, m)).collect(),
|
||||
Err(e) => {
|
||||
error!(error = %e, "restore_music_metadata: mm rows query failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut artists_by_inode: HashMap<i64, Vec<String>> = HashMap::new();
|
||||
let artist_rows = match artists::Entity::find()
|
||||
.filter(ArtCol::Inode.is_in(unchanged_music_inodes.clone()))
|
||||
.all(client)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
error!(error = %e, "restore_music_metadata: artists query failed");
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
for row in artist_rows {
|
||||
artists_by_inode
|
||||
.entry(row.inode)
|
||||
.or_default()
|
||||
.push(row.artist);
|
||||
}
|
||||
|
||||
let mut other_tags_by_inode: HashMap<i64, Vec<(i32, String)>> = HashMap::new();
|
||||
let other_tag_rows = match other_tags::Entity::find()
|
||||
.filter(OtCol::Inode.is_in(unchanged_music_inodes.clone()))
|
||||
.all(client)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
error!(error = %e, "restore_music_metadata: other_tags query failed");
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
for row in other_tag_rows {
|
||||
other_tags_by_inode
|
||||
.entry(row.inode)
|
||||
.or_default()
|
||||
.push((row.position, row.tag));
|
||||
}
|
||||
|
||||
let mut pictures_by_inode: HashMap<i64, Vec<pictures::Model>> = HashMap::new();
|
||||
let picture_rows = match pictures::Entity::find()
|
||||
.filter(PicCol::Inode.is_in(unchanged_music_inodes))
|
||||
.all(client)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
error!(error = %e, "restore_music_metadata: pictures query failed");
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
for row in picture_rows {
|
||||
pictures_by_inode.entry(row.inode).or_default().push(row);
|
||||
}
|
||||
|
||||
debug!(restored = mm_rows.len(), "restored music metadata from db");
|
||||
|
||||
for (inode, mm_row) in mm_rows {
|
||||
let ino = INodeNo(inode as u64);
|
||||
let Some(item) = snapshot.get_mut(&ino) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let mut sorted_tags = other_tags_by_inode.remove(&inode).unwrap_or_default();
|
||||
sorted_tags.sort_by_key(|(pos, _)| *pos);
|
||||
|
||||
let mut sorted_pics = pictures_by_inode.remove(&inode).unwrap_or_default();
|
||||
sorted_pics.sort_by_key(|p| p.position);
|
||||
|
||||
let picture_block_headers: Vec<Vec<u8>> =
|
||||
sorted_pics.iter().map(|p| p.block_header.clone()).collect();
|
||||
|
||||
let picture_data_ranges: Vec<(u64, u64)> = sorted_pics
|
||||
.iter()
|
||||
.map(|p| (p.data_offset as u64, p.data_length as u64))
|
||||
.collect();
|
||||
|
||||
let mut music_metadata = MusicMetadata {
|
||||
artist: artists_by_inode.remove(&inode).unwrap_or_default(),
|
||||
album_artist: None,
|
||||
album: mm_row.album,
|
||||
track_number: mm_row.track_number,
|
||||
track_title: mm_row.track_title,
|
||||
other_tags: sorted_tags.into_iter().map(|(_, tag)| tag).collect(),
|
||||
header: mm_row.header,
|
||||
picture_block_headers,
|
||||
picture_data_ranges,
|
||||
real_audio_start: mm_row.real_audio_start as u64,
|
||||
vorbis_comment_offset: 0,
|
||||
vorbis_comment_length: 0,
|
||||
};
|
||||
music_metadata.find_vorbis_offsets();
|
||||
item.music_metadata = Some(music_metadata);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod db;
|
||||
pub use musicfs_core::music::{metadata, parse};
|
||||
@@ -0,0 +1,215 @@
|
||||
use std::{
|
||||
fs,
|
||||
io::{self, Read, Seek, SeekFrom},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
pub fn read_bytes_at(path: &Path, offset: u64, len: usize) -> io::Result<Vec<u8>> {
|
||||
let mut f = fs::File::open(path)?;
|
||||
f.seek(SeekFrom::Start(offset))?;
|
||||
let mut buf = vec![0u8; len];
|
||||
let n = f.read(&mut buf)?;
|
||||
buf.truncate(n);
|
||||
|
||||
return Ok(buf);
|
||||
}
|
||||
|
||||
fn assemble_virtual_read(
|
||||
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
|
||||
header: &[u8],
|
||||
pic_hdrs: &[Vec<u8>],
|
||||
pic_ranges: &[(u64, u64)],
|
||||
real_audio_start: u64,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let end = offset + size as u64;
|
||||
let header_end = header.len() as u64;
|
||||
|
||||
if end <= header_end {
|
||||
return Ok(header[offset as usize..end as usize].to_vec());
|
||||
}
|
||||
|
||||
let mut buf = Vec::with_capacity(size as usize);
|
||||
if offset < header_end {
|
||||
buf.extend_from_slice(&header[offset as usize..]);
|
||||
}
|
||||
|
||||
let mut virt_pos = header_end;
|
||||
for (pic_hdr, (data_real_offset, data_len)) in pic_hdrs.iter().zip(pic_ranges.iter()) {
|
||||
let pic_hdr_len = pic_hdr.len() as u64;
|
||||
let pic_hdr_end = virt_pos + pic_hdr_len;
|
||||
let pic_end = pic_hdr_end + data_len;
|
||||
|
||||
if end <= virt_pos {
|
||||
break;
|
||||
}
|
||||
if offset >= pic_end {
|
||||
virt_pos = pic_end;
|
||||
continue;
|
||||
}
|
||||
|
||||
let hdr_from = (offset.max(virt_pos) - virt_pos) as usize;
|
||||
let hdr_to = ((end.min(pic_hdr_end)) - virt_pos) as usize;
|
||||
if hdr_from < hdr_to {
|
||||
buf.extend_from_slice(&pic_hdr[hdr_from..hdr_to.min(pic_hdr.len())]);
|
||||
}
|
||||
|
||||
let data_start = offset.max(pic_hdr_end);
|
||||
let data_end = end.min(pic_end);
|
||||
if data_start < data_end {
|
||||
let real_off = data_real_offset + (data_start - pic_hdr_end);
|
||||
let bytes = reader(real_off, (data_end - data_start) as usize)?;
|
||||
buf.extend_from_slice(&bytes);
|
||||
}
|
||||
|
||||
virt_pos = pic_end;
|
||||
}
|
||||
|
||||
let audio_start = offset.max(virt_pos);
|
||||
if audio_start < end {
|
||||
let real_off = real_audio_start + (audio_start - virt_pos);
|
||||
let bytes = reader(real_off, (end - audio_start) as usize)?;
|
||||
buf.extend_from_slice(&bytes);
|
||||
}
|
||||
|
||||
return Ok(buf);
|
||||
}
|
||||
|
||||
pub fn assemble_flac_read(
|
||||
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
|
||||
header: &[u8],
|
||||
pic_hdrs: &[Vec<u8>],
|
||||
pic_ranges: &[(u64, u64)],
|
||||
real_audio_start: u64,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
assemble_virtual_read(
|
||||
reader,
|
||||
header,
|
||||
pic_hdrs,
|
||||
pic_ranges,
|
||||
real_audio_start,
|
||||
offset,
|
||||
size,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn assemble_mp3_read(
|
||||
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
|
||||
header: &[u8],
|
||||
pic_hdrs: &[Vec<u8>],
|
||||
pic_ranges: &[(u64, u64)],
|
||||
real_audio_start: u64,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
assemble_virtual_read(
|
||||
reader,
|
||||
header,
|
||||
pic_hdrs,
|
||||
pic_ranges,
|
||||
real_audio_start,
|
||||
offset,
|
||||
size,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn file_reader(path: PathBuf) -> impl Fn(u64, usize) -> io::Result<Vec<u8>> {
|
||||
return move |offset, len| read_bytes_at(&path, offset, len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_bytes_at_full_file() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
f.write_all(b"hello world").unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path();
|
||||
|
||||
let result = read_bytes_at(path, 0, 11).unwrap();
|
||||
assert_eq!(result, b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_bytes_at_offset() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
f.write_all(b"abcdefghij").unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path();
|
||||
|
||||
let result = read_bytes_at(path, 3, 4).unwrap();
|
||||
assert_eq!(result, b"defg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_header_only() {
|
||||
let header = b"HEADERDATA";
|
||||
let noop = |_, _| Ok(vec![]);
|
||||
let result = assemble_flac_read(&noop, header, &[], &[], 0, 2, 4).unwrap();
|
||||
assert_eq!(result, b"ADER");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_audio_region() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let content: Vec<u8> = (0..20).collect();
|
||||
f.write_all(&content).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let reader = file_reader(path);
|
||||
let header = b"HDR";
|
||||
let result = assemble_flac_read(&reader, header, &[], &[], 10, 5, 4).unwrap();
|
||||
// offset=5, header_len=3, so we read from header[3..] (0 bytes) + audio at real_off=10+(5-3)=12
|
||||
// content[12..16] = [12, 13, 14, 15]
|
||||
assert_eq!(result, vec![12, 13, 14, 15]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_header_audio_boundary() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let content: Vec<u8> = (0..110).collect();
|
||||
f.write_all(&content).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let reader = file_reader(path);
|
||||
let header = b"ABCD";
|
||||
let result = assemble_flac_read(&reader, header, &[], &[], 100, 2, 6).unwrap();
|
||||
// offset=2, size=6, header_len=4
|
||||
// First 2 bytes from header[2..4] = "CD"
|
||||
// Remaining 4 bytes from audio at real_off=100+(2+2-4)=100
|
||||
// content[100..104] = [100, 101, 102, 103]
|
||||
let expected = [b'C', b'D', 100, 101, 102, 103];
|
||||
assert_eq!(result, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_mp3_read_header_only() {
|
||||
let header = b"ID3\x04\x00DATAHERE";
|
||||
let noop = |_, _| Ok(vec![]);
|
||||
let result = assemble_mp3_read(&noop, header, &[], &[], 0, 4, 4).unwrap();
|
||||
assert_eq!(result, &header[4..8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_mp3_read_audio_region() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let content: Vec<u8> = (0..20).collect();
|
||||
f.write_all(&content).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let reader = file_reader(path);
|
||||
let header = b"ID3";
|
||||
let result = assemble_mp3_read(&reader, header, &[], &[], 10, 5, 4).unwrap();
|
||||
assert_eq!(result, vec![12, 13, 14, 15]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
pub mod file_io;
|
||||
pub mod snapshot;
|
||||
pub mod watcher;
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
io,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
|
||||
use crate::item::Item;
|
||||
use crate::origins::{ByteSource, FileWatcher, Origin};
|
||||
pub struct LocalOrigin {
|
||||
pub(crate) source: PathBuf,
|
||||
pub(crate) destination: PathBuf,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LocalOrigin {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return f
|
||||
.debug_struct("LocalOrigin")
|
||||
.field("source", &self.source)
|
||||
.field("destination", &self.destination)
|
||||
.finish();
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalOrigin {
|
||||
pub fn new(source: String, destination: String) -> LocalOrigin {
|
||||
return LocalOrigin {
|
||||
source: source.into(),
|
||||
destination: destination.into(),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn source(&self) -> &Path {
|
||||
return &self.source;
|
||||
}
|
||||
}
|
||||
|
||||
impl Origin for LocalOrigin {
|
||||
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
|
||||
return snapshot::build_snapshot(&self.source, &self.destination);
|
||||
}
|
||||
|
||||
fn byte_source(&self) -> Arc<dyn ByteSource> {
|
||||
return Arc::new(LocalByteSource);
|
||||
}
|
||||
|
||||
fn watcher(&self) -> Box<dyn FileWatcher> {
|
||||
return Box::new(watcher::LocalOriginFileWatcher::new(
|
||||
self.source.clone(),
|
||||
self.destination.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LocalByteSource;
|
||||
|
||||
impl ByteSource for LocalByteSource {
|
||||
fn read_at(
|
||||
&self,
|
||||
_inode: INodeNo,
|
||||
locator: &Path,
|
||||
offset: u64,
|
||||
len: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
return file_io::read_bytes_at(locator, offset, len);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs, io,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::music::parse::parse_music_metadata_for_path;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
use crate::virtual_dirs::ensure_virtual_dirs;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub fn fill_fileset(map: &Arc<Mutex<BTreeMap<INodeNo, Item>>>, source: &Path, destination: &Path) {
|
||||
match build_snapshot(source, destination) {
|
||||
Ok(new_snapshot) => {
|
||||
let count = new_snapshot.len();
|
||||
let mut files = map.lock().unwrap();
|
||||
|
||||
files.retain(|ino, _| new_snapshot.contains_key(ino));
|
||||
|
||||
for (ino, new_item) in new_snapshot {
|
||||
match files.get(&ino) {
|
||||
Some(existing) if existing.hash == new_item.hash => {}
|
||||
_ => {
|
||||
files.insert(ino, new_item);
|
||||
}
|
||||
}
|
||||
}
|
||||
info!(count, "snapshot rebuilt");
|
||||
}
|
||||
Err(e) => error!(source = %source.display(), error = %e, "error while reading source"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_snapshot(
|
||||
source: &Path,
|
||||
destination: &Path,
|
||||
) -> Result<BTreeMap<INodeNo, Item>, io::Error> {
|
||||
let mut map = BTreeMap::new();
|
||||
|
||||
let local_root = Item::new(
|
||||
INodeNo::ROOT,
|
||||
INodeNo::ROOT,
|
||||
"/".to_string(),
|
||||
source.to_path_buf(),
|
||||
destination.to_path_buf(),
|
||||
FileType::Directory,
|
||||
FileAttrs::from(&fs::metadata(source)?),
|
||||
None,
|
||||
);
|
||||
map.insert(INodeNo::ROOT, local_root);
|
||||
|
||||
read_into_map(source, destination, &mut map)?;
|
||||
return Ok(map);
|
||||
}
|
||||
|
||||
pub fn read_into_map(
|
||||
source: &Path,
|
||||
destination: &Path,
|
||||
map: &mut BTreeMap<INodeNo, Item>,
|
||||
) -> Result<(), io::Error> {
|
||||
for item in fs::read_dir(source)? {
|
||||
let entry = item?;
|
||||
let item_path = entry.path();
|
||||
|
||||
if entry.file_type()?.is_dir() {
|
||||
read_into_map(&item_path, destination, map)?;
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
let metadata = entry.metadata()?;
|
||||
let music_metadata = parse_music_metadata_for_path(&item_path);
|
||||
|
||||
let mut local_path = PathBuf::new();
|
||||
if let Some(ref mm) = music_metadata {
|
||||
let joined;
|
||||
let artist_dir = match mm.album_artist.as_deref() {
|
||||
Some(a) => a,
|
||||
None => {
|
||||
joined = mm.artist.join("-");
|
||||
&joined
|
||||
}
|
||||
};
|
||||
local_path.push(artist_dir);
|
||||
local_path.push(&mm.album);
|
||||
}
|
||||
local_path.push(&name);
|
||||
|
||||
let inode = INodeNo(metadata.ino());
|
||||
let parent_inode = ensure_virtual_dirs(&local_path, source, map);
|
||||
let local_item = Item::new(
|
||||
inode,
|
||||
parent_inode,
|
||||
name,
|
||||
item_path,
|
||||
local_path,
|
||||
FileType::File,
|
||||
FileAttrs::from(&metadata),
|
||||
music_metadata,
|
||||
);
|
||||
|
||||
map.insert(inode, local_item);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Mutex, mpsc},
|
||||
thread,
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use notify::{Event, EventKind, RecursiveMode, Watcher};
|
||||
|
||||
use crate::item::Item;
|
||||
use crate::origins::{FileWatcher, WatcherHandle};
|
||||
use tracing::{debug, error, info, trace};
|
||||
|
||||
pub struct LocalOriginFileWatcher {
|
||||
source: PathBuf,
|
||||
destination: PathBuf,
|
||||
}
|
||||
|
||||
impl LocalOriginFileWatcher {
|
||||
pub fn new(source: PathBuf, destination: PathBuf) -> Self {
|
||||
return LocalOriginFileWatcher {
|
||||
source,
|
||||
destination,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
impl FileWatcher for LocalOriginFileWatcher {
|
||||
fn watch(&self, files: Arc<Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle {
|
||||
let source = self.source.clone();
|
||||
let destination = self.destination.clone();
|
||||
|
||||
info!(source = %source.display(), "starting file watcher");
|
||||
thread::spawn(move || {
|
||||
let (tx, rx): (
|
||||
mpsc::Sender<Result<Event, notify::Error>>,
|
||||
mpsc::Receiver<Result<Event, notify::Error>>,
|
||||
) = mpsc::channel();
|
||||
|
||||
let mut watcher: notify::INotifyWatcher = match notify::recommended_watcher(tx) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
error!(error = %e, "watcher: failed to create notify watcher");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = watcher.watch(&source, RecursiveMode::Recursive) {
|
||||
error!(source = %source.display(), error = %e, "watcher: failed to watch source");
|
||||
return;
|
||||
}
|
||||
for res in rx {
|
||||
match res {
|
||||
Ok(event) => {
|
||||
trace!(?event, "watcher event");
|
||||
|
||||
match event.kind {
|
||||
EventKind::Any => {
|
||||
debug!("watcher: EventKind::Any, ignoring");
|
||||
}
|
||||
EventKind::Access(_access_kind) => {
|
||||
debug!("watcher: item accessed");
|
||||
}
|
||||
EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) => {
|
||||
debug!("watcher: create/modify/remove; rebuilding snapshot");
|
||||
super::snapshot::fill_fileset(&files, &source, &destination);
|
||||
}
|
||||
EventKind::Other => {
|
||||
debug!("watcher: other event, ignoring");
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => error!(error = %e, "watcher: notify error"),
|
||||
}
|
||||
}
|
||||
});
|
||||
return WatcherHandle::detached();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,631 @@
|
||||
pub use musicfs_core::attrs;
|
||||
pub mod local;
|
||||
pub mod network;
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
io,
|
||||
path::Path,
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use fuser::{Errno, FileAttr, Filesystem, Generation, INodeNo, Request};
|
||||
use sea_orm::{ActiveModelTrait, DatabaseConnection};
|
||||
use tracing::{debug, error, trace};
|
||||
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::music::db::save_music_metadata;
|
||||
use crate::origins::local::file_io;
|
||||
use crate::virtual_dirs::parent_inode_from_path;
|
||||
|
||||
/// Transport-agnostic byte-range reader. `locator` is whatever string the
|
||||
/// origin treats as a file key: a filesystem path for `LocalOrigin`, a remote
|
||||
/// key for the future network origin. Both produce bytes at `(offset, len)`.
|
||||
///
|
||||
/// `inode` is the FUSE-level inode; NetworkOrigin uses it as the server-side
|
||||
/// file id (and as the Postgres cache key). LocalOrigin ignores it.
|
||||
pub trait ByteSource: Send + Sync {
|
||||
fn read_at(
|
||||
&self,
|
||||
inode: INodeNo,
|
||||
locator: &Path,
|
||||
offset: u64,
|
||||
len: usize,
|
||||
) -> io::Result<Vec<u8>>;
|
||||
}
|
||||
|
||||
pub trait FileWatcher: Send + Sync {
|
||||
fn watch(&self, files: Arc<Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle;
|
||||
}
|
||||
|
||||
/// Returned by [`FileWatcher::watch`] so the caller can stop a background
|
||||
/// watcher before the process (and its tokio runtime) shuts down. Without
|
||||
/// this, an async watcher blocked on a timer/stream panics when the runtime is
|
||||
/// torn down out from under it.
|
||||
pub struct WatcherHandle {
|
||||
stop: Option<Box<dyn FnOnce() + Send>>,
|
||||
}
|
||||
|
||||
impl WatcherHandle {
|
||||
pub fn new(stop: impl FnOnce() + Send + 'static) -> Self {
|
||||
return WatcherHandle {
|
||||
stop: Some(Box::new(stop)),
|
||||
};
|
||||
}
|
||||
|
||||
/// A watcher with no shutdown work — its thread exits on its own.
|
||||
pub fn detached() -> Self {
|
||||
return WatcherHandle { stop: None };
|
||||
}
|
||||
|
||||
/// Signal the watcher to stop and wait for it to finish.
|
||||
pub fn stop(mut self) {
|
||||
if let Some(stop) = self.stop.take() {
|
||||
stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Origin: Send + Sync {
|
||||
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>>;
|
||||
fn byte_source(&self) -> Arc<dyn ByteSource>;
|
||||
fn watcher(&self) -> Box<dyn FileWatcher>;
|
||||
}
|
||||
|
||||
pub struct FuseFs {
|
||||
pub files: Arc<Mutex<BTreeMap<INodeNo, Item>>>,
|
||||
pub bytes: Arc<dyn ByteSource>,
|
||||
pub client: DatabaseConnection,
|
||||
pub runtime_handle: tokio::runtime::Handle,
|
||||
}
|
||||
|
||||
pub(crate) fn file_to_attr(item: &Item) -> FileAttr {
|
||||
let attrs = &item.attrs;
|
||||
let kind = match item.file_type {
|
||||
FileType::Directory => fuser::FileType::Directory,
|
||||
FileType::File => fuser::FileType::RegularFile,
|
||||
};
|
||||
|
||||
let size = match &item.music_metadata {
|
||||
Some(mm) if !mm.header.is_empty() => mm.virtual_size(attrs.size),
|
||||
_ => attrs.size,
|
||||
};
|
||||
|
||||
return FileAttr {
|
||||
ino: item.inode,
|
||||
size,
|
||||
blocks: attrs.blocks,
|
||||
atime: attrs.atime,
|
||||
mtime: attrs.mtime,
|
||||
ctime: attrs.ctime,
|
||||
crtime: attrs.crtime,
|
||||
kind,
|
||||
perm: attrs.perm,
|
||||
nlink: attrs.nlink,
|
||||
uid: attrs.uid,
|
||||
gid: attrs.gid,
|
||||
rdev: attrs.rdev,
|
||||
blksize: attrs.blksize,
|
||||
flags: 0,
|
||||
};
|
||||
}
|
||||
|
||||
impl Filesystem for FuseFs {
|
||||
fn open(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_flags: fuser::OpenFlags,
|
||||
reply: fuser::ReplyOpen,
|
||||
) {
|
||||
trace!(%ino, "open");
|
||||
if self.files.lock().unwrap().contains_key(&ino) {
|
||||
reply.opened(fuser::FileHandle(ino.0), fuser::FopenFlags::empty());
|
||||
} else {
|
||||
debug!(%ino, "open: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
fn setattr(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_mode: Option<u32>,
|
||||
_uid: Option<u32>,
|
||||
_gid: Option<u32>,
|
||||
_size: Option<u64>,
|
||||
_atime: Option<fuser::TimeOrNow>,
|
||||
_mtime: Option<fuser::TimeOrNow>,
|
||||
_ctime: Option<std::time::SystemTime>,
|
||||
_fh: Option<fuser::FileHandle>,
|
||||
_crtime: Option<std::time::SystemTime>,
|
||||
_chgtime: Option<std::time::SystemTime>,
|
||||
_bkuptime: Option<std::time::SystemTime>,
|
||||
_flags: Option<fuser::BsdFileFlags>,
|
||||
reply: fuser::ReplyAttr,
|
||||
) {
|
||||
trace!(%ino, "setattr");
|
||||
match self.files.lock().unwrap().get(&ino) {
|
||||
Some(file) => reply.attr(&Duration::new(1, 0), &file_to_attr(file)),
|
||||
None => {
|
||||
debug!(%ino, "setattr: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_fh: fuser::FileHandle,
|
||||
offset: u64,
|
||||
data: &[u8],
|
||||
_write_flags: fuser::WriteFlags,
|
||||
_flags: fuser::OpenFlags,
|
||||
_lock_owner: Option<fuser::LockOwner>,
|
||||
reply: fuser::ReplyWrite,
|
||||
) {
|
||||
trace!(%ino, offset, len = data.len(), "write");
|
||||
let written = data.len() as u32;
|
||||
let write_start = offset;
|
||||
let write_end = write_start + data.len() as u64;
|
||||
|
||||
let updated_music_metadata = {
|
||||
let mut files = self.files.lock().unwrap();
|
||||
let item = match files.get_mut(&ino) {
|
||||
Some(item) => item,
|
||||
None => {
|
||||
debug!(%ino, "write: not found");
|
||||
reply.written(written);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match &mut item.music_metadata {
|
||||
Some(mm) if mm.vorbis_comment_length > 0 => {
|
||||
let vc_data_offset = mm.vorbis_comment_offset;
|
||||
let vc_hdr_offset = vc_data_offset - 4;
|
||||
if write_start <= vc_hdr_offset && write_end >= vc_data_offset {
|
||||
let hdr_from = (vc_hdr_offset - write_start) as usize;
|
||||
let new_length = u32::from_be_bytes([
|
||||
0,
|
||||
data[hdr_from + 1],
|
||||
data[hdr_from + 2],
|
||||
data[hdr_from + 3],
|
||||
]) as u64;
|
||||
let vc_data_end = vc_data_offset + new_length;
|
||||
if write_end >= vc_data_end {
|
||||
let from = (vc_data_offset - write_start) as usize;
|
||||
let to = (vc_data_end - write_start) as usize;
|
||||
mm.update_from_vorbis_comment_data(&data[from..to]);
|
||||
debug!(%ino, "write: vorbis comment tag update detected");
|
||||
Some(mm.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Some(mm)
|
||||
if !mm.header.is_empty()
|
||||
&& write_start == 0
|
||||
&& data.len() >= 3
|
||||
&& &data[0..3] == b"ID3" =>
|
||||
{
|
||||
mm.update_from_id3_data(data);
|
||||
debug!(%ino, "write: ID3v2 tag update detected");
|
||||
Some(mm.clone())
|
||||
}
|
||||
Some(mm) if mm.header.is_empty() && data.len() == 128 && &data[0..3] == b"TAG" => {
|
||||
mm.update_from_id3v1_data(data);
|
||||
debug!(%ino, "write: ID3v1 tag update detected");
|
||||
Some(mm.clone())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(music_metadata) = updated_music_metadata {
|
||||
let client = self.client.clone();
|
||||
let ino_i64 = ino.0 as i64;
|
||||
self.runtime_handle.block_on(async move {
|
||||
if let Err(e) = save_music_metadata(ino_i64, &music_metadata, &client).await {
|
||||
error!(ino = ino_i64, error = %e, "write: save_music_metadata failed");
|
||||
} else {
|
||||
debug!(ino = ino_i64, "write: persisted updated music metadata");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
reply.written(written);
|
||||
}
|
||||
|
||||
fn getattr(
|
||||
&self,
|
||||
_req: &fuser::Request,
|
||||
ino: INodeNo,
|
||||
_fh: Option<fuser::FileHandle>,
|
||||
reply: fuser::ReplyAttr,
|
||||
) {
|
||||
trace!(%ino, "getattr");
|
||||
match self.files.lock().unwrap().get(&ino) {
|
||||
Some(file) => {
|
||||
let ttl = Duration::new(1, 0);
|
||||
let attr = file_to_attr(file);
|
||||
reply.attr(&ttl, &attr);
|
||||
}
|
||||
None => {
|
||||
debug!(%ino, "getattr: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn readdir(
|
||||
&self,
|
||||
_req: &fuser::Request,
|
||||
ino: INodeNo,
|
||||
fh: fuser::FileHandle,
|
||||
offset: u64,
|
||||
mut reply: fuser::ReplyDirectory,
|
||||
) {
|
||||
trace!(%ino, %fh, offset, "readdir");
|
||||
|
||||
let files = self.files.lock().unwrap();
|
||||
|
||||
let parent_inode = match files.get(&ino) {
|
||||
Some(dir) => dir.parent_inode,
|
||||
None => {
|
||||
debug!(%ino, "readdir: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if offset < 1 {
|
||||
if reply.add(ino, 1, fuser::FileType::Directory, ".") {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if offset < 2 {
|
||||
if reply.add(parent_inode, 2, fuser::FileType::Directory, "..") {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let skip_count = if offset <= 2 {
|
||||
0
|
||||
} else {
|
||||
(offset - 2) as usize
|
||||
};
|
||||
|
||||
for (i, (key, value)) in files
|
||||
.iter()
|
||||
.filter(|(_, v)| v.parent_inode == ino && v.inode != ino)
|
||||
.skip(skip_count)
|
||||
.enumerate()
|
||||
{
|
||||
let entry_offset = (skip_count + i + 3) as u64;
|
||||
let file_type = match value.file_type {
|
||||
FileType::Directory => fuser::FileType::Directory,
|
||||
FileType::File => fuser::FileType::RegularFile,
|
||||
};
|
||||
if reply.add(*key, entry_offset, file_type, &value.name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn rename(
|
||||
&self,
|
||||
_req: &Request,
|
||||
parent: INodeNo,
|
||||
name: &std::ffi::OsStr,
|
||||
newparent: INodeNo,
|
||||
newname: &std::ffi::OsStr,
|
||||
_flags: fuser::RenameFlags,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
trace!(
|
||||
%parent,
|
||||
name = %name.display(),
|
||||
%newparent,
|
||||
newname = %newname.display(),
|
||||
"rename"
|
||||
);
|
||||
let name_str = match name.to_str() {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
error!(%parent, "rename: source name is not valid UTF-8");
|
||||
reply.error(Errno::EINVAL);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let newname_str = match newname.to_str() {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
error!(%newparent, "rename: target name is not valid UTF-8");
|
||||
reply.error(Errno::EINVAL);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut files = self.files.lock().unwrap();
|
||||
|
||||
let item_ino = match files
|
||||
.iter()
|
||||
.find(|(_, v)| v.parent_inode == parent && v.name == name_str)
|
||||
{
|
||||
Some((ino, _)) => *ino,
|
||||
None => {
|
||||
debug!(%parent, name = %name_str, "rename: source not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let new_local_path = if newparent == INodeNo::ROOT {
|
||||
std::path::PathBuf::from(newname_str)
|
||||
} else {
|
||||
match files.get(&newparent) {
|
||||
Some(dir) => dir.local_path.join(newname_str),
|
||||
None => {
|
||||
debug!(%newparent, "rename: target parent not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
debug!(ino = %item_ino, from = %name_str, to = %newname_str, "rename");
|
||||
|
||||
let new_parent_inode = parent_inode_from_path(&new_local_path);
|
||||
|
||||
let item = files.get_mut(&item_ino).unwrap();
|
||||
item.name = newname_str.to_string();
|
||||
item.local_path = new_local_path.clone();
|
||||
item.parent_inode = new_parent_inode;
|
||||
|
||||
let inode_i64 = item_ino.0 as i64;
|
||||
let new_name_owned = newname_str.to_string();
|
||||
let new_local_path_str = new_local_path.to_string_lossy().into_owned();
|
||||
let client = self.client.clone();
|
||||
|
||||
drop(files);
|
||||
|
||||
self.runtime_handle.block_on(async move {
|
||||
use sea_orm::ActiveValue::Set;
|
||||
if let Err(e) = (crate::db::entities::ActiveModel {
|
||||
inode: Set(inode_i64),
|
||||
name: Set(new_name_owned),
|
||||
local_path: Set(new_local_path_str),
|
||||
..Default::default()
|
||||
}
|
||||
.update(&client)
|
||||
.await)
|
||||
{
|
||||
error!(ino = inode_i64, error = %e, "rename: db update failed");
|
||||
}
|
||||
});
|
||||
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn read(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_fh: fuser::FileHandle,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
_flags: fuser::OpenFlags,
|
||||
_lock_owner: Option<fuser::LockOwner>,
|
||||
reply: fuser::ReplyData,
|
||||
) {
|
||||
trace!(%ino, offset, size, "read");
|
||||
let (inode, locator, virtual_layout) = {
|
||||
let files = self.files.lock().unwrap();
|
||||
let item = match files.get(&ino) {
|
||||
Some(f) => f,
|
||||
None => {
|
||||
debug!(%ino, "read: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let inode = item.inode;
|
||||
let locator = item.original_path.clone();
|
||||
let virtual_layout = item
|
||||
.music_metadata
|
||||
.as_ref()
|
||||
.filter(|mm| !mm.header.is_empty())
|
||||
.map(|mm| {
|
||||
(
|
||||
mm.header.starts_with(b"ID3"),
|
||||
mm.header.clone(),
|
||||
mm.picture_block_headers.clone(),
|
||||
mm.picture_data_ranges.clone(),
|
||||
mm.real_audio_start,
|
||||
)
|
||||
});
|
||||
(inode, locator, virtual_layout)
|
||||
};
|
||||
|
||||
let Some((is_mp3, header, pic_hdrs, pic_ranges, real_audio_start)) = virtual_layout else {
|
||||
match self.bytes.read_at(inode, &locator, offset, size as usize) {
|
||||
Ok(bytes) => reply.data(&bytes),
|
||||
Err(e) => {
|
||||
error!(%inode, offset, size, error = %e, "read: read_at failed; returning EIO");
|
||||
reply.error(Errno::EIO);
|
||||
}
|
||||
}
|
||||
return;
|
||||
};
|
||||
|
||||
let bytes = &self.bytes;
|
||||
let reader = |off: u64, len: usize| bytes.read_at(inode, &locator, off, len);
|
||||
let result = if is_mp3 {
|
||||
file_io::assemble_mp3_read(
|
||||
&reader,
|
||||
&header,
|
||||
&pic_hdrs,
|
||||
&pic_ranges,
|
||||
real_audio_start,
|
||||
offset,
|
||||
size,
|
||||
)
|
||||
} else {
|
||||
file_io::assemble_flac_read(
|
||||
&reader,
|
||||
&header,
|
||||
&pic_hdrs,
|
||||
&pic_ranges,
|
||||
real_audio_start,
|
||||
offset,
|
||||
size,
|
||||
)
|
||||
};
|
||||
match result {
|
||||
Ok(bytes) => reply.data(&bytes),
|
||||
Err(e) => {
|
||||
error!(%inode, offset, size, error = %e, "read: assembly failed; returning EIO");
|
||||
reply.error(Errno::EIO);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn lookup(
|
||||
&self,
|
||||
_req: &Request,
|
||||
parent: INodeNo,
|
||||
name: &std::ffi::OsStr,
|
||||
reply: fuser::ReplyEntry,
|
||||
) {
|
||||
trace!(%parent, name = %name.display(), "lookup");
|
||||
|
||||
match self
|
||||
.files
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|item| item.1.parent_inode == parent && item.1.name == name.to_str().unwrap())
|
||||
{
|
||||
Some(item) => {
|
||||
let ttl = Duration::new(1, 0);
|
||||
let attr = file_to_attr(item.1);
|
||||
|
||||
reply.entry(&ttl, &attr, Generation(0));
|
||||
}
|
||||
None => {
|
||||
debug!(%parent, name = %name.display(), "lookup: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::item::FileType;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
use fuser::INodeNo;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn file_to_attr_uses_real_size_for_non_flac() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
let attrs = FileAttrs::from(&metadata);
|
||||
let item = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
tmp.path().to_path_buf(),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs.clone(),
|
||||
None,
|
||||
);
|
||||
|
||||
let attr = file_to_attr(&item);
|
||||
assert_eq!(attr.size, metadata.size());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_to_attr_uses_virtual_size_for_flac() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let test_file = tmp.path().join("test.flac");
|
||||
std::fs::write(&test_file, vec![0u8; 1000]).unwrap();
|
||||
let metadata = std::fs::metadata(&test_file).unwrap();
|
||||
let attrs = FileAttrs::from(&metadata);
|
||||
let mm = MusicMetadata {
|
||||
real_audio_start: 500,
|
||||
header: vec![0u8; 100],
|
||||
picture_data_ranges: vec![(0, 50)],
|
||||
..MusicMetadata::default()
|
||||
};
|
||||
let item = Item::new(
|
||||
INodeNo(43),
|
||||
INodeNo::ROOT,
|
||||
"test_flac".to_string(),
|
||||
test_file.clone(),
|
||||
PathBuf::from("test_flac"),
|
||||
FileType::File,
|
||||
attrs.clone(),
|
||||
Some(mm.clone()),
|
||||
);
|
||||
|
||||
let attr = file_to_attr(&item);
|
||||
let expected_virtual_size = mm.virtual_size(attrs.size);
|
||||
assert_eq!(attr.size, expected_virtual_size);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_to_attr_kind_directory() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
let item = Item::new(
|
||||
INodeNo(44),
|
||||
INodeNo::ROOT,
|
||||
"test_dir".to_string(),
|
||||
tmp.path().to_path_buf(),
|
||||
PathBuf::from("test_dir"),
|
||||
FileType::Directory,
|
||||
FileAttrs::from(&metadata),
|
||||
None,
|
||||
);
|
||||
|
||||
let attr = file_to_attr(&item);
|
||||
assert_eq!(attr.kind, fuser::FileType::Directory);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_to_attr_kind_file() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
let item = Item::new(
|
||||
INodeNo(45),
|
||||
INodeNo::ROOT,
|
||||
"test_file".to_string(),
|
||||
tmp.path().to_path_buf(),
|
||||
PathBuf::from("test_file"),
|
||||
FileType::File,
|
||||
FileAttrs::from(&metadata),
|
||||
None,
|
||||
);
|
||||
|
||||
let attr = file_to_attr(&item);
|
||||
assert_eq!(attr.kind, fuser::FileType::RegularFile);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
pub mod transport;
|
||||
pub mod watcher;
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
io,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, RwLock},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use sea_orm::EntityTrait;
|
||||
|
||||
use crate::db::cache::{delete_cached_bytes_for, get_cached_bytes, put_cached_bytes};
|
||||
use crate::db::entities as item_entities;
|
||||
use crate::db::sync::{run_db_blocking, sync_items_to_db};
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::music::db::restore_music_metadata_from_db;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
use crate::origins::{ByteSource, FileWatcher, Origin};
|
||||
use crate::proto::ManifestEntry as ProtoManifestEntry;
|
||||
use crate::virtual_dirs::{ensure_virtual_dirs, restore_virtual_paths};
|
||||
use tracing::{debug, error, info, trace};
|
||||
|
||||
use self::transport::NetworkTransport;
|
||||
|
||||
pub struct NetworkOrigin {
|
||||
pub(crate) endpoint: String,
|
||||
pub(crate) destination: PathBuf,
|
||||
/// Handle to the process-wide runtime (main's `#[tokio::main]`). Used by the
|
||||
/// sync FUSE byte/watcher paths to drive async transport via `block_on`.
|
||||
/// A Handle (not an owned Runtime) so dropping NetworkOrigin in async
|
||||
/// context can't panic, and the runtime outlives any clones we hand out.
|
||||
handle: tokio::runtime::Handle,
|
||||
transport: NetworkTransport,
|
||||
client: sea_orm::DatabaseConnection,
|
||||
/// In-memory cache of the latest manifest, keyed by server id. Written by
|
||||
/// `snapshot()`, read by the watcher to know what state to reconcile
|
||||
/// against on the next pass without round-tripping through the DB.
|
||||
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for NetworkOrigin {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return f
|
||||
.debug_struct("NetworkOrigin")
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("destination", &self.destination)
|
||||
.finish();
|
||||
}
|
||||
}
|
||||
|
||||
impl NetworkOrigin {
|
||||
pub fn new(
|
||||
endpoint: String,
|
||||
destination: String,
|
||||
client: sea_orm::DatabaseConnection,
|
||||
) -> io::Result<Self> {
|
||||
// Must be called from within the process runtime (main's #[tokio::main]).
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
let transport = NetworkTransport::new(endpoint.clone()).map_err(io_err)?;
|
||||
return Ok(NetworkOrigin {
|
||||
endpoint,
|
||||
destination: destination.into(),
|
||||
handle,
|
||||
transport,
|
||||
client,
|
||||
latest_manifest: Arc::new(RwLock::new(BTreeMap::new())),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn runtime_handle(&self) -> tokio::runtime::Handle {
|
||||
return self.handle.clone();
|
||||
}
|
||||
}
|
||||
|
||||
impl Origin for NetworkOrigin {
|
||||
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
|
||||
return run_db_blocking(self.snapshot_async()).map_err(io_err);
|
||||
}
|
||||
|
||||
fn byte_source(&self) -> Arc<dyn ByteSource> {
|
||||
return Arc::new(NetworkByteSource {
|
||||
transport: self.transport.clone(),
|
||||
runtime_handle: self.runtime_handle(),
|
||||
client: self.client.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
fn watcher(&self) -> Box<dyn FileWatcher> {
|
||||
return Box::new(watcher::NetworkOriginFileWatcher::new(
|
||||
self.transport.clone(),
|
||||
self.runtime_handle(),
|
||||
self.client.clone(),
|
||||
self.destination.clone(),
|
||||
self.latest_manifest.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
impl NetworkOrigin {
|
||||
/// Async snapshot driven directly on the caller's runtime. `main` (already
|
||||
/// `#[tokio::main]`) awaits this; the sync `Origin::snapshot()` wrapper is
|
||||
/// only for non-async callers and must not be invoked from within a runtime
|
||||
/// (it builds and `block_on`s a throwaway one).
|
||||
pub async fn snapshot_async(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
|
||||
// 1. Pull client's current (inode, hash) pairs from the DB.
|
||||
let client_entries: Vec<(u64, u64)> = item_entities::Entity::find()
|
||||
.all(&self.client)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "network snapshot: client entries DB read failed");
|
||||
io_err(e)
|
||||
})?
|
||||
.into_iter()
|
||||
.filter(|m| m.file_type == "file")
|
||||
.map(|m| (m.inode as u64, m.hash as u64))
|
||||
.collect();
|
||||
|
||||
// 2. Ask server what changed.
|
||||
let response = self
|
||||
.transport
|
||||
.reconcile(client_entries)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "network snapshot: reconcile with server failed");
|
||||
io_err(e)
|
||||
})?;
|
||||
|
||||
// 3. Invalidate cached bytes for changed + deleted inodes — they are
|
||||
// stale by definition.
|
||||
let mut changed_or_deleted: Vec<i64> =
|
||||
response.changed.iter().map(|ih| ih.inode as i64).collect();
|
||||
changed_or_deleted.extend(response.deleted.iter().map(|i| *i as i64));
|
||||
delete_cached_bytes_for(&changed_or_deleted, &self.client).await;
|
||||
|
||||
// latest_manifest is in-memory; on restart it's empty so the reconcile
|
||||
// delta misses unchanged files. Fetch full manifest then, delta otherwise.
|
||||
let mut current_manifest = self.latest_manifest.read().unwrap().clone();
|
||||
if current_manifest.is_empty() {
|
||||
let entries = self.transport.get_manifest().await.map_err(|e| {
|
||||
error!(error = %e, "network snapshot: get_manifest from server failed");
|
||||
io_err(e)
|
||||
})?;
|
||||
current_manifest = entries.into_iter().map(|e| (e.id, e)).collect();
|
||||
} else {
|
||||
let wanted: Vec<u64> = response.changed.iter().map(|ih| ih.inode).collect();
|
||||
if !wanted.is_empty() {
|
||||
let wanted_count = wanted.len();
|
||||
let entries = self.transport.get_metadata(wanted).await.map_err(|e| {
|
||||
error!(
|
||||
wanted = wanted_count,
|
||||
error = %e,
|
||||
"network snapshot: get_metadata from server failed"
|
||||
);
|
||||
io_err(e)
|
||||
})?;
|
||||
for entry in entries {
|
||||
current_manifest.insert(entry.id, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
for inode in &response.deleted {
|
||||
current_manifest.remove(inode);
|
||||
}
|
||||
*self.latest_manifest.write().unwrap() = current_manifest.clone();
|
||||
|
||||
let snapshot =
|
||||
build_snapshot_from_manifest(¤t_manifest, &self.destination, &self.client)
|
||||
.await?;
|
||||
|
||||
info!(
|
||||
changed = response.changed.len(),
|
||||
deleted = response.deleted.len(),
|
||||
files = snapshot.len(),
|
||||
"network snapshot complete"
|
||||
);
|
||||
return Ok(snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a complete Items snapshot from the server manifest, sync it to the
|
||||
/// DB, and restore music metadata + virtual paths from existing DB rows.
|
||||
///
|
||||
/// Shared between `snapshot_async` (initial mount) and the watcher's
|
||||
/// `reconcile_once` (runtime updates) so both paths produce identical
|
||||
/// snapshots and keep the DB in sync.
|
||||
pub(crate) async fn build_snapshot_from_manifest(
|
||||
manifest: &BTreeMap<u64, ProtoManifestEntry>,
|
||||
destination: &Path,
|
||||
client: &sea_orm::DatabaseConnection,
|
||||
) -> io::Result<BTreeMap<INodeNo, Item>> {
|
||||
let source_root = PathBuf::from("/");
|
||||
let mut snapshot = BTreeMap::new();
|
||||
let root_attrs = FileAttrs {
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
perm: 0o755,
|
||||
nlink: 2,
|
||||
uid: 0,
|
||||
gid: 0,
|
||||
rdev: 0,
|
||||
blksize: 4096,
|
||||
};
|
||||
snapshot.insert(
|
||||
INodeNo::ROOT,
|
||||
Item::new(
|
||||
INodeNo::ROOT,
|
||||
INodeNo::ROOT,
|
||||
"/".to_string(),
|
||||
destination.to_path_buf(),
|
||||
destination.to_path_buf(),
|
||||
FileType::Directory,
|
||||
root_attrs,
|
||||
None,
|
||||
),
|
||||
);
|
||||
|
||||
for (_id, entry) in manifest {
|
||||
let item = manifest_entry_to_item(entry, &source_root, &mut snapshot);
|
||||
snapshot.insert(item.inode, item);
|
||||
}
|
||||
|
||||
let db_items: std::collections::HashMap<i64, item_entities::Model> =
|
||||
item_entities::Entity::find()
|
||||
.all(client)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "build_snapshot_from_manifest: DB read failed");
|
||||
io_err(e)
|
||||
})?
|
||||
.into_iter()
|
||||
.map(|e| (e.inode, e))
|
||||
.collect();
|
||||
sync_items_to_db(&snapshot, &db_items, client).await;
|
||||
restore_music_metadata_from_db(&mut snapshot, &db_items, client).await;
|
||||
restore_virtual_paths(&mut snapshot, &db_items, &source_root);
|
||||
|
||||
return Ok(snapshot);
|
||||
}
|
||||
|
||||
fn manifest_entry_to_item(
|
||||
entry: &ProtoManifestEntry,
|
||||
source_root: &Path,
|
||||
snapshot: &mut BTreeMap<INodeNo, Item>,
|
||||
) -> Item {
|
||||
let inode = INodeNo(entry.id);
|
||||
let original_path = PathBuf::from(&entry.rel_path);
|
||||
let attrs = FileAttrs {
|
||||
size: entry.size,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.mtime),
|
||||
mtime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.mtime),
|
||||
ctime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.ctime),
|
||||
crtime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.crtime),
|
||||
perm: 0o644,
|
||||
nlink: 1,
|
||||
uid: 0,
|
||||
gid: 0,
|
||||
rdev: 0,
|
||||
blksize: 4096,
|
||||
};
|
||||
let music_metadata: Option<MusicMetadata> =
|
||||
entry.music_metadata.clone().map(music_metadata_from_proto);
|
||||
|
||||
let mut local_path = PathBuf::new();
|
||||
if let Some(mm) = &music_metadata {
|
||||
let joined;
|
||||
let artist_dir = match mm.album_artist.as_deref() {
|
||||
Some(a) => a,
|
||||
None => {
|
||||
joined = mm.artist.join("-");
|
||||
&joined
|
||||
}
|
||||
};
|
||||
local_path.push(artist_dir);
|
||||
local_path.push(&mm.album);
|
||||
}
|
||||
let name = Path::new(&entry.rel_path)
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| entry.rel_path.clone());
|
||||
local_path.push(&name);
|
||||
|
||||
let parent_inode = ensure_virtual_dirs(&local_path, source_root, snapshot);
|
||||
return Item::new(
|
||||
inode,
|
||||
parent_inode,
|
||||
name,
|
||||
original_path,
|
||||
local_path,
|
||||
FileType::File,
|
||||
attrs,
|
||||
music_metadata,
|
||||
);
|
||||
}
|
||||
|
||||
fn music_metadata_from_proto(mm: crate::proto::MusicMetadata) -> MusicMetadata {
|
||||
MusicMetadata {
|
||||
artist: mm.artist,
|
||||
album_artist: mm.album_artist,
|
||||
album: mm.album,
|
||||
track_number: mm.track_number,
|
||||
track_title: mm.track_title,
|
||||
other_tags: mm.other_tags,
|
||||
header: mm.header,
|
||||
picture_block_headers: mm.picture_block_headers,
|
||||
picture_data_ranges: mm
|
||||
.picture_data_ranges
|
||||
.into_iter()
|
||||
.map(|p| (p.offset, p.length))
|
||||
.collect(),
|
||||
real_audio_start: mm.real_audio_start,
|
||||
vorbis_comment_offset: mm.vorbis_comment_offset,
|
||||
vorbis_comment_length: mm.vorbis_comment_length,
|
||||
}
|
||||
}
|
||||
|
||||
fn io_err<E: std::fmt::Display>(e: E) -> io::Error {
|
||||
return io::Error::new(io::ErrorKind::Other, e.to_string());
|
||||
}
|
||||
|
||||
pub struct NetworkByteSource {
|
||||
transport: NetworkTransport,
|
||||
runtime_handle: tokio::runtime::Handle,
|
||||
client: sea_orm::DatabaseConnection,
|
||||
}
|
||||
|
||||
impl ByteSource for NetworkByteSource {
|
||||
fn read_at(
|
||||
&self,
|
||||
inode: INodeNo,
|
||||
_locator: &Path,
|
||||
offset: u64,
|
||||
len: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let inode_i64 = inode.0 as i64;
|
||||
trace!(%inode, offset, len, "network read_at");
|
||||
|
||||
let cached = self
|
||||
.runtime_handle
|
||||
.block_on(get_cached_bytes(inode_i64, &self.client));
|
||||
if let Some(data) = cached {
|
||||
debug!(%inode, "read_at cache hit");
|
||||
return slice_range(&data, offset, len);
|
||||
}
|
||||
|
||||
debug!(%inode, "read_at cache miss; fetching from server");
|
||||
let (data, _total_size) = self
|
||||
.runtime_handle
|
||||
.block_on(self.transport.fetch_file_range(inode.0, 0, 0))
|
||||
.map_err(|e| {
|
||||
error!(%inode, error = %e, "read_at: fetch_file_range failed");
|
||||
io_err(e)
|
||||
})?;
|
||||
self.runtime_handle
|
||||
.block_on(put_cached_bytes(inode_i64, data.clone(), &self.client));
|
||||
|
||||
return slice_range(&data, offset, len);
|
||||
}
|
||||
}
|
||||
|
||||
fn slice_range(data: &[u8], offset: u64, len: usize) -> io::Result<Vec<u8>> {
|
||||
let start = (offset as usize).min(data.len());
|
||||
let end = (start + len).min(data.len());
|
||||
return Ok(data[start..end].to_vec());
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use tokio_stream::StreamExt;
|
||||
use tonic::codec::Streaming;
|
||||
|
||||
use crate::proto::{
|
||||
ChangeEvent, GetFileRequest, GetManifestRequest, GetMetadataRequest, InodeHash, ManifestEntry,
|
||||
MusicFsClient, ReconcileRequest, ReconcileResponse, SubscribeEventsRequest,
|
||||
};
|
||||
|
||||
/// Thin wrapper over the generated tonic client. Owns the connection and
|
||||
/// exposes four operations matching the four RPCs NetworkOrigin needs.
|
||||
///
|
||||
/// All methods are async and must be polled on the runtime whose Handle was
|
||||
/// passed in at construction (or a child of it). The sync FUSE path uses
|
||||
/// `runtime_handle.block_on(...)` to enter this runtime.
|
||||
#[derive(Clone)]
|
||||
pub struct NetworkTransport {
|
||||
client: Arc<tokio::sync::Mutex<MusicFsClient<tonic::transport::Channel>>>,
|
||||
endpoint: tonic::transport::Endpoint,
|
||||
}
|
||||
|
||||
impl NetworkTransport {
|
||||
/// Build a transport that connects lazily. The first RPC establishes the
|
||||
/// connection; reconnects happen automatically if the channel drops.
|
||||
pub fn new(url: String) -> Result<Self> {
|
||||
let endpoint: tonic::transport::Endpoint = url
|
||||
.try_into()
|
||||
.map_err(|e| anyhow!("invalid server URL: {e}"))?;
|
||||
let endpoint = endpoint
|
||||
.timeout(Duration::from_secs(3))
|
||||
.connect_timeout(Duration::from_secs(5))
|
||||
.http2_keep_alive_interval(Duration::from_secs(10))
|
||||
.keep_alive_timeout(Duration::from_secs(5));
|
||||
let channel = endpoint.connect_lazy();
|
||||
let client = Arc::new(tokio::sync::Mutex::new(MusicFsClient::new(channel)));
|
||||
return Ok(NetworkTransport { client, endpoint });
|
||||
}
|
||||
|
||||
pub async fn reconcile(&self, entries: Vec<(u64, u64)>) -> Result<ReconcileResponse> {
|
||||
let request = ReconcileRequest {
|
||||
entries: entries
|
||||
.into_iter()
|
||||
.map(|(inode, hash)| InodeHash { inode, hash })
|
||||
.collect(),
|
||||
};
|
||||
let mut client = self.client.lock().await;
|
||||
let response = client
|
||||
.reconcile(request)
|
||||
.await
|
||||
.context("Reconcile RPC failed")?;
|
||||
return Ok(response.into_inner());
|
||||
}
|
||||
|
||||
pub async fn get_metadata(&self, inodes: Vec<u64>) -> Result<Vec<ManifestEntry>> {
|
||||
let request = GetMetadataRequest { inodes };
|
||||
let mut client = self.client.lock().await;
|
||||
let mut stream: Streaming<ManifestEntry> = client
|
||||
.get_metadata(request)
|
||||
.await
|
||||
.context("GetMetadata RPC failed")?
|
||||
.into_inner();
|
||||
let mut out = Vec::new();
|
||||
while let Some(entry) = stream.next().await {
|
||||
out.push(entry.context("GetMetadata stream error")?);
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
pub async fn get_manifest(&self) -> Result<Vec<ManifestEntry>> {
|
||||
let mut client = self.client.lock().await;
|
||||
let mut stream: Streaming<ManifestEntry> = client
|
||||
.get_manifest(GetManifestRequest {})
|
||||
.await
|
||||
.context("GetManifest RPC failed")?
|
||||
.into_inner();
|
||||
let mut out = Vec::new();
|
||||
while let Some(entry) = stream.next().await {
|
||||
out.push(entry.context("GetManifest stream error")?);
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
pub async fn fetch_file_range(
|
||||
&self,
|
||||
id: u64,
|
||||
start: u64,
|
||||
length: u64,
|
||||
) -> Result<(Vec<u8>, u64)> {
|
||||
let request = GetFileRequest { id, start, length };
|
||||
let mut client = self.client.lock().await;
|
||||
let mut stream = client
|
||||
.get_file(request)
|
||||
.await
|
||||
.context("GetFile RPC failed")?
|
||||
.into_inner();
|
||||
let first = stream
|
||||
.next()
|
||||
.await
|
||||
.ok_or_else(|| anyhow!("GetFile returned empty stream for id {id}"))?
|
||||
.context("GetFile stream error")?;
|
||||
let total_size = first.total_size;
|
||||
let mut data = first.data;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("GetFile stream error")?;
|
||||
data.extend_from_slice(&chunk.data);
|
||||
}
|
||||
return Ok((data, total_size));
|
||||
}
|
||||
|
||||
/// Take a fresh receiver on the SubscribeEvents stream. Each call opens a
|
||||
/// new server-streaming RPC; the caller owns the lifetime.
|
||||
pub async fn subscribe_events(&self) -> Result<Streaming<ChangeEvent>> {
|
||||
let mut client = self.client.lock().await;
|
||||
let stream = client
|
||||
.subscribe_events(SubscribeEventsRequest {})
|
||||
.await
|
||||
.context("SubscribeEvents RPC failed")?
|
||||
.into_inner();
|
||||
return Ok(stream);
|
||||
}
|
||||
|
||||
/// Reconnect — used by the watcher when the channel has gone bad.
|
||||
#[allow(dead_code)]
|
||||
pub async fn reconnect(&self) -> Result<()> {
|
||||
let channel = self.endpoint.connect().await.context("reconnect failed")?;
|
||||
let new_client = MusicFsClient::new(channel);
|
||||
let mut guard = self.client.lock().await;
|
||||
*guard = new_client;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
path::PathBuf,
|
||||
sync::{Arc, RwLock},
|
||||
thread,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use sea_orm::{DatabaseConnection, EntityTrait};
|
||||
use tokio::sync::Notify;
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
use crate::item::Item;
|
||||
use crate::origins::network::transport::NetworkTransport;
|
||||
use crate::origins::{FileWatcher, WatcherHandle};
|
||||
use crate::proto::ManifestEntry as ProtoManifestEntry;
|
||||
use tracing::{info, warn};
|
||||
|
||||
const INITIAL_RETRY_DELAY: Duration = Duration::from_secs(1);
|
||||
const MAX_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
pub struct NetworkOriginFileWatcher {
|
||||
transport: NetworkTransport,
|
||||
runtime_handle: tokio::runtime::Handle,
|
||||
client: DatabaseConnection,
|
||||
destination: PathBuf,
|
||||
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
|
||||
}
|
||||
|
||||
impl NetworkOriginFileWatcher {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
transport: NetworkTransport,
|
||||
runtime_handle: tokio::runtime::Handle,
|
||||
client: DatabaseConnection,
|
||||
destination: PathBuf,
|
||||
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
|
||||
) -> Self {
|
||||
return NetworkOriginFileWatcher {
|
||||
transport,
|
||||
runtime_handle,
|
||||
client,
|
||||
destination,
|
||||
latest_manifest,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
impl FileWatcher for NetworkOriginFileWatcher {
|
||||
fn watch(&self, files: Arc<std::sync::Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle {
|
||||
let runtime_handle = self.runtime_handle.clone();
|
||||
let shutdown = Arc::new(Notify::new());
|
||||
let state = WatcherState {
|
||||
transport: self.transport.clone(),
|
||||
client: self.client.clone(),
|
||||
destination: self.destination.clone(),
|
||||
latest_manifest: self.latest_manifest.clone(),
|
||||
files,
|
||||
};
|
||||
let loop_shutdown = shutdown.clone();
|
||||
let join = thread::spawn(move || {
|
||||
runtime_handle.block_on(state.run_loop(loop_shutdown));
|
||||
});
|
||||
return WatcherHandle::new(move || {
|
||||
shutdown.notify_one();
|
||||
let _ = join.join();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct WatcherState {
|
||||
transport: NetworkTransport,
|
||||
client: DatabaseConnection,
|
||||
destination: PathBuf,
|
||||
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
|
||||
files: Arc<std::sync::Mutex<BTreeMap<INodeNo, Item>>>,
|
||||
}
|
||||
|
||||
impl WatcherState {
|
||||
async fn run_loop(self, shutdown: Arc<Notify>) {
|
||||
let mut retry_delay = INITIAL_RETRY_DELAY;
|
||||
loop {
|
||||
let subscribed = tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.notified() => return,
|
||||
s = self.transport.subscribe_events() => s,
|
||||
};
|
||||
match subscribed {
|
||||
Ok(mut stream) => {
|
||||
info!("network watcher: subscribed to /events");
|
||||
retry_delay = INITIAL_RETRY_DELAY;
|
||||
loop {
|
||||
let item = tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.notified() => return,
|
||||
item = stream.next() => item,
|
||||
};
|
||||
match item {
|
||||
Some(Ok(_event)) => {
|
||||
if let Err(e) = self.reconcile_once().await {
|
||||
warn!(error = %e, "network watcher: reconcile after event failed");
|
||||
}
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
warn!(error = %e, "network watcher: stream error; reconnecting");
|
||||
break;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "network watcher: subscribe failed; will retry");
|
||||
}
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.notified() => return,
|
||||
_ = tokio::time::sleep(retry_delay) => {}
|
||||
}
|
||||
retry_delay = (retry_delay * 2).min(MAX_RETRY_DELAY);
|
||||
if let Err(e) = self.reconcile_once().await {
|
||||
warn!(error = %e, "network watcher: poll reconcile failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn reconcile_once(&self) -> anyhow::Result<()> {
|
||||
let client_entries: Vec<(u64, u64)> = crate::db::entities::Entity::find()
|
||||
.all(&self.client)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|m| m.file_type == "file")
|
||||
.map(|m| (m.inode as u64, m.hash as u64))
|
||||
.collect();
|
||||
|
||||
let response = self.transport.reconcile(client_entries).await?;
|
||||
|
||||
let mut changed_or_deleted: Vec<i64> =
|
||||
response.changed.iter().map(|ih| ih.inode as i64).collect();
|
||||
changed_or_deleted.extend(response.deleted.iter().map(|i| *i as i64));
|
||||
crate::db::cache::delete_cached_bytes_for(&changed_or_deleted, &self.client).await;
|
||||
|
||||
let wanted: Vec<u64> = response.changed.iter().map(|ih| ih.inode).collect();
|
||||
let mut current_manifest = self.latest_manifest.read().unwrap().clone();
|
||||
if !wanted.is_empty() {
|
||||
let entries = self.transport.get_metadata(wanted).await?;
|
||||
for entry in entries {
|
||||
current_manifest.insert(entry.id, entry);
|
||||
}
|
||||
}
|
||||
for inode in &response.deleted {
|
||||
current_manifest.remove(inode);
|
||||
}
|
||||
*self.latest_manifest.write().unwrap() = current_manifest.clone();
|
||||
|
||||
let new_snapshot =
|
||||
super::build_snapshot_from_manifest(¤t_manifest, &self.destination, &self.client)
|
||||
.await?;
|
||||
|
||||
{
|
||||
let mut files = self.files.lock().unwrap();
|
||||
files.retain(|ino, _| new_snapshot.contains_key(ino));
|
||||
for (ino, new_item) in &new_snapshot {
|
||||
match files.get(ino) {
|
||||
Some(existing) if existing.hash == new_item.hash => {}
|
||||
_ => {
|
||||
files.insert(*ino, new_item.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
changed = response.changed.len(),
|
||||
deleted = response.deleted.len(),
|
||||
total = current_manifest.len(),
|
||||
"network watcher: reconcile applied"
|
||||
);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs,
|
||||
hash::Hasher,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use twox_hash::XxHash64;
|
||||
|
||||
use crate::db::entities::Model;
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
|
||||
pub fn virtual_inode(path: &str) -> INodeNo {
|
||||
let mut hasher = XxHash64::with_seed(5678);
|
||||
hasher.write(path.as_bytes());
|
||||
|
||||
return INodeNo(hasher.finish() | (1u64 << 63));
|
||||
}
|
||||
|
||||
pub fn parent_inode_from_path(local_path: &Path) -> INodeNo {
|
||||
let components: Vec<_> = local_path
|
||||
.components()
|
||||
.filter(|c| matches!(c, std::path::Component::Normal(_)))
|
||||
.collect();
|
||||
if components.len() <= 1 {
|
||||
return INodeNo::ROOT;
|
||||
}
|
||||
let mut parent_path = String::new();
|
||||
for (i, comp) in components[..components.len() - 1].iter().enumerate() {
|
||||
if i > 0 {
|
||||
parent_path.push('/');
|
||||
}
|
||||
parent_path.push_str(comp.as_os_str().to_str().unwrap_or_default());
|
||||
}
|
||||
|
||||
return virtual_inode(&parent_path);
|
||||
}
|
||||
|
||||
pub fn ensure_virtual_dirs(
|
||||
local_path: &Path,
|
||||
source: &Path,
|
||||
map: &mut BTreeMap<INodeNo, Item>,
|
||||
) -> INodeNo {
|
||||
let components: Vec<_> = local_path
|
||||
.components()
|
||||
.filter(|c| matches!(c, std::path::Component::Normal(_)))
|
||||
.collect();
|
||||
|
||||
if components.len() <= 1 {
|
||||
return INodeNo::ROOT;
|
||||
}
|
||||
|
||||
let mut current_parent = INodeNo::ROOT;
|
||||
let mut current_path = String::new();
|
||||
|
||||
for component in &components[..components.len() - 1] {
|
||||
let comp_str = component.as_os_str().to_str().unwrap_or_default();
|
||||
if !current_path.is_empty() {
|
||||
current_path.push('/');
|
||||
}
|
||||
current_path.push_str(comp_str);
|
||||
|
||||
let virt_ino = virtual_inode(¤t_path);
|
||||
|
||||
if !map.contains_key(&virt_ino) {
|
||||
let virt_item = Item::new(
|
||||
virt_ino,
|
||||
current_parent,
|
||||
comp_str.to_string(),
|
||||
source.to_path_buf(),
|
||||
PathBuf::from(¤t_path),
|
||||
FileType::Directory,
|
||||
FileAttrs::from(&fs::metadata(source).unwrap()),
|
||||
None,
|
||||
);
|
||||
map.insert(virt_ino, virt_item);
|
||||
}
|
||||
|
||||
current_parent = virt_ino;
|
||||
}
|
||||
|
||||
return current_parent;
|
||||
}
|
||||
|
||||
pub fn restore_virtual_paths(
|
||||
snapshot: &mut BTreeMap<INodeNo, Item>,
|
||||
db_items: &std::collections::HashMap<i64, Model>,
|
||||
source: &Path,
|
||||
) {
|
||||
let restorations: Vec<(INodeNo, String, PathBuf)> = db_items
|
||||
.values()
|
||||
.filter_map(|db_item| {
|
||||
let ino = INodeNo(db_item.inode as u64);
|
||||
if ino == INodeNo::ROOT {
|
||||
return None;
|
||||
}
|
||||
snapshot
|
||||
.get(&ino)
|
||||
.filter(|item| item.hash as i64 == db_item.hash)
|
||||
.map(|_| {
|
||||
(
|
||||
ino,
|
||||
db_item.name.clone(),
|
||||
PathBuf::from(&db_item.local_path),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (_, _, local_path) in &restorations {
|
||||
ensure_virtual_dirs(local_path, source, snapshot);
|
||||
}
|
||||
for (ino, name, local_path) in restorations {
|
||||
if let Some(item) = snapshot.get_mut(&ino) {
|
||||
item.name = name;
|
||||
item.parent_inode = parent_inode_from_path(&local_path);
|
||||
item.local_path = local_path;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[test]
|
||||
fn virtual_inode_deterministic() {
|
||||
let ino1 = virtual_inode("foo");
|
||||
let ino2 = virtual_inode("foo");
|
||||
assert_eq!(ino1, ino2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn virtual_inode_high_bit_set() {
|
||||
let ino = virtual_inode("test");
|
||||
assert_eq!(ino.0 & (1u64 << 63), 1u64 << 63);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn virtual_inode_different_inputs() {
|
||||
let ino1 = virtual_inode("foo");
|
||||
let ino2 = virtual_inode("bar");
|
||||
assert_ne!(ino1, ino2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parent_inode_single_component() {
|
||||
let ino = parent_inode_from_path(Path::new("file.txt"));
|
||||
assert_eq!(ino, INodeNo::ROOT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parent_inode_multi_component() {
|
||||
let ino = parent_inode_from_path(Path::new("a/b/c"));
|
||||
let expected = virtual_inode("a/b");
|
||||
assert_eq!(ino, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parent_inode_absolute_path_filtered() {
|
||||
let ino = parent_inode_from_path(Path::new("/file"));
|
||||
assert_eq!(ino, INodeNo::ROOT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_virtual_dirs_single_component() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let source = tmp.path();
|
||||
let mut map = BTreeMap::new();
|
||||
|
||||
let ino = ensure_virtual_dirs(Path::new("file.txt"), source, &mut map);
|
||||
assert_eq!(ino, INodeNo::ROOT);
|
||||
assert!(map.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_virtual_dirs_creates_hierarchy() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let source = tmp.path();
|
||||
let mut map = BTreeMap::new();
|
||||
|
||||
let ino = ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
|
||||
|
||||
let a_ino = virtual_inode("a");
|
||||
let ab_ino = virtual_inode("a/b");
|
||||
|
||||
assert_eq!(ino, ab_ino);
|
||||
assert_eq!(map.len(), 2);
|
||||
assert!(map.contains_key(&a_ino));
|
||||
assert!(map.contains_key(&ab_ino));
|
||||
|
||||
let a_item = &map[&a_ino];
|
||||
assert_eq!(a_item.parent_inode, INodeNo::ROOT);
|
||||
|
||||
let ab_item = &map[&ab_ino];
|
||||
assert_eq!(ab_item.parent_inode, a_ino);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_virtual_dirs_idempotent() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let source = tmp.path();
|
||||
let mut map = BTreeMap::new();
|
||||
|
||||
ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
|
||||
let size_after_first = map.len();
|
||||
|
||||
ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
|
||||
let size_after_second = map.len();
|
||||
|
||||
assert_eq!(size_after_first, size_after_second);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_virtual_paths_skips_root() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let source = tmp.path();
|
||||
let mut snapshot = BTreeMap::new();
|
||||
|
||||
let real_ino = INodeNo(42);
|
||||
let real_item = Item::new(
|
||||
real_ino,
|
||||
INodeNo::ROOT,
|
||||
"real_file".to_string(),
|
||||
source.to_path_buf(),
|
||||
PathBuf::from("real_file"),
|
||||
FileType::File,
|
||||
FileAttrs::from(&fs::metadata(source).unwrap()),
|
||||
None,
|
||||
);
|
||||
snapshot.insert(real_ino, real_item);
|
||||
|
||||
let mut db_items = HashMap::new();
|
||||
db_items.insert(
|
||||
1i64,
|
||||
Model {
|
||||
inode: 1,
|
||||
hash: 0,
|
||||
name: "root".to_string(),
|
||||
original_path: "/tmp/test".to_string(),
|
||||
local_path: "/tmp/test".to_string(),
|
||||
file_type: "directory".to_string(),
|
||||
},
|
||||
);
|
||||
db_items.insert(
|
||||
42i64,
|
||||
Model {
|
||||
inode: 42,
|
||||
hash: 0,
|
||||
name: "real_file".to_string(),
|
||||
original_path: "real_file".to_string(),
|
||||
local_path: "real_file".to_string(),
|
||||
file_type: "file".to_string(),
|
||||
},
|
||||
);
|
||||
|
||||
restore_virtual_paths(&mut snapshot, &db_items, source);
|
||||
|
||||
for item in snapshot.values() {
|
||||
assert_ne!(item.name, "/");
|
||||
assert_ne!(item.name, "tmp");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use musicfs::origins::local::file_io::{assemble_flac_read, read_bytes_at};
|
||||
|
||||
fn file_reader(path: PathBuf) -> impl Fn(u64, usize) -> std::io::Result<Vec<u8>> {
|
||||
return move |offset, len| read_bytes_at(&path, offset, len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_picture_header_region() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let data: Vec<u8> = (0..100).collect();
|
||||
f.write_all(&data).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let header = b"ABCD";
|
||||
let pic_hdr = vec![0x86u8, 0x00, 0x00, 0x05];
|
||||
let pic_ranges = [(10u64, 5u64)];
|
||||
let real_audio_start = 50u64;
|
||||
|
||||
let reader = file_reader(path);
|
||||
let result = assemble_flac_read(
|
||||
&reader,
|
||||
header,
|
||||
&[pic_hdr.clone()],
|
||||
&pic_ranges,
|
||||
real_audio_start,
|
||||
4,
|
||||
4,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, pic_hdr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_picture_data_region() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let data: Vec<u8> = (0..100).collect();
|
||||
f.write_all(&data).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let header = b"ABCD";
|
||||
let pic_hdr = vec![0x86u8, 0x00, 0x00, 0x05];
|
||||
let pic_ranges = [(10u64, 5u64)];
|
||||
let real_audio_start = 50u64;
|
||||
|
||||
let reader = file_reader(path);
|
||||
let result = assemble_flac_read(
|
||||
&reader,
|
||||
header,
|
||||
&[pic_hdr.clone()],
|
||||
&pic_ranges,
|
||||
real_audio_start,
|
||||
8,
|
||||
5,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, vec![10, 11, 12, 13, 14]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_full_virtual_layout() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let data: Vec<u8> = (0..200).map(|i| i as u8).collect();
|
||||
f.write_all(&data).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let header = vec![0xAA; 8];
|
||||
let real_audio_start = 100u64;
|
||||
|
||||
let reader = file_reader(path);
|
||||
let result = assemble_flac_read(&reader, &header, &[], &[], real_audio_start, 0, 18).unwrap();
|
||||
|
||||
let mut expected = vec![0xAA; 8];
|
||||
expected.extend_from_slice(&(100..110).map(|i| i as u8).collect::<Vec<u8>>());
|
||||
assert_eq!(result, expected);
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use fuser::INodeNo;
|
||||
|
||||
use musicfs::item::FileType;
|
||||
use musicfs::origins::local::snapshot::{build_snapshot, fill_fileset};
|
||||
|
||||
#[test]
|
||||
fn build_snapshot_empty_dir() {
|
||||
let source = tempfile::tempdir().unwrap();
|
||||
let dest = tempfile::tempdir().unwrap();
|
||||
|
||||
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
|
||||
|
||||
assert_eq!(snapshot.len(), 1);
|
||||
assert!(snapshot.contains_key(&INodeNo::ROOT));
|
||||
|
||||
let root = &snapshot[&INodeNo::ROOT];
|
||||
assert_eq!(root.inode, INodeNo::ROOT);
|
||||
assert_eq!(root.parent_inode, INodeNo::ROOT);
|
||||
assert_eq!(root.file_type, FileType::Directory);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_snapshot_flat_files() {
|
||||
let source = tempfile::tempdir().unwrap();
|
||||
let dest = tempfile::tempdir().unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("a.txt")).unwrap();
|
||||
f.write_all(b"content a").unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("b.txt")).unwrap();
|
||||
f.write_all(b"content b").unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("c.txt")).unwrap();
|
||||
f.write_all(b"content c").unwrap();
|
||||
|
||||
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
|
||||
|
||||
assert_eq!(snapshot.len(), 4);
|
||||
assert!(snapshot.contains_key(&INodeNo::ROOT));
|
||||
|
||||
let file_entries: Vec<_> = snapshot
|
||||
.values()
|
||||
.filter(|item| item.file_type == FileType::File)
|
||||
.collect();
|
||||
assert_eq!(file_entries.len(), 3);
|
||||
|
||||
for file_entry in file_entries {
|
||||
assert_eq!(file_entry.parent_inode, INodeNo::ROOT);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_snapshot_nested_dirs() {
|
||||
let source = tempfile::tempdir().unwrap();
|
||||
let dest = tempfile::tempdir().unwrap();
|
||||
|
||||
fs::create_dir(source.path().join("subdir")).unwrap();
|
||||
let nested_path = source.path().join("subdir").join("file.txt");
|
||||
let mut f = fs::File::create(&nested_path).unwrap();
|
||||
f.write_all(b"nested content").unwrap();
|
||||
|
||||
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
|
||||
|
||||
assert!(snapshot.contains_key(&INodeNo::ROOT));
|
||||
|
||||
let file_entry = snapshot
|
||||
.values()
|
||||
.find(|item| item.name == "file.txt" && item.file_type == FileType::File)
|
||||
.expect("file.txt not found");
|
||||
|
||||
assert_eq!(file_entry.parent_inode, INodeNo::ROOT);
|
||||
assert_eq!(file_entry.original_path, nested_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fill_fileset_removes_deleted() {
|
||||
let source = tempfile::tempdir().unwrap();
|
||||
let dest = tempfile::tempdir().unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("file1.txt")).unwrap();
|
||||
f.write_all(b"content 1").unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("file2.txt")).unwrap();
|
||||
f.write_all(b"content 2").unwrap();
|
||||
|
||||
let initial_snapshot = build_snapshot(source.path(), dest.path()).unwrap();
|
||||
let map = Arc::new(Mutex::new(initial_snapshot));
|
||||
|
||||
assert_eq!(map.lock().unwrap().len(), 3);
|
||||
|
||||
fs::remove_file(source.path().join("file1.txt")).unwrap();
|
||||
|
||||
fill_fileset(&map, source.path(), dest.path());
|
||||
|
||||
let final_map = map.lock().unwrap();
|
||||
assert_eq!(final_map.len(), 2);
|
||||
assert!(final_map.contains_key(&INodeNo::ROOT));
|
||||
|
||||
let remaining_files: Vec<_> = final_map
|
||||
.values()
|
||||
.filter(|item| item.file_type == FileType::File)
|
||||
.collect();
|
||||
assert_eq!(remaining_files.len(), 1);
|
||||
assert_eq!(remaining_files[0].name, "file2.txt");
|
||||
}
|
||||
Reference in New Issue
Block a user