diff --git a/Cargo.lock b/Cargo.lock index c6b9e59..75295e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,7 +1700,7 @@ dependencies = [ ] [[package]] -name = "musicfs" +name = "musicfs-client" version = "0.1.0" dependencies = [ "anyhow", @@ -1712,24 +1712,58 @@ dependencies = [ "http", "libc", "log", + "musicfs-core", + "musicfs-proto", "notify", - "prost", "sea-orm", - "symphonia", "tempfile", - "time", "tokio", "tokio-stream", "tonic", - "tonic-health", - "tonic-prost", - "tonic-reflection", + "tracing", + "twox-hash", +] + +[[package]] +name = "musicfs-core" +version = "0.1.0" +dependencies = [ + "symphonia", + "tempfile", "tracing", "tracing-appender", "tracing-subscriber", "twox-hash", ] +[[package]] +name = "musicfs-proto" +version = "0.1.0" +dependencies = [ + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "musicfs-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "clap", + "musicfs-core", + "musicfs-proto", + "notify", + "tempfile", + "tokio", + "tokio-stream", + "tonic", + "tonic-health", + "tonic-reflection", + "tracing", +] + [[package]] name = "nix" version = "0.29.0" diff --git a/Cargo.toml b/Cargo.toml index 3846970..249ff21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,57 +1,51 @@ -[package] -name = "musicfs" +[workspace] +members = [ + "crates/musicfs-proto", + "crates/musicfs-core", + "crates/musicfs-server", + "crates/musicfs-client", +] +resolver = "2" + +[workspace.package] version = "0.1.0" edition = "2024" -# `cargo run` (and devenv's `cargo run -- ...`) picks the FUSE binary, not -# musicfs-server, when two [[bin]] targets exist. -default-run = "musicfs" -[lib] -name = "musicfs" -path = "src/lib.rs" +[workspace.dependencies] +musicfs-proto = { path = "crates/musicfs-proto" } +musicfs-core = { path = "crates/musicfs-core" } -[[bin]] -name = "musicfs" -path = "src/main.rs" +prost = "0.14" +tonic = "0.14" +tonic-prost = "0.14" +tonic-health = "0.14" +tonic-reflection = "0.14" -[[bin]] -name = "musicfs-server" -path = "src/bin/musicfs-server.rs" - -[dependencies] -clap = { version = "4.6.1", features = ["derive"] } -fuser = "0.17.0" -libc = "0.2.186" -notify = "8.2.0" -time = "0.3.49" symphonia = { version = "0.5", default-features = false, features = [ - "aac", "alac", "flac", "mp3", "ogg", "vorbis", "wav" + "aac", "alac", "flac", "mp3", "ogg", "vorbis", "wav", ] } twox-hash = "2.1.2" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +tracing-appender = "0.2.5" -sea-orm = { version = "2.0.0-rc", features = [ "sqlx-postgres", "runtime-tokio", "macros" ] } -tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal", "fs", "io-util", "sync"] } - +tokio = { version = "1", features = [ + "macros", "rt-multi-thread", "signal", "fs", "io-util", "sync", +] } anyhow = "1" async-trait = "0.1" -bytes = "1" -tonic = "0.14" -tonic-health = "0.14" -tonic-prost = "0.14" -tonic-reflection = "0.14" -prost = "0.14" +clap = { version = "4.6.1", features = ["derive"] } +notify = "8.2.0" tokio-stream = "0.1" + +fuser = "0.17.0" +libc = "0.2.186" +sea-orm = { version = "2.0.0-rc", features = [ + "sqlx-postgres", "runtime-tokio", "macros", +] } +log = "0.4" +bytes = "1" http = "1" chrono = { version = "0.4", default-features = false, features = ["clock"] } -tracing = "0.1" -# For sea_orm's ConnectOptions::sqlx_logging_level, which takes a log::LevelFilter. -log = "0.4" -tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } -# Pinned to 0.2.5: Builder::max_log_files (used for retention) landed in the -# 0.2.3 Builder API and is current here. An unqualified "0.2" could otherwise -# resolve to an older patch without the retention method. -tracing-appender = "0.2.5" - -[dev-dependencies] tempfile = "3" diff --git a/crates/musicfs-client/Cargo.toml b/crates/musicfs-client/Cargo.toml new file mode 100644 index 0000000..7c7aafb --- /dev/null +++ b/crates/musicfs-client/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "musicfs-client" +version.workspace = true +edition.workspace = true +default-run = "musicfs" + +[lib] +name = "musicfs" +path = "src/lib.rs" + +[[bin]] +name = "musicfs" +path = "src/main.rs" + +[dependencies] +musicfs-core.workspace = true +musicfs-proto.workspace = true +fuser.workspace = true +libc.workspace = true +sea-orm.workspace = true +tokio.workspace = true +anyhow.workspace = true +async-trait.workspace = true +clap.workspace = true +notify.workspace = true +tokio-stream.workspace = true +tonic.workspace = true +bytes.workspace = true +http.workspace = true +tracing.workspace = true +log.workspace = true +twox-hash.workspace = true +chrono.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/src/db/cache.rs b/crates/musicfs-client/src/db/cache.rs similarity index 100% rename from src/db/cache.rs rename to crates/musicfs-client/src/db/cache.rs diff --git a/src/db/entities.rs b/crates/musicfs-client/src/db/entities.rs similarity index 100% rename from src/db/entities.rs rename to crates/musicfs-client/src/db/entities.rs diff --git a/src/db/mod.rs b/crates/musicfs-client/src/db/mod.rs similarity index 100% rename from src/db/mod.rs rename to crates/musicfs-client/src/db/mod.rs diff --git a/src/db/sync.rs b/crates/musicfs-client/src/db/sync.rs similarity index 100% rename from src/db/sync.rs rename to crates/musicfs-client/src/db/sync.rs diff --git a/src/item.rs b/crates/musicfs-client/src/item.rs similarity index 81% rename from src/item.rs rename to crates/musicfs-client/src/item.rs index 281cbcb..e47d4a0 100644 --- a/src/item.rs +++ b/crates/musicfs-client/src/item.rs @@ -1,5 +1,4 @@ use std::{ - hash::Hasher, path::PathBuf, time::{SystemTime, UNIX_EPOCH}, }; @@ -7,7 +6,6 @@ use std::{ use std::os::unix::ffi::OsStrExt; use fuser::INodeNo; -use twox_hash::XxHash64; use crate::music::metadata::MusicMetadata; use crate::origins::attrs::FileAttrs; @@ -38,27 +36,6 @@ fn secs_since_epoch(time: SystemTime) -> u64 { .unwrap_or(0); } -/// Hash inputs shared by every Item regardless of origin. Server manifests -/// and client Items both feed these same five values through xxhash, so a -/// file's hash is identical on both sides — that's what makes `Reconcile`'s -/// diff correct. -pub fn compute_item_hash( - inode: u64, - original_path: &[u8], - ctime_secs: u64, - mtime_secs: u64, - crtime_secs: u64, -) -> u64 { - let seed = 1234; - let mut hasher = XxHash64::with_seed(seed); - hasher.write_u64(inode); - hasher.write(original_path); - hasher.write_u64(ctime_secs); - hasher.write_u64(mtime_secs); - hasher.write_u64(crtime_secs); - return hasher.finish(); -} - impl Item { #[allow(clippy::too_many_arguments)] pub fn new( @@ -88,13 +65,13 @@ impl Item { } pub fn compute_hash(&self) -> u64 { - return compute_item_hash( + musicfs_core::compute_item_hash( self.inode.0, self.original_path.as_os_str().as_bytes(), secs_since_epoch(self.attrs.ctime), secs_since_epoch(self.attrs.mtime), secs_since_epoch(self.attrs.crtime), - ); + ) } } diff --git a/src/lib.rs b/crates/musicfs-client/src/lib.rs similarity index 55% rename from src/lib.rs rename to crates/musicfs-client/src/lib.rs index efd3009..594b597 100644 --- a/src/lib.rs +++ b/crates/musicfs-client/src/lib.rs @@ -1,8 +1,8 @@ pub mod db; pub mod item; -pub mod logging; pub mod music; pub mod origins; -pub mod proto; -pub mod server; pub mod virtual_dirs; + +pub use musicfs_core::logging; +pub use musicfs_proto as proto; diff --git a/src/main.rs b/crates/musicfs-client/src/main.rs similarity index 100% rename from src/main.rs rename to crates/musicfs-client/src/main.rs diff --git a/src/music/db.rs b/crates/musicfs-client/src/music/db.rs similarity index 71% rename from src/music/db.rs rename to crates/musicfs-client/src/music/db.rs index 7668b56..8f2c2b5 100644 --- a/src/music/db.rs +++ b/crates/musicfs-client/src/music/db.rs @@ -6,11 +6,93 @@ use sea_orm::entity::prelude::*; use crate::db::entities::Model; use crate::item::Item; use crate::music::metadata::MusicMetadata; -use crate::music::metadata::db::{ - artists, music_metadata as music_metadata_entity, other_tags, pictures, -}; use tracing::{debug, error}; +mod entities { + pub mod music_metadata { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "music_metadata")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub inode: i64, + pub track_title: String, + pub album: String, + pub track_number: i32, + #[sea_orm(column_type = "Blob")] + pub header: Vec, + pub real_audio_start: i64, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} + } + + pub mod artists { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "music_metadata_artists")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub inode: i64, + #[sea_orm(primary_key, auto_increment = false)] + pub artist: String, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} + } + + pub mod other_tags { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "music_metadata_other_tags")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub inode: i64, + #[sea_orm(primary_key, auto_increment = false)] + pub position: i32, + pub tag: String, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} + } + + pub mod pictures { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "music_metadata_pictures")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub inode: i64, + #[sea_orm(primary_key, auto_increment = false)] + pub position: i32, + #[sea_orm(column_type = "Blob")] + pub block_header: Vec, + pub data_offset: i64, + pub data_length: i64, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} + } +} + +use entities::{artists, music_metadata as music_metadata_entity, other_tags, pictures}; + pub async fn save_music_metadata( inode: i64, music_metadata: &MusicMetadata, diff --git a/crates/musicfs-client/src/music/mod.rs b/crates/musicfs-client/src/music/mod.rs new file mode 100644 index 0000000..15acc4e --- /dev/null +++ b/crates/musicfs-client/src/music/mod.rs @@ -0,0 +1,2 @@ +pub mod db; +pub use musicfs_core::music::{metadata, parse}; diff --git a/src/origins/local/file_io.rs b/crates/musicfs-client/src/origins/local/file_io.rs similarity index 100% rename from src/origins/local/file_io.rs rename to crates/musicfs-client/src/origins/local/file_io.rs diff --git a/src/origins/local/mod.rs b/crates/musicfs-client/src/origins/local/mod.rs similarity index 100% rename from src/origins/local/mod.rs rename to crates/musicfs-client/src/origins/local/mod.rs diff --git a/src/origins/local/snapshot.rs b/crates/musicfs-client/src/origins/local/snapshot.rs similarity index 100% rename from src/origins/local/snapshot.rs rename to crates/musicfs-client/src/origins/local/snapshot.rs diff --git a/src/origins/local/watcher.rs b/crates/musicfs-client/src/origins/local/watcher.rs similarity index 100% rename from src/origins/local/watcher.rs rename to crates/musicfs-client/src/origins/local/watcher.rs diff --git a/src/origins/mod.rs b/crates/musicfs-client/src/origins/mod.rs similarity index 99% rename from src/origins/mod.rs rename to crates/musicfs-client/src/origins/mod.rs index d2645a8..7365d9e 100644 --- a/src/origins/mod.rs +++ b/crates/musicfs-client/src/origins/mod.rs @@ -1,4 +1,4 @@ -pub mod attrs; +pub use musicfs_core::attrs; pub mod local; pub mod network; diff --git a/src/origins/network/mod.rs b/crates/musicfs-client/src/origins/network/mod.rs similarity index 92% rename from src/origins/network/mod.rs rename to crates/musicfs-client/src/origins/network/mod.rs index 15b716e..3cf5390 100644 --- a/src/origins/network/mod.rs +++ b/crates/musicfs-client/src/origins/network/mod.rs @@ -266,10 +266,8 @@ fn manifest_entry_to_item( rdev: 0, blksize: 4096, }; - let music_metadata: Option = entry - .music_metadata - .clone() - .map(|mm| MusicMetadata::from(mm)); + let music_metadata: Option = + entry.music_metadata.clone().map(music_metadata_from_proto); let mut local_path = PathBuf::new(); if let Some(mm) = &music_metadata { @@ -303,26 +301,24 @@ fn manifest_entry_to_item( ); } -impl From for MusicMetadata { - fn from(mm: crate::proto::MusicMetadata) -> Self { - return MusicMetadata { - artist: mm.artist, - album_artist: mm.album_artist, - album: mm.album, - track_number: mm.track_number, - track_title: mm.track_title, - other_tags: mm.other_tags, - header: mm.header, - picture_block_headers: mm.picture_block_headers, - picture_data_ranges: mm - .picture_data_ranges - .into_iter() - .map(|p| (p.offset, p.length)) - .collect(), - real_audio_start: mm.real_audio_start, - vorbis_comment_offset: mm.vorbis_comment_offset, - vorbis_comment_length: mm.vorbis_comment_length, - }; +fn music_metadata_from_proto(mm: crate::proto::MusicMetadata) -> MusicMetadata { + MusicMetadata { + artist: mm.artist, + album_artist: mm.album_artist, + album: mm.album, + track_number: mm.track_number, + track_title: mm.track_title, + other_tags: mm.other_tags, + header: mm.header, + picture_block_headers: mm.picture_block_headers, + picture_data_ranges: mm + .picture_data_ranges + .into_iter() + .map(|p| (p.offset, p.length)) + .collect(), + real_audio_start: mm.real_audio_start, + vorbis_comment_offset: mm.vorbis_comment_offset, + vorbis_comment_length: mm.vorbis_comment_length, } } diff --git a/src/origins/network/transport.rs b/crates/musicfs-client/src/origins/network/transport.rs similarity index 100% rename from src/origins/network/transport.rs rename to crates/musicfs-client/src/origins/network/transport.rs diff --git a/src/origins/network/watcher.rs b/crates/musicfs-client/src/origins/network/watcher.rs similarity index 100% rename from src/origins/network/watcher.rs rename to crates/musicfs-client/src/origins/network/watcher.rs diff --git a/src/virtual_dirs.rs b/crates/musicfs-client/src/virtual_dirs.rs similarity index 100% rename from src/virtual_dirs.rs rename to crates/musicfs-client/src/virtual_dirs.rs diff --git a/tests/file_io_test.rs b/crates/musicfs-client/tests/file_io_test.rs similarity index 100% rename from tests/file_io_test.rs rename to crates/musicfs-client/tests/file_io_test.rs diff --git a/tests/snapshot_test.rs b/crates/musicfs-client/tests/snapshot_test.rs similarity index 100% rename from tests/snapshot_test.rs rename to crates/musicfs-client/tests/snapshot_test.rs diff --git a/crates/musicfs-core/Cargo.toml b/crates/musicfs-core/Cargo.toml new file mode 100644 index 0000000..08863b8 --- /dev/null +++ b/crates/musicfs-core/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "musicfs-core" +version.workspace = true +edition.workspace = true + +[dependencies] +symphonia.workspace = true +twox-hash.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +tracing-appender.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/src/origins/attrs.rs b/crates/musicfs-core/src/attrs.rs similarity index 100% rename from src/origins/attrs.rs rename to crates/musicfs-core/src/attrs.rs diff --git a/crates/musicfs-core/src/hash.rs b/crates/musicfs-core/src/hash.rs new file mode 100644 index 0000000..8f0cd97 --- /dev/null +++ b/crates/musicfs-core/src/hash.rs @@ -0,0 +1,20 @@ +use std::hash::Hasher; + +use twox_hash::XxHash64; + +pub fn compute_item_hash( + inode: u64, + original_path: &[u8], + ctime_secs: u64, + mtime_secs: u64, + crtime_secs: u64, +) -> u64 { + let seed = 1234; + let mut hasher = XxHash64::with_seed(seed); + hasher.write_u64(inode); + hasher.write(original_path); + hasher.write_u64(ctime_secs); + hasher.write_u64(mtime_secs); + hasher.write_u64(crtime_secs); + hasher.finish() +} diff --git a/crates/musicfs-core/src/lib.rs b/crates/musicfs-core/src/lib.rs new file mode 100644 index 0000000..29979ab --- /dev/null +++ b/crates/musicfs-core/src/lib.rs @@ -0,0 +1,8 @@ +pub mod attrs; +pub mod hash; +pub mod logging; +pub mod music; + +pub use attrs::FileAttrs; +pub use hash::compute_item_hash; +pub use music::metadata::MusicMetadata; diff --git a/src/logging.rs b/crates/musicfs-core/src/logging.rs similarity index 100% rename from src/logging.rs rename to crates/musicfs-core/src/logging.rs diff --git a/src/music/encoder.rs b/crates/musicfs-core/src/music/encoder.rs similarity index 100% rename from src/music/encoder.rs rename to crates/musicfs-core/src/music/encoder.rs diff --git a/src/music/flac.rs b/crates/musicfs-core/src/music/flac.rs similarity index 100% rename from src/music/flac.rs rename to crates/musicfs-core/src/music/flac.rs diff --git a/crates/musicfs-core/src/music/metadata.rs b/crates/musicfs-core/src/music/metadata.rs new file mode 100644 index 0000000..fdb8de0 --- /dev/null +++ b/crates/musicfs-core/src/music/metadata.rs @@ -0,0 +1,62 @@ +use symphonia::core::meta::{MetadataRevision, StandardTagKey}; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct MusicMetadata { + pub artist: Vec, + pub album_artist: Option, + pub album: String, + pub track_number: i32, + pub track_title: String, + pub other_tags: Vec, + pub header: Vec, + pub picture_block_headers: Vec>, + pub picture_data_ranges: Vec<(u64, u64)>, + pub real_audio_start: u64, + pub vorbis_comment_offset: u64, + pub vorbis_comment_length: u64, +} + +impl MusicMetadata { + pub fn virtual_size(&self, real_file_size: u64) -> u64 { + let pictures_size: u64 = self + .picture_block_headers + .iter() + .zip(self.picture_data_ranges.iter()) + .map(|(prefix, (_, len))| prefix.len() as u64 + len) + .sum(); + self.header.len() as u64 + pictures_size + (real_file_size - self.real_audio_start) + } +} + +pub(crate) fn extract_standard_tags(revision: &MetadataRevision, out: &mut MusicMetadata) { + for tag in revision.tags() { + let value = tag.value.to_string(); + match tag.std_key { + Some(StandardTagKey::Artist) => out.artist.push(value), + Some(StandardTagKey::AlbumArtist) => out.album_artist = Some(value), + Some(StandardTagKey::Album) => out.album = value, + Some(StandardTagKey::TrackNumber) => { + out.track_number = value.parse::().unwrap_or(0) + } + Some(StandardTagKey::TrackTitle) => out.track_title = value, + _ => out.other_tags.push(format!("{}={}", tag.key, value)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn virtual_size_calculation() { + let mut metadata = MusicMetadata::default(); + metadata.header = vec![0u8; 100]; + metadata.picture_block_headers = vec![vec![0u8; 4]]; + metadata.picture_data_ranges = vec![(0, 50)]; + metadata.real_audio_start = 200; + let file_size = 1000u64; + let virtual_size = metadata.virtual_size(file_size); + assert_eq!(virtual_size, 954); + } +} diff --git a/src/music/mod.rs b/crates/musicfs-core/src/music/mod.rs similarity index 88% rename from src/music/mod.rs rename to crates/musicfs-core/src/music/mod.rs index 4277032..4490316 100644 --- a/src/music/mod.rs +++ b/crates/musicfs-core/src/music/mod.rs @@ -1,4 +1,3 @@ -pub mod db; pub mod encoder; pub mod flac; pub mod metadata; diff --git a/src/music/mp3.rs b/crates/musicfs-core/src/music/mp3.rs similarity index 100% rename from src/music/mp3.rs rename to crates/musicfs-core/src/music/mp3.rs diff --git a/src/music/parse.rs b/crates/musicfs-core/src/music/parse.rs similarity index 100% rename from src/music/parse.rs rename to crates/musicfs-core/src/music/parse.rs diff --git a/src/music/parser.rs b/crates/musicfs-core/src/music/parser.rs similarity index 100% rename from src/music/parser.rs rename to crates/musicfs-core/src/music/parser.rs diff --git a/crates/musicfs-proto/Cargo.toml b/crates/musicfs-proto/Cargo.toml new file mode 100644 index 0000000..a60702d --- /dev/null +++ b/crates/musicfs-proto/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "musicfs-proto" +version.workspace = true +edition.workspace = true + +[dependencies] +prost.workspace = true +tonic.workspace = true +tonic-prost.workspace = true diff --git a/src/proto/generated/musicfs/musicfs.rs b/crates/musicfs-proto/src/generated/musicfs/musicfs.rs similarity index 100% rename from src/proto/generated/musicfs/musicfs.rs rename to crates/musicfs-proto/src/generated/musicfs/musicfs.rs diff --git a/src/proto/generated/musicfs/musicfs.tonic.rs b/crates/musicfs-proto/src/generated/musicfs/musicfs.tonic.rs similarity index 100% rename from src/proto/generated/musicfs/musicfs.tonic.rs rename to crates/musicfs-proto/src/generated/musicfs/musicfs.tonic.rs diff --git a/crates/musicfs-proto/src/lib.rs b/crates/musicfs-proto/src/lib.rs new file mode 100644 index 0000000..df0e147 --- /dev/null +++ b/crates/musicfs-proto/src/lib.rs @@ -0,0 +1,11 @@ +pub mod musicfs { + include!("generated/musicfs/musicfs.rs"); +} + +pub use musicfs::{ + ChangeEvent, FileChunk, GetFileRequest, GetManifestRequest, GetMetadataRequest, InodeHash, + ManifestEntry, MusicMetadata, PictureDataRange, ReconcileRequest, ReconcileResponse, + SubscribeEventsRequest, + music_fs_client::MusicFsClient, + music_fs_server::{MusicFs, MusicFsServer}, +}; diff --git a/crates/musicfs-server/Cargo.toml b/crates/musicfs-server/Cargo.toml new file mode 100644 index 0000000..c5375eb --- /dev/null +++ b/crates/musicfs-server/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "musicfs-server" +version.workspace = true +edition.workspace = true +default-run = "musicfs-server" + +[[bin]] +name = "musicfs-server" +path = "src/bin/musicfs-server.rs" + +[dependencies] +musicfs-core.workspace = true +musicfs-proto.workspace = true +tonic.workspace = true +tonic-health.workspace = true +tonic-reflection.workspace = true +tokio.workspace = true +anyhow.workspace = true +async-trait.workspace = true +clap.workspace = true +notify.workspace = true +tokio-stream.workspace = true +tracing.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/src/bin/musicfs-server.rs b/crates/musicfs-server/src/bin/musicfs-server.rs similarity index 97% rename from src/bin/musicfs-server.rs rename to crates/musicfs-server/src/bin/musicfs-server.rs index 602719b..f46ad3a 100644 --- a/src/bin/musicfs-server.rs +++ b/crates/musicfs-server/src/bin/musicfs-server.rs @@ -2,8 +2,8 @@ use std::{net::SocketAddr, path::PathBuf}; use anyhow::{Context, Result}; use clap::Parser; -use musicfs::logging::{LogConfig, init}; -use musicfs::server::{ +use musicfs_core::logging::{LogConfig, init}; +use musicfs_server::server::{ state::ServerState, transport::{self, TransportArgs}, watcher::ServerWatcher, diff --git a/crates/musicfs-server/src/lib.rs b/crates/musicfs-server/src/lib.rs new file mode 100644 index 0000000..74f47ad --- /dev/null +++ b/crates/musicfs-server/src/lib.rs @@ -0,0 +1 @@ +pub mod server; diff --git a/src/server/manifest.rs b/crates/musicfs-server/src/server/manifest.rs similarity index 97% rename from src/server/manifest.rs rename to crates/musicfs-server/src/server/manifest.rs index 6d3bb60..50ef1fd 100644 --- a/src/server/manifest.rs +++ b/crates/musicfs-server/src/server/manifest.rs @@ -1,5 +1,5 @@ -use crate::item::compute_item_hash; -use crate::music::metadata::MusicMetadata; +use musicfs_core::compute_item_hash; +use musicfs_core::music::metadata::MusicMetadata; /// One row of the in-memory manifest: every field the client needs to /// reconstruct an `Item` whose hash matches the server's hash. diff --git a/src/server/mod.rs b/crates/musicfs-server/src/server/mod.rs similarity index 100% rename from src/server/mod.rs rename to crates/musicfs-server/src/server/mod.rs diff --git a/src/server/range.rs b/crates/musicfs-server/src/server/range.rs similarity index 100% rename from src/server/range.rs rename to crates/musicfs-server/src/server/range.rs diff --git a/src/server/state.rs b/crates/musicfs-server/src/server/state.rs similarity index 97% rename from src/server/state.rs rename to crates/musicfs-server/src/server/state.rs index cc7c735..6d94781 100644 --- a/src/server/state.rs +++ b/crates/musicfs-server/src/server/state.rs @@ -8,9 +8,9 @@ use std::{ use std::os::unix::fs::MetadataExt; -use crate::music::parse::parse_music_metadata_for_path; -use crate::origins::attrs::FileAttrs; use crate::server::manifest::ManifestEntry; +use musicfs_core::FileAttrs; +use musicfs_core::music::parse::parse_music_metadata_for_path; use tracing::info; /// Server-side entry for one file. Built once on startup from a directory @@ -20,7 +20,7 @@ pub struct FileEntry { pub abs_path: PathBuf, pub rel_path: String, pub attrs: FileAttrs, - pub music_metadata: Option, + pub music_metadata: Option, } #[derive(Clone)] diff --git a/src/server/transport/grpc.rs b/crates/musicfs-server/src/server/transport/grpc.rs similarity index 89% rename from src/server/transport/grpc.rs rename to crates/musicfs-server/src/server/transport/grpc.rs index 86838a4..167c3c5 100644 --- a/src/server/transport/grpc.rs +++ b/crates/musicfs-server/src/server/transport/grpc.rs @@ -10,19 +10,19 @@ use async_trait::async_trait; use tokio::sync::broadcast::Receiver; use tonic::{Request, Response, Status, transport::Server}; -use crate::music::metadata::MusicMetadata; -use crate::proto as proto_types; -use crate::proto::MusicFs as MusicFsTrait; -use crate::proto::{ +use crate::server::manifest::ManifestEntry as DomainManifestEntry; +use crate::server::state::ServerState; +use crate::server::transport::{MusicTransport, TransportArgs}; +use crate::server::watcher::{ChangeEvent, ChangeKind}; +use musicfs_core::music::metadata::MusicMetadata; +use musicfs_proto as proto_types; +use musicfs_proto::MusicFs as MusicFsTrait; +use musicfs_proto::{ ChangeEvent as ProtoChangeEvent, FileChunk, GetFileRequest, GetManifestRequest, GetMetadataRequest, InodeHash, ManifestEntry, MusicFsServer, MusicMetadata as ProtoMusicMetadata, PictureDataRange, ReconcileRequest, ReconcileResponse, SubscribeEventsRequest, }; -use crate::server::manifest::ManifestEntry as DomainManifestEntry; -use crate::server::state::ServerState; -use crate::server::transport::{MusicTransport, TransportArgs}; -use crate::server::watcher::{ChangeEvent, ChangeKind}; use tracing::{debug, info, warn}; pub struct GrpcTransport { @@ -53,11 +53,11 @@ impl MusicTransport for GrpcTransport { // v1alpha covers older clients; v1 is the current standard. let reflection_v1 = tonic_reflection::server::Builder::configure() - .register_encoded_file_descriptor_set(crate::proto::musicfs::FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(musicfs_proto::musicfs::FILE_DESCRIPTOR_SET) .register_encoded_file_descriptor_set(tonic_health::pb::FILE_DESCRIPTOR_SET) .build_v1()?; let reflection_v1alpha = tonic_reflection::server::Builder::configure() - .register_encoded_file_descriptor_set(crate::proto::musicfs::FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(musicfs_proto::musicfs::FILE_DESCRIPTOR_SET) .register_encoded_file_descriptor_set(tonic_health::pb::FILE_DESCRIPTOR_SET) .build_v1alpha()?; @@ -298,31 +298,29 @@ impl From for ManifestEntry { mtime: entry.mtime, ctime: entry.ctime, crtime: entry.crtime, - music_metadata: entry.music_metadata.map(Into::into), + music_metadata: entry.music_metadata.map(music_metadata_to_proto), }; } } -impl From for ProtoMusicMetadata { - fn from(mm: MusicMetadata) -> Self { - return ProtoMusicMetadata { - artist: mm.artist, - album_artist: mm.album_artist, - album: mm.album, - track_number: mm.track_number, - track_title: mm.track_title, - other_tags: mm.other_tags, - header: mm.header, - picture_block_headers: mm.picture_block_headers, - picture_data_ranges: mm - .picture_data_ranges - .into_iter() - .map(|(offset, length)| PictureDataRange { offset, length }) - .collect(), - real_audio_start: mm.real_audio_start, - vorbis_comment_offset: mm.vorbis_comment_offset, - vorbis_comment_length: mm.vorbis_comment_length, - }; +fn music_metadata_to_proto(mm: MusicMetadata) -> ProtoMusicMetadata { + ProtoMusicMetadata { + artist: mm.artist, + album_artist: mm.album_artist, + album: mm.album, + track_number: mm.track_number, + track_title: mm.track_title, + other_tags: mm.other_tags, + header: mm.header, + picture_block_headers: mm.picture_block_headers, + picture_data_ranges: mm + .picture_data_ranges + .into_iter() + .map(|(offset, length)| PictureDataRange { offset, length }) + .collect(), + real_audio_start: mm.real_audio_start, + vorbis_comment_offset: mm.vorbis_comment_offset, + vorbis_comment_length: mm.vorbis_comment_length, } } diff --git a/src/server/transport/mod.rs b/crates/musicfs-server/src/server/transport/mod.rs similarity index 100% rename from src/server/transport/mod.rs rename to crates/musicfs-server/src/server/transport/mod.rs diff --git a/src/server/watcher.rs b/crates/musicfs-server/src/server/watcher.rs similarity index 100% rename from src/server/watcher.rs rename to crates/musicfs-server/src/server/watcher.rs diff --git a/devenv.nix b/devenv.nix index b6e89ab..2204ec1 100644 --- a/devenv.nix +++ b/devenv.nix @@ -42,7 +42,7 @@ processes.musicfs = { after = [ "devenv:processes:postgres" ]; exec = lib.mkDefault '' - cargo run -- \ + cargo run -p musicfs-client -- \ --source /home/fujin/Music \ --mountpoint /tmp/rust-fuse \ --database "postgresql://fujin@localhost/musicfs?host=$PGHOST" @@ -53,7 +53,7 @@ processes.musicfs = { after = [ "devenv:processes:postgres" ]; exec = '' - cargo run -- \ + cargo run -p musicfs-client -- \ --source /home/fujin/Music \ --mountpoint /tmp/rust-fuse \ --database "postgresql://fujin@localhost/musicfs?host=$PGHOST" @@ -65,7 +65,7 @@ processes.musicfs = { after = [ "devenv:processes:postgres" ]; exec = '' - cargo run -- \ + cargo run -p musicfs-client -- \ --source http://10.185.226.145:50051 \ --mountpoint /tmp/rust-fuse \ --database "postgresql://fujin@localhost/musicfs?host=$PGHOST" @@ -77,7 +77,7 @@ processes.musicfs = { after = [ "devenv:processes:postgres" ]; exec = '' - cargo run -- \ + cargo run -p musicfs-client -- \ --source http://127.0.0.1:50061 \ --mountpoint ./target/e2e/mnt \ --database "postgresql://fujin@localhost/musicfs_e2e?host=$PGHOST" \ diff --git a/proto/buf.gen.yaml b/proto/buf.gen.yaml index 94be0bf..b95eb1c 100644 --- a/proto/buf.gen.yaml +++ b/proto/buf.gen.yaml @@ -1,8 +1,8 @@ version: v2 plugins: - remote: buf.build/community/neoeinstein-prost - out: ../src/proto/generated + out: ../crates/musicfs-proto/src/generated opt: - file_descriptor_set - remote: buf.build/community/neoeinstein-tonic - out: ../src/proto/generated + out: ../crates/musicfs-proto/src/generated diff --git a/scripts/e2e/run-resilience.sh b/scripts/e2e/run-resilience.sh index b55a193..7448749 100755 --- a/scripts/e2e/run-resilience.sh +++ b/scripts/e2e/run-resilience.sh @@ -65,9 +65,19 @@ stop_postgres() { # ──────────────────────── VM management ──────────────────────────────── vm_ip() { incus list "$VM_NAME" -f csv -c 4 2>/dev/null | head -1 | awk '{print $1}'; } +detect_e2e_ip() { + local bridge_cidr + bridge_cidr="$(incus network get incusbr0 ipv4.address 2>/dev/null || true)" + if [[ -z "$bridge_cidr" ]]; then echo ""; return; fi + local gateway="${bridge_cidr%%/*}" + echo "${gateway%.*}.146" +} + start_vm() { log "starting VM '$VM_NAME' (port $PORT)…" + local e2e_ip; e2e_ip="$(detect_e2e_ip)" VM_NAME="$VM_NAME" \ + VM_IP="$e2e_ip" \ MUSIC_SOURCE="$MUSIC_DIR" \ LISTEN_PORT="$PORT" \ RUST_LOG=debug \ diff --git a/scripts/e2e/run.sh b/scripts/e2e/run.sh index f3ebdc7..5a7cfa7 100755 --- a/scripts/e2e/run.sh +++ b/scripts/e2e/run.sh @@ -62,9 +62,21 @@ stop_postgres() { # ──────────────────────── VM management ──────────────────────────────── vm_ip() { incus list "$VM_NAME" -f csv -c 4 2>/dev/null | head -1 | awk '{print $1}'; } +detect_e2e_ip() { + local bridge_cidr + bridge_cidr="$(incus network get incusbr0 ipv4.address 2>/dev/null || true)" + if [[ -z "$bridge_cidr" ]]; then echo ""; return; fi + local gateway="${bridge_cidr%%/*}" + echo "${gateway%.*}.146" +} + start_vm() { log "starting VM '$VM_NAME' (port $PORT)…" + # e2e VM uses .146 to avoid colliding with the dev VM (.145) + local e2e_ip + e2e_ip="$(detect_e2e_ip)" VM_NAME="$VM_NAME" \ + VM_IP="$e2e_ip" \ MUSIC_SOURCE="$MUSIC_DIR" \ LISTEN_PORT="$PORT" \ RUST_LOG=debug \ diff --git a/scripts/vm.sh b/scripts/vm.sh index fefbcf7..3e0f2f1 100755 --- a/scripts/vm.sh +++ b/scripts/vm.sh @@ -179,11 +179,7 @@ wait_for_vm() { deploy() { local bundle; bundle="$(build_bundle)" log "deploying bundle to $VM_NAME:$SERVER_DIR ..." - # Stop first: overwriting a running binary fails with ETXTBSY ("text file busy"). - if incus exec "$VM_NAME" -- systemctl is-active --quiet "$UNIT_NAME" 2>/dev/null; then - log "stopping $UNIT_NAME (binary in use)..." - incus exec "$VM_NAME" -- systemctl stop "$UNIT_NAME" - fi + incus exec "$VM_NAME" -- systemctl stop "$UNIT_NAME" 2>/dev/null || true incus exec "$VM_NAME" -- mkdir -p "$SERVER_DIR/lib" "$LOG_DIR" incus file push "$bundle/server" "$VM_NAME$SERVER_DIR/server" --mode=0755 # Ship the bundled libs one by one (robust across incus file-push versions). diff --git a/src/music/metadata.rs b/src/music/metadata.rs deleted file mode 100644 index ef752bb..0000000 --- a/src/music/metadata.rs +++ /dev/null @@ -1,153 +0,0 @@ -use symphonia::core::meta::{MetadataRevision, StandardTagKey}; - -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct MusicMetadata { - pub artist: Vec, - pub album_artist: Option, - pub album: String, - pub track_number: i32, - pub track_title: String, - pub other_tags: Vec, - pub header: Vec, - /// Per-externalized-region reinsert prefix kept in memory. FLAC stores its - /// 4-byte PICTURE block header here; MP3 keeps whole frames in the original - /// file and stores an empty prefix. - pub picture_block_headers: Vec>, - pub picture_data_ranges: Vec<(u64, u64)>, - pub real_audio_start: u64, - pub vorbis_comment_offset: u64, - pub vorbis_comment_length: u64, -} - -impl MusicMetadata { - /// Size of the virtual file: rebuilt header + externalized regions + the - /// original audio tail. Format-agnostic. - pub fn virtual_size(&self, real_file_size: u64) -> u64 { - let pictures_size: u64 = self - .picture_block_headers - .iter() - .zip(self.picture_data_ranges.iter()) - .map(|(prefix, (_, len))| prefix.len() as u64 + len) - .sum(); - self.header.len() as u64 + pictures_size + (real_file_size - self.real_audio_start) - } -} - -/// Map symphonia's normalized tags onto the shared `MusicMetadata` fields. -/// Used by every format parser so the field mapping lives in one place. -pub(crate) fn extract_standard_tags(revision: &MetadataRevision, out: &mut MusicMetadata) { - for tag in revision.tags() { - let value = tag.value.to_string(); - match tag.std_key { - Some(StandardTagKey::Artist) => out.artist.push(value), - Some(StandardTagKey::AlbumArtist) => out.album_artist = Some(value), - Some(StandardTagKey::Album) => out.album = value, - Some(StandardTagKey::TrackNumber) => { - out.track_number = value.parse::().unwrap_or(0) - } - Some(StandardTagKey::TrackTitle) => out.track_title = value, - _ => out.other_tags.push(format!("{}={}", tag.key, value)), - } - } -} - -pub mod db { - pub mod music_metadata { - use sea_orm::entity::prelude::*; - - #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] - #[sea_orm(table_name = "music_metadata")] - pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub inode: i64, - pub track_title: String, - pub album: String, - pub track_number: i32, - #[sea_orm(column_type = "Blob")] - pub header: Vec, - pub real_audio_start: i64, - } - - #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] - pub enum Relation {} - - impl ActiveModelBehavior for ActiveModel {} - } - - pub mod artists { - use sea_orm::entity::prelude::*; - - #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] - #[sea_orm(table_name = "music_metadata_artists")] - pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub inode: i64, - #[sea_orm(primary_key, auto_increment = false)] - pub artist: String, - } - - #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] - pub enum Relation {} - - impl ActiveModelBehavior for ActiveModel {} - } - - pub mod other_tags { - use sea_orm::entity::prelude::*; - - #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] - #[sea_orm(table_name = "music_metadata_other_tags")] - pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub inode: i64, - #[sea_orm(primary_key, auto_increment = false)] - pub position: i32, - pub tag: String, - } - - #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] - pub enum Relation {} - - impl ActiveModelBehavior for ActiveModel {} - } - - pub mod pictures { - use sea_orm::entity::prelude::*; - - #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] - #[sea_orm(table_name = "music_metadata_pictures")] - pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub inode: i64, - #[sea_orm(primary_key, auto_increment = false)] - pub position: i32, - #[sea_orm(column_type = "Blob")] - pub block_header: Vec, - pub data_offset: i64, - pub data_length: i64, - } - - #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] - pub enum Relation {} - - impl ActiveModelBehavior for ActiveModel {} - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn virtual_size_calculation() { - let mut metadata = MusicMetadata::default(); - metadata.header = vec![0u8; 100]; - metadata.picture_block_headers = vec![vec![0u8; 4]]; - metadata.picture_data_ranges = vec![(0, 50)]; - metadata.real_audio_start = 200; - let file_size = 1000u64; - let virtual_size = metadata.virtual_size(file_size); - // 100 (header) + 4 (prefix) + 50 (picture data) + 800 (audio tail) - assert_eq!(virtual_size, 954); - } -} diff --git a/src/proto/mod.rs b/src/proto/mod.rs deleted file mode 100644 index 76917a3..0000000 --- a/src/proto/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Generated protobuf + tonic bindings for the musicfs service. -//! -//! `buf generate` writes prost messages to `generated/musicfs/musicfs.rs` and -//! the tonic service to `generated/musicfs/musicfs.tonic.rs`. The prost plugin -//! appends `include!("musicfs.tonic.rs")` to the end of `musicfs.rs`, so -//! including that one file here also pulls in the service. The only glue buf -//! does not generate is the module named after the proto package (`musicfs`), -//! which we provide below before re-exporting the types callers use. -pub mod musicfs { - include!("generated/musicfs/musicfs.rs"); -} - -pub use musicfs::{ - ChangeEvent, FileChunk, GetFileRequest, GetManifestRequest, GetMetadataRequest, InodeHash, - ManifestEntry, MusicMetadata, PictureDataRange, ReconcileRequest, ReconcileResponse, - SubscribeEventsRequest, - music_fs_client::MusicFsClient, - music_fs_server::{MusicFs, MusicFsServer}, -};