Compare commits
35 Commits
823aaf3fe4
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
 Alexander
|
154f85bd9b | ||
|
Alexander
|
61457e1f89 | ||
|
Alexander
|
4a1b68981e | ||
|
Alexander
|
b88583707d | ||
|
Alexander
|
18024dbc62 | ||
|
Alexander
|
b0c41e3fa0 | ||
|
Alexander
|
1a7f70ae1c | ||
|
Alexander
|
391f556286 | ||
|
Alexander
|
9623644263 | ||
|
Alexander
|
487b119935 | ||
|
Alexander
|
c826bcf35f | ||
|
Alexander
|
ebf4044a01 | ||
|
Alexander
|
4f4a4169f8 | ||
|
Alexander
|
84bbd8f630 | ||
|
Alexander
|
128a6e079e | ||
|
Alexander
|
693b4f067b | ||
|
Alexander
|
66cd4e945c | ||
|
Alexander
|
9d74f1a7a3 | ||
|
Alexander
|
6e20ffe939 | ||
|
Alexander
|
daffd518d1 | ||
|
Alexander
|
a705d4d3b9 | ||
|
Alexander
|
e4bf557151 | ||
|
Alexander
|
39622be117 | ||
|
Alexander
|
265f4958f0 | ||
|
Alexander
|
305d027c8b | ||
|
Alexander
|
90e9683076 | ||
|
Alexander
|
0ff2a17ab7 | ||
|
Alexander
|
3038c94b8c | ||
|
Alexander
|
5da96ffab2 | ||
|
Alexander
|
4e394c60ec | ||
|
Alexander
|
6285eeb6c0 | ||
|
Alexander
|
24086cc744 | ||
|
Alexander
|
e3eeba4650 | ||
|
Alexander
|
00f14930cd | ||
|
Alexander
|
c6aa47f440 |
+35
@@ -14,4 +14,39 @@ tests/*.log
|
||||
|
||||
# Nix
|
||||
result
|
||||
|
||||
.cargo/
|
||||
.direnv/
|
||||
.pre-commit-config.yaml
|
||||
|
||||
###
|
||||
# Rust
|
||||
###
|
||||
result-*
|
||||
# Generated by Cargo
|
||||
# will have compiled files and executables
|
||||
debug
|
||||
target
|
||||
|
||||
# These are backup files generated by rustfmt
|
||||
**/*.rs.bk
|
||||
|
||||
# MSVC Windows builds of rustc generate these, which store debugging information
|
||||
*.pdb
|
||||
|
||||
# Generated by cargo mutants
|
||||
# Contains mutation testing data
|
||||
**/mutants.out*/
|
||||
|
||||
# rustc will dump stack traces when hitting an internal compiler error to PWD
|
||||
rustc-ice-*.txt
|
||||
|
||||
# RustRover
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
dev/
|
||||
|
||||
.sisyphus/
|
||||
|
||||
Generated
+451
-30
@@ -158,7 +158,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -169,7 +169,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -274,6 +274,17 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bmrng"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9758e48498ae13d49b51a979d553d254e67021b203d9597e82a04ebd81025b2"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"loom",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.2"
|
||||
@@ -322,6 +333,12 @@ version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "cfg_aliases"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.44"
|
||||
@@ -366,7 +383,7 @@ dependencies = [
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -599,6 +616,27 @@ dependencies = [
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
|
||||
dependencies = [
|
||||
"csv-core",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv-core"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "5.5.3"
|
||||
@@ -612,6 +650,12 @@ dependencies = [
|
||||
"parking_lot_core 0.9.12",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
|
||||
|
||||
[[package]]
|
||||
name = "debugid"
|
||||
version = "0.8.0"
|
||||
@@ -692,7 +736,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -738,6 +782,17 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
|
||||
|
||||
[[package]]
|
||||
name = "fail"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c"
|
||||
dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.3.0"
|
||||
@@ -889,6 +944,21 @@ dependencies = [
|
||||
"zerocopy 0.7.35",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.32"
|
||||
@@ -896,6 +966,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -904,6 +975,34 @@ version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.32"
|
||||
@@ -922,8 +1021,13 @@ version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
@@ -950,6 +1054,19 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generator"
|
||||
version = "0.6.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "061d3be1afec479d56fa3bd182bf966c7999ec175fcfdb87ac14d417241366c6"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"log",
|
||||
"rustversion",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
@@ -1022,7 +1139,7 @@ dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tokio-util 0.7.18",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -1173,6 +1290,20 @@ dependencies = [
|
||||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-rustls"
|
||||
version = "0.24.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"http",
|
||||
"hyper",
|
||||
"rustls",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-timeout"
|
||||
version = "0.4.1"
|
||||
@@ -1587,12 +1718,52 @@ dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lofty"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dec4feeff6c7d75093278133a06e827d7af6d2bfe20b0f331f9d10338a5ec7ca"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"data-encoding",
|
||||
"flate2",
|
||||
"lofty_attr",
|
||||
"log",
|
||||
"ogg_pager",
|
||||
"paste",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lofty_attr"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "458ace39169e4b83c4f77ae3d42d5d1d11c422feef590219a97c973d3b524557"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "loom"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27a6650b2f722ae8c0e2ebc46d07f80c9923464fc206d962332f1eff83143530"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
"generator",
|
||||
"scoped-tls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.12.5"
|
||||
@@ -1720,6 +1891,18 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mockall_double"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dffc15b97456ecc84d2bde8c1df79145e154f45225828c4361f676e1b82acd6"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "moka"
|
||||
version = "0.12.15"
|
||||
@@ -1753,8 +1936,10 @@ checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
|
||||
name = "musicfs-cache"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
"image",
|
||||
"lofty",
|
||||
"musicfs-cas",
|
||||
"musicfs-core",
|
||||
"musicfs-metadata",
|
||||
@@ -1762,6 +1947,7 @@ dependencies = [
|
||||
"rmp-serde",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sled",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
@@ -1775,11 +1961,13 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"dirs",
|
||||
"fail",
|
||||
"hex",
|
||||
"musicfs-cache",
|
||||
"musicfs-core",
|
||||
"musicfs-origins",
|
||||
"musicfs-sync",
|
||||
"parking_lot 0.12.5",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"sled",
|
||||
@@ -1797,13 +1985,23 @@ dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"dirs",
|
||||
"libc",
|
||||
"musicfs-cache",
|
||||
"musicfs-cas",
|
||||
"musicfs-core",
|
||||
"musicfs-fuse",
|
||||
"musicfs-grpc",
|
||||
"musicfs-metadata",
|
||||
"musicfs-origins",
|
||||
"parking_lot 0.12.5",
|
||||
"sd-notify",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util 0.7.18",
|
||||
"toml",
|
||||
"tonic",
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
"tracing-journald",
|
||||
@@ -1815,6 +2013,7 @@ name = "musicfs-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"hex",
|
||||
"parking_lot 0.12.5",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
@@ -1847,10 +2046,15 @@ name = "musicfs-grpc"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"csv",
|
||||
"hex",
|
||||
"hmac",
|
||||
"musicfs-cache",
|
||||
"musicfs-cas",
|
||||
"musicfs-core",
|
||||
"musicfs-metadata",
|
||||
"musicfs-search",
|
||||
"parking_lot 0.12.5",
|
||||
"prost",
|
||||
"reqwest",
|
||||
"serde",
|
||||
@@ -1882,8 +2086,10 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"dashmap",
|
||||
"futures",
|
||||
"libc",
|
||||
"musicfs-core",
|
||||
"parking_lot 0.12.5",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
@@ -1942,6 +2148,33 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "musicfs-test-utils"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"fail",
|
||||
"libc",
|
||||
"musicfs-cache",
|
||||
"musicfs-cas",
|
||||
"musicfs-core",
|
||||
"musicfs-origins",
|
||||
"musicfs-search",
|
||||
"nix",
|
||||
"noxious-client",
|
||||
"parking_lot 0.12.5",
|
||||
"reqwest",
|
||||
"rlimit",
|
||||
"sd-notify",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tokio-test",
|
||||
"tokio-util 0.7.18",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "native-tls"
|
||||
version = "0.2.18"
|
||||
@@ -1959,6 +2192,18 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
@@ -1988,6 +2233,39 @@ dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "noxious"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e68998924150ba54dbf1adf4c3f7f7c10bb5d3c6789ab71af11e34fe4c667970"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bmrng",
|
||||
"bytes",
|
||||
"futures",
|
||||
"mockall_double",
|
||||
"pin-project-lite",
|
||||
"rand",
|
||||
"serde",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tokio-util 0.6.10",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "noxious-client"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b7ab7a9efb5768cd07e2b2455f80b3998d7397be76398c2ac03a52a42b652e7"
|
||||
dependencies = [
|
||||
"noxious",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.50.3"
|
||||
@@ -2044,6 +2322,15 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ogg_pager"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d36b1d6964c3ac92b7aea701057e02b6b91143d70d83b20abf75a231a3c0216"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
@@ -2084,7 +2371,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2217,7 +2504,7 @@ checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2282,7 +2569,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2321,7 +2608,7 @@ dependencies = [
|
||||
"prost",
|
||||
"prost-types",
|
||||
"regex",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
@@ -2335,7 +2622,7 @@ dependencies = [
|
||||
"itertools",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2524,6 +2811,7 @@ dependencies = [
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-tls",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
@@ -2533,6 +2821,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustls",
|
||||
"rustls-pemfile",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -2541,14 +2830,39 @@ dependencies = [
|
||||
"system-configuration",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tokio-rustls",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"webpki-roots",
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.17.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"getrandom 0.2.17",
|
||||
"libc",
|
||||
"untrusted",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rlimit"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmp"
|
||||
version = "0.8.15"
|
||||
@@ -2630,6 +2944,18 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.21.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
|
||||
dependencies = [
|
||||
"log",
|
||||
"ring",
|
||||
"rustls-webpki",
|
||||
"sct",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pemfile"
|
||||
version = "1.0.4"
|
||||
@@ -2639,6 +2965,16 @@ dependencies = [
|
||||
"base64 0.21.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.101.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.22"
|
||||
@@ -2669,12 +3005,37 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scoped-tls"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "sct"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sd-notify"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "3.7.0"
|
||||
@@ -2731,7 +3092,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3036,6 +3397,17 @@ dependencies = [
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
@@ -3061,7 +3433,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3277,7 +3649,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3288,7 +3660,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3376,7 +3748,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3389,6 +3761,16 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
|
||||
dependencies = [
|
||||
"rustls",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.18"
|
||||
@@ -3400,6 +3782,31 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-test"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.6.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"log",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.18"
|
||||
@@ -3409,6 +3816,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
@@ -3491,7 +3899,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"prost-build",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3508,7 +3916,7 @@ dependencies = [
|
||||
"rand",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tokio-util 0.7.18",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
@@ -3532,6 +3940,7 @@ version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
|
||||
dependencies = [
|
||||
"log",
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
"tracing-core",
|
||||
@@ -3558,7 +3967,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3654,6 +4063,12 @@ version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.8"
|
||||
@@ -3799,7 +4214,7 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@@ -3981,7 +4396,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
"wasmtime-component-util",
|
||||
"wasmtime-wit-bindgen",
|
||||
"wit-parser 0.201.0",
|
||||
@@ -4155,7 +4570,7 @@ checksum = "ffaafa5c12355b1a9ee068e9295d50c4ca0a400c721950cdae4f5b54391a2da5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4225,6 +4640,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.25.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
@@ -4293,7 +4714,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4304,7 +4725,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4543,7 +4964,7 @@ dependencies = [
|
||||
"heck 0.5.0",
|
||||
"indexmap 2.14.0",
|
||||
"prettyplease",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
"wasm-metadata",
|
||||
"wit-bindgen-core",
|
||||
"wit-component",
|
||||
@@ -4559,7 +4980,7 @@ dependencies = [
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
"wit-bindgen-core",
|
||||
"wit-bindgen-rust",
|
||||
]
|
||||
@@ -4650,7 +5071,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4681,7 +5102,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4692,7 +5113,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4712,7 +5133,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4746,7 +5167,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13,7 +13,9 @@ repository = "https://github.com/user/musicfs"
|
||||
[workspace.dependencies]
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-util = { version = "0.7", features = ["rt"] }
|
||||
async-trait = "0.1"
|
||||
futures = "0.3"
|
||||
|
||||
# Error handling
|
||||
thiserror = "1"
|
||||
@@ -61,6 +63,12 @@ clap = { version = "4", features = ["derive"] }
|
||||
|
||||
# Testing
|
||||
tempfile = "3"
|
||||
fail = "0.5"
|
||||
rlimit = "0.10"
|
||||
nix = { version = "0.29", features = ["signal", "process"] }
|
||||
wiremock = "0.6"
|
||||
assert_cmd = "2.0"
|
||||
noxious-client = "1.0"
|
||||
|
||||
# Platform-specific
|
||||
libc = "0.2"
|
||||
@@ -81,5 +89,7 @@ tokio-stream = "0.1"
|
||||
image = { version = "0.24", default-features = false, features = ["jpeg", "png"] }
|
||||
chrono = "0.4"
|
||||
|
||||
sd-notify = "0.4"
|
||||
|
||||
[workspace.dependencies.tonic-build]
|
||||
version = "0.11"
|
||||
@@ -0,0 +1,879 @@
|
||||
# MusicFS
|
||||
|
||||
> A read-only FUSE filesystem that presents your music library organized by metadata — artist, album, track — regardless of how files are stored on disk.
|
||||
|
||||
Browse `/Artist/Album/Track.flac` in any media player or file manager. Original files are never touched.
|
||||
|
||||
---
|
||||
|
||||
## What It Does
|
||||
|
||||
MusicFS mounts as a virtual filesystem. Point it at your music storage (local drive, NFS share, S3 bucket, SFTP server) and it exposes a clean metadata-based directory tree:
|
||||
|
||||
```
|
||||
/mnt/music/
|
||||
├── Metallica/
|
||||
│ └── 72 Seasons (2023) [FLAC]/
|
||||
│ ├── 01 - 72 Seasons.flac
|
||||
│ ├── 02 - Shadows Follow.flac
|
||||
│ └── cover.jpg
|
||||
├── Pink Floyd/
|
||||
│ └── The Wall (1979) [FLAC]/
|
||||
│ ├── 01 - In the Flesh?.flac
|
||||
│ └── ...
|
||||
└── .search/
|
||||
└── (full-text search — see Search section)
|
||||
```
|
||||
|
||||
Files are read directly from origin storage with local chunk caching. Once cached, playback works entirely offline. Write operations return `EROFS` — origin files are always safe.
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
| Feature | Details |
|
||||
|---------|---------|
|
||||
| **Instant mount** | O(1) regardless of library size (<500ms) |
|
||||
| **Metadata-organized paths** | Configurable path templates via `$artist`, `$album`, `$year`, etc. |
|
||||
| **Multi-origin federation** | Local, NFS, SMB, S3, SFTP — automatic failover by priority |
|
||||
| **Content-addressable cache** | Chunk-level deduplication, LRU eviction, delta sync (>90% bandwidth savings) |
|
||||
| **Full-text search** | `/.search/metallica/` returns instant results across 1M+ tracks |
|
||||
| **Metadata overlay** | Set/override tags in the virtual layer without modifying originals |
|
||||
| **Album art** | Virtual `cover.jpg` per album, extracted from embedded tags |
|
||||
| **Plugin system** | Native `.so` and WASM plugins for custom origins, formats, metadata sources |
|
||||
| **gRPC control API** | Cache stats, origin health, live event streaming, metadata management |
|
||||
| **systemd integration** | `sd_notify` ready, journald logging, clean SIGTERM handling |
|
||||
|
||||
**Supported formats:** FLAC, MP3, OGG, WAV, M4A, AAC, Opus
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Enter dev environment (provides Rust, FUSE3, SQLite, everything)
|
||||
nix develop
|
||||
|
||||
# 2. Build
|
||||
cargo build
|
||||
|
||||
# 3. Mount your music library
|
||||
./target/debug/musicfs mount /mnt/music --origin /path/to/your/music
|
||||
|
||||
# 4. Browse
|
||||
ls /mnt/music
|
||||
mpv /mnt/music/Artist/Album/01\ -\ Track.flac
|
||||
|
||||
# 5. Unmount
|
||||
fusermount -u /mnt/music
|
||||
```
|
||||
|
||||
No `rustup`, no `apt install`. The Nix flake provides the full toolchain.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
### From Nix (recommended)
|
||||
|
||||
```bash
|
||||
# Development shell — everything you need
|
||||
nix develop
|
||||
|
||||
# Or install the binary into your profile
|
||||
nix profile install .#musicfs
|
||||
```
|
||||
|
||||
### From Source
|
||||
|
||||
**Prerequisites (non-Nix):**
|
||||
- Rust 1.75+
|
||||
- `libfuse3-dev` / `fuse3` (package name varies by distro)
|
||||
- `libsqlite3-dev`
|
||||
- `libssl-dev`
|
||||
- `protobuf-compiler` (for gRPC)
|
||||
- `clang` + `lld`
|
||||
|
||||
```bash
|
||||
git clone https://github.com/user/musicfs
|
||||
cd musicfs/musicfs
|
||||
cargo build --release
|
||||
sudo cp target/release/musicfs /usr/local/bin/
|
||||
```
|
||||
|
||||
### System Requirements
|
||||
|
||||
| Resource | Minimum | Recommended |
|
||||
|----------|---------|-------------|
|
||||
| CPU | 1 core | 4 cores |
|
||||
| RAM | 256 MB | 2 GB |
|
||||
| Disk (cache) | 1 GB | 50 GB |
|
||||
| Linux kernel | 4.x+ | 5.x+ |
|
||||
| FUSE module | required | — |
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
MusicFS can be configured via file (`--config`), CLI flags, or environment variables (`RUST_LOG` for log level).
|
||||
|
||||
### Minimal Config
|
||||
|
||||
```toml
|
||||
mount_point = "/mnt/music"
|
||||
cache_dir = "/home/user/.cache/musicfs"
|
||||
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/mnt/nas/music"
|
||||
```
|
||||
|
||||
```bash
|
||||
musicfs mount --config /etc/musicfs/config.toml
|
||||
```
|
||||
|
||||
### Full Config Reference
|
||||
|
||||
<!-- embedme config.example.toml -->
|
||||
```toml
|
||||
# MusicFS Configuration
|
||||
# Copy to /etc/musicfs/config.toml or ~/.config/musicfs/config.toml
|
||||
|
||||
# Required: where to mount the virtual filesystem
|
||||
mount_point = "/mnt/music"
|
||||
|
||||
# Required: directory for cache data (CAS chunks, metadata, search index)
|
||||
cache_dir = "/var/cache/musicfs"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Origins - music sources (at least one required)
|
||||
# Supported types: local, nfs, smb, s3, sftp
|
||||
# Lower priority number = preferred source for failover
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[[origins]]
|
||||
id = "local-music"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
enabled = true
|
||||
path = "/home/user/Music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas-nfs"
|
||||
origin_type = "nfs"
|
||||
priority = 2
|
||||
enabled = true
|
||||
path = "/mnt/nas/music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas-smb"
|
||||
origin_type = "smb"
|
||||
priority = 3
|
||||
enabled = false
|
||||
path = "/mnt/smb/music"
|
||||
|
||||
[[origins]]
|
||||
id = "cloud-backup"
|
||||
origin_type = "s3"
|
||||
priority = 10
|
||||
enabled = false
|
||||
bucket = "my-music-backup"
|
||||
region = "us-east-1"
|
||||
|
||||
[[origins]]
|
||||
id = "remote-server"
|
||||
origin_type = "sftp"
|
||||
priority = 10
|
||||
enabled = false
|
||||
host = "music.example.com"
|
||||
port = 22
|
||||
user = "musicfs"
|
||||
path = "/srv/music"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Cache settings
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[cache]
|
||||
# In-memory metadata cache size (artist/album/track info)
|
||||
metadata_cache_mb = 100
|
||||
|
||||
# On-disk content cache size (audio chunks)
|
||||
content_cache_gb = 10
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Health monitoring for origin failover
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[health]
|
||||
# How often to check origin health
|
||||
check_interval_secs = 30
|
||||
|
||||
# Timeout for health check probes
|
||||
timeout_ms = 5000
|
||||
|
||||
# Consecutive failures before marking origin unhealthy
|
||||
unhealthy_threshold = 3
|
||||
|
||||
# Per-origin type thresholds (overrides unhealthy_threshold)
|
||||
[health.per_origin_thresholds]
|
||||
local = 1
|
||||
nfs = 3
|
||||
smb = 3
|
||||
s3 = 3
|
||||
sftp = 3
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[logging]
|
||||
# Directory for log files
|
||||
log_dir = "/var/log/musicfs"
|
||||
|
||||
# Output logs as JSON (for log aggregators)
|
||||
json_output = false
|
||||
|
||||
# Send logs to systemd journal
|
||||
journald = true
|
||||
|
||||
# Log level filter (tracing format)
|
||||
# Examples: "info", "debug", "musicfs=debug,warn", "musicfs_fuse=trace"
|
||||
level = "musicfs=info,warn"
|
||||
|
||||
# Trace sampling rate for performance tracing (0.0 to 1.0)
|
||||
trace_sample_rate = 1.0
|
||||
```
|
||||
|
||||
### Cache Layout on Disk
|
||||
|
||||
```
|
||||
~/.cache/musicfs/
|
||||
├── musicfs.db # SQLite: file metadata, virtual tree, overlay data
|
||||
├── musicfs.lock # Single-instance lock
|
||||
├── musicfs.pid # Daemon PID
|
||||
├── chunks/ # Content-addressable chunk files
|
||||
│ ├── aa/ # 256 subdirs (first 2 hex chars of hash)
|
||||
│ │ └── aa1b2c… # 64 KB average chunk
|
||||
│ └── ...
|
||||
├── search.idx/ # Tantivy full-text search index
|
||||
└── chunks.sled/ # Sled KV: content hash → chunk location
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
musicfs [OPTIONS] <COMMAND>
|
||||
|
||||
OPTIONS:
|
||||
-l, --log-level <LEVEL> Log verbosity [default: info]
|
||||
```
|
||||
|
||||
### `mount` — Start the filesystem
|
||||
|
||||
```bash
|
||||
# From CLI flags (quick start)
|
||||
musicfs mount /mnt/music --origin /path/to/music
|
||||
|
||||
# From config file
|
||||
musicfs mount --config /etc/musicfs/config.toml
|
||||
|
||||
# All flags
|
||||
musicfs mount [MOUNTPOINT] \
|
||||
--config <path> # Config file (overrides flags)
|
||||
--origin <path> # Source music directory
|
||||
--cache-dir <path> # Cache location [default: ~/.cache/musicfs]
|
||||
--grpc-port <port> # gRPC server port [default: 50052]
|
||||
```
|
||||
|
||||
### `status` — Daemon status
|
||||
|
||||
```bash
|
||||
musicfs status
|
||||
```
|
||||
|
||||
### `cache` — Cache management
|
||||
|
||||
```bash
|
||||
musicfs cache stats # Hit rate, size, dedup ratio
|
||||
musicfs cache clear # Clear all caches
|
||||
musicfs cache clear <origin-id> # Clear cache for one origin
|
||||
musicfs cache prefetch <path> [path…] # Pre-warm cache for paths
|
||||
```
|
||||
|
||||
### `search` — Full-text search
|
||||
|
||||
```bash
|
||||
musicfs search "metallica" # Search across all metadata
|
||||
musicfs search "dark side" --limit 20 # Limit results [default: 100]
|
||||
```
|
||||
|
||||
Search results are also browsable as a virtual directory (see [Search](#search)).
|
||||
|
||||
### `origin` — Origin management
|
||||
|
||||
```bash
|
||||
musicfs origin list # List all configured origins
|
||||
musicfs origin health <id> # Check health of one origin
|
||||
musicfs origin rescan <id> # Force re-scan and re-index
|
||||
```
|
||||
|
||||
### `metadata` — Metadata overlay
|
||||
|
||||
```bash
|
||||
# Requires running daemon
|
||||
musicfs metadata get "/Artist/Album/01 - Track.flac"
|
||||
musicfs metadata get "/Artist/Album/01 - Track.flac" --field artist
|
||||
|
||||
musicfs metadata set "/Artist/Album/01 - Track.flac" \
|
||||
--title "New Title" \
|
||||
--artist "New Artist" \
|
||||
--album "New Album" \
|
||||
--track 1 \
|
||||
--genre "Rock" \
|
||||
--date "2023"
|
||||
|
||||
# Set from JSON
|
||||
musicfs metadata set "/path/to/file.flac" --json '{"title":"foo","year":2023}'
|
||||
|
||||
# Show current (overlaid) metadata
|
||||
musicfs metadata diff "/path/to/file.flac"
|
||||
|
||||
# Revert overlay — restore original metadata
|
||||
musicfs metadata clear "/path/to/file.flac"
|
||||
|
||||
# Bulk import/export
|
||||
musicfs metadata import library.csv
|
||||
musicfs metadata import library.json
|
||||
musicfs metadata export --output library.json
|
||||
musicfs metadata export --output library.csv --query "artist:Metallica"
|
||||
```
|
||||
|
||||
> **Note:** `--endpoint` flag (default `http://[::1]:50051`) selects the gRPC server.
|
||||
|
||||
### `trash` — Deleted file recovery
|
||||
|
||||
When files disappear from the origin, MusicFS moves them to a virtual trash rather than removing them immediately.
|
||||
|
||||
```bash
|
||||
musicfs trash list --config /etc/musicfs/config.toml
|
||||
musicfs trash list --since 7d # Deleted in last 7 days
|
||||
musicfs trash list --origin local # Filter by origin
|
||||
musicfs trash list --path "/Metallica" # Filter by path prefix
|
||||
|
||||
musicfs trash restore "/Metallica/72 Seasons" # Restore folder
|
||||
musicfs trash restore --all # Restore everything
|
||||
|
||||
musicfs trash empty --older-than 30d # Permanently delete old entries
|
||||
musicfs trash empty --pattern "/Unknown*" # Delete by pattern
|
||||
```
|
||||
|
||||
### `events` — Live event stream
|
||||
|
||||
```bash
|
||||
musicfs events # All events
|
||||
musicfs events --type file_added # Filter by type
|
||||
# Event types: file_added, file_removed, file_modified,
|
||||
# origin_connected, origin_disconnected,
|
||||
# sync_started, sync_completed, cache_eviction
|
||||
```
|
||||
|
||||
### `shutdown` — Stop the daemon
|
||||
|
||||
```bash
|
||||
musicfs shutdown # Graceful (drain in-flight ops)
|
||||
musicfs shutdown --graceful false # Immediate
|
||||
musicfs shutdown --timeout 60 # Max drain timeout seconds
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Storage Origins
|
||||
|
||||
### Local Filesystem
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/mnt/nas/music"
|
||||
```
|
||||
|
||||
Changes detected via `inotify`. Zero-latency access.
|
||||
|
||||
### NFS
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "nfs"
|
||||
origin_type = "nfs"
|
||||
priority = 2
|
||||
host = "nas.local"
|
||||
export = "/exports/music"
|
||||
```
|
||||
|
||||
### SMB / CIFS
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "smb"
|
||||
origin_type = "smb"
|
||||
priority = 3
|
||||
host = "nas.local"
|
||||
share = "music"
|
||||
```
|
||||
|
||||
### S3 (stub — not yet functional)
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "s3"
|
||||
origin_type = "s3"
|
||||
priority = 4
|
||||
bucket = "my-music"
|
||||
region = "us-east-1"
|
||||
# Credentials via AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars
|
||||
```
|
||||
|
||||
### SFTP (stub — not yet functional)
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "sftp"
|
||||
origin_type = "sftp"
|
||||
priority = 4
|
||||
host = "server.example.com"
|
||||
port = 22
|
||||
username = "alice"
|
||||
# Auth via SSH agent or key file — never store passwords in config
|
||||
```
|
||||
|
||||
### Multi-Origin Failover
|
||||
|
||||
Multiple origins are federates into a single virtual tree. MusicFS selects origins by priority, falling back automatically when one becomes unhealthy. Health is polled every `check_interval_secs` (default: 30s). When all origins for a file are unavailable, cached data is served seamlessly.
|
||||
|
||||
---
|
||||
|
||||
## Virtual Filesystem Layout
|
||||
|
||||
### Path Templates
|
||||
|
||||
The virtual path for each file is built from its audio metadata using a configurable template. Variables are sanitized (no `/`, `\`, `:`).
|
||||
|
||||
**Default template:**
|
||||
```
|
||||
$artist/$album ($year) [$format_upper]/$track - $title.$format
|
||||
```
|
||||
|
||||
**Template variables:**
|
||||
|
||||
| Variable | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `$artist` | Track artist | `Metallica` |
|
||||
| `$album` | Album name | `72 Seasons` |
|
||||
| `$title` | Track title | `Lux Æterna` |
|
||||
| `$track` | Track number (zero-padded) | `03` |
|
||||
| `$disc` | Disc number | `1` |
|
||||
| `$year` | Release year | `2023` |
|
||||
| `$genre` | Genre | `Metal` |
|
||||
| `$format` | File extension (lowercase) | `flac` |
|
||||
| `$format_upper` | File extension (uppercase) | `FLAC` |
|
||||
|
||||
Files with missing metadata fall back to `Unknown Artist/Unknown Album/filename`.
|
||||
|
||||
### Album Art
|
||||
|
||||
Each album directory includes a virtual `cover.jpg` extracted from the embedded tags of the first track. No files are written to disk by MusicFS — the image is synthesized on read.
|
||||
|
||||
### Search
|
||||
|
||||
The `/.search/` virtual directory exposes full-text search as filesystem paths:
|
||||
|
||||
```bash
|
||||
# Search via filesystem — use the query as a directory name
|
||||
ls "/mnt/music/.search/dark side of the moon/"
|
||||
# → Returns matching tracks as symlinks to their virtual paths
|
||||
|
||||
# Or use the CLI
|
||||
musicfs search "dark side of the moon"
|
||||
musicfs search "artist:Metallica" --limit 50
|
||||
```
|
||||
|
||||
**Query syntax** (powered by [tantivy](https://github.com/quickwit-oss/tantivy)):
|
||||
|
||||
| Syntax | Example | Matches |
|
||||
|--------|---------|---------|
|
||||
| Simple terms | `metallica sandman` | All fields contain both words |
|
||||
| Field-specific | `artist:Metallica` | Artist field only |
|
||||
| Phrase | `album:"Master of Puppets"` | Exact phrase in album |
|
||||
| Fuzzy | `metalica~1` | Within Levenshtein distance 1 |
|
||||
| Range | `year:[1980 TO 1989]` | Numeric range |
|
||||
| Boolean | `genre:Metal AND year:[1980 TO 1989]` | Combined conditions |
|
||||
|
||||
Indexed fields: `title`, `artist`, `album`, `album_artist`, `genre`, `composer`, `year`.
|
||||
Results cached for 5 minutes. Max 1000 results per query. Queries capped at 256 characters.
|
||||
|
||||
### Smart Collections
|
||||
|
||||
Built-in and custom query-based virtual folders appear alongside regular directories:
|
||||
|
||||
- **Recently Added** — tracks added in the last 30 days
|
||||
- **80s Music** — year 1980–1989
|
||||
- **90s Music** — year 1990–1999
|
||||
|
||||
Custom collections can be defined via the gRPC API with compound boolean queries over any indexed field.
|
||||
|
||||
---
|
||||
|
||||
## Metadata Overlay
|
||||
|
||||
MusicFS lets you override metadata in the virtual layer **without touching origin files**. Overlaid metadata is synthesized into the audio file header on read — players see your corrected tags, the origin file is unchanged.
|
||||
|
||||
```bash
|
||||
# Fix a misnamed artist
|
||||
musicfs metadata set "/Unknown/Best Of/01 - Track.flac" \
|
||||
--artist "The Beatles" \
|
||||
--album "Past Masters"
|
||||
|
||||
# Verify
|
||||
musicfs metadata get "/The Beatles/Past Masters/01 - Track.flac"
|
||||
|
||||
# See what's been overlaid vs. original
|
||||
musicfs metadata diff "/The Beatles/Past Masters/01 - Track.flac"
|
||||
|
||||
# Revert
|
||||
musicfs metadata clear "/The Beatles/Past Masters/01 - Track.flac"
|
||||
```
|
||||
|
||||
Supported fields: `title`, `artist`, `album`, `album-artist`, `track`, `disc`, `genre`, `date`, `composer`, `comment`, `lyrics`, `copyright`, `compilation`, sort fields (`artist-sort`, etc.), MusicBrainz IDs, ReplayGain values, and arbitrary custom tags.
|
||||
|
||||
---
|
||||
|
||||
## Plugin Development
|
||||
|
||||
Plugins extend MusicFS without modifying core code. Three plugin types:
|
||||
|
||||
| Type | Purpose | Examples |
|
||||
|------|---------|---------|
|
||||
| **Origin** | Custom storage backends | Google Drive, Dropbox, custom NAS protocol |
|
||||
| **Metadata** | External tag enrichment | MusicBrainz, Discogs, Last.fm |
|
||||
| **Format** | Custom audio formats | Game audio, proprietary codecs |
|
||||
|
||||
### Native Plugin (`.so`)
|
||||
|
||||
```rust
|
||||
// Cargo.toml
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
musicfs-plugins = { path = "..." }
|
||||
semver = "1"
|
||||
serde_json = "1"
|
||||
```
|
||||
|
||||
```rust
|
||||
use musicfs_plugins::{declare_plugin, Plugin, PluginType, FormatPlugin};
|
||||
use musicfs_core::AudioMeta;
|
||||
use semver::Version;
|
||||
use serde_json::Value;
|
||||
|
||||
struct MyFormatPlugin;
|
||||
|
||||
impl Plugin for MyFormatPlugin {
|
||||
fn name(&self) -> &str { "my-format" }
|
||||
fn version(&self) -> Version { Version::new(1, 0, 0) }
|
||||
fn plugin_type(&self) -> PluginType { PluginType::Format }
|
||||
fn init(&mut self, _config: Value) -> musicfs_plugins::Result<()> { Ok(()) }
|
||||
fn shutdown(&mut self) -> musicfs_plugins::Result<()> { Ok(()) }
|
||||
}
|
||||
|
||||
impl FormatPlugin for MyFormatPlugin {
|
||||
fn extensions(&self) -> &[&str] { &["xyz"] }
|
||||
|
||||
fn parse(&self, reader: &mut dyn std::io::Read) -> musicfs_plugins::Result<AudioMeta> {
|
||||
// Parse your format and return metadata
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn synthesize_header(&self, metadata: &AudioMeta) -> musicfs_plugins::Result<Vec<u8>> {
|
||||
// Build a new file header with updated metadata
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
// Required export — MusicFS calls this to instantiate the plugin
|
||||
declare_plugin!(MyFormatPlugin, MyFormatPlugin);
|
||||
```
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
# produces target/release/libmy_format_plugin.so
|
||||
```
|
||||
|
||||
### Loading Plugins
|
||||
|
||||
```toml
|
||||
[plugins]
|
||||
enabled = true
|
||||
search_paths = ["/usr/lib/musicfs/plugins"] # Auto-discover .so files here
|
||||
|
||||
[plugins.plugins.my-format]
|
||||
path = "/path/to/libmy_format_plugin.so"
|
||||
enabled = true
|
||||
config = { key = "value" } # Passed to Plugin::init()
|
||||
```
|
||||
|
||||
### WASM Plugins (experimental)
|
||||
|
||||
```toml
|
||||
[plugins.wasm]
|
||||
enabled = true
|
||||
max_memory_mb = 64
|
||||
max_cpu_time_ms = 5000
|
||||
```
|
||||
|
||||
Load a `.wasm` binary at runtime via the gRPC API or by placing it in a search path. WASM plugins run sandboxed inside [wasmtime](https://wasmtime.dev/).
|
||||
|
||||
### Plugin API Version
|
||||
|
||||
Current: `0.1.0`. Breaking changes will increment the major version. MusicFS checks `musicfs_plugin_api_version()` before loading any native plugin.
|
||||
|
||||
---
|
||||
|
||||
## Control API (gRPC)
|
||||
|
||||
MusicFS exposes a gRPC API for programmatic control. The server starts automatically with the daemon.
|
||||
|
||||
**Default port:** `50052` (override with `--grpc-port`)
|
||||
**Proto definition:** `crates/musicfs-grpc/proto/musicfs.proto`
|
||||
|
||||
### Available RPCs
|
||||
|
||||
```
|
||||
MusicFS service:
|
||||
GetStatus → daemon version, uptime, mount state, open handles
|
||||
Shutdown → graceful or forced stop
|
||||
GetCacheStats → hit rate, chunk count, dedup ratio, per-tier breakdown
|
||||
ClearCache → clear all or per-origin, per-tier, dry-run supported
|
||||
Prefetch → pre-warm cache for paths or search queries
|
||||
ListOrigins → all configured origins with file count and health
|
||||
GetOriginHealth → health status and latency for one origin
|
||||
RescanOrigin → force re-scan with streaming progress
|
||||
Search → full-text search (paginated or streaming)
|
||||
SubscribeEvents → server-streaming live event feed
|
||||
|
||||
MetadataService:
|
||||
GetMetadata → all tags for a virtual path
|
||||
UpdateMetadata → set overlay tags for a file
|
||||
ClearOverlay → revert to original metadata
|
||||
ImportMetadata → bulk import from CSV/JSON (streaming progress)
|
||||
```
|
||||
|
||||
### Query with `grpcurl`
|
||||
|
||||
```bash
|
||||
# Daemon status
|
||||
grpcurl -plaintext localhost:50052 musicfs.v1.MusicFS/GetStatus
|
||||
|
||||
# Search
|
||||
grpcurl -plaintext -d '{"query": "metallica", "limit": 10}' \
|
||||
localhost:50052 musicfs.v1.MusicFS/Search
|
||||
|
||||
# Cache stats
|
||||
grpcurl -plaintext localhost:50052 musicfs.v1.MusicFS/GetCacheStats
|
||||
|
||||
# List origins
|
||||
grpcurl -plaintext localhost:50052 musicfs.v1.MusicFS/ListOrigins
|
||||
|
||||
# Trigger rescan with live progress
|
||||
grpcurl -plaintext -d '{"origin_id": "local"}' \
|
||||
localhost:50052 musicfs.v1.MusicFS/RescanOrigin
|
||||
|
||||
# Live event stream
|
||||
grpcurl -plaintext localhost:50052 musicfs.v1.MusicFS/SubscribeEvents
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Production Deployment
|
||||
|
||||
### systemd
|
||||
|
||||
```bash
|
||||
sudo cp dist/musicfs.service /etc/systemd/system/
|
||||
|
||||
# Edit the service to match your paths:
|
||||
# ExecStart=/usr/bin/musicfs mount --config /etc/musicfs/config.toml
|
||||
|
||||
sudo systemctl enable --now musicfs
|
||||
sudo systemctl status musicfs
|
||||
```
|
||||
|
||||
<!-- embedme dist/musicfs.service -->
|
||||
```ini
|
||||
[Unit]
|
||||
Description=MusicFS - Virtual FUSE Filesystem for Music
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/bin/musicfs mount /mnt/music --origin /path/to/music
|
||||
ExecStopPost=/usr/bin/fusermount -u /mnt/music
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
MusicFS sends `sd_notify(READY)` when the mount is live and `sd_notify(STOPPING)` during shutdown. Use `Type=notify` for precise readiness tracking.
|
||||
|
||||
### Signals
|
||||
|
||||
| Signal | Behavior |
|
||||
|--------|---------|
|
||||
| `SIGTERM` | Graceful shutdown — drains in-flight ops, unmounts |
|
||||
| `SIGINT` | Graceful shutdown (same) |
|
||||
| `SIGHUP` | Process pending file restores from trash |
|
||||
|
||||
### Security Notes
|
||||
|
||||
- Run as an **unprivileged user** — no root required.
|
||||
- Store remote credentials in the **system keyring** or environment variables. Never put them in the config file.
|
||||
- Credentials are redacted from logs and `RUST_LOG` output.
|
||||
- WASM plugins run sandboxed. Native `.so` plugins have full process access — only load plugins you trust.
|
||||
|
||||
---
|
||||
|
||||
## Observability
|
||||
|
||||
### Logs
|
||||
|
||||
```bash
|
||||
# Set level at startup
|
||||
musicfs mount ... --log-level debug
|
||||
# or via env
|
||||
RUST_LOG=musicfs=debug,warn musicfs mount ...
|
||||
```
|
||||
|
||||
| Level | Content |
|
||||
|-------|---------|
|
||||
| `error` | Unrecoverable failures, data corruption |
|
||||
| `warn` | Recoverable failures, origin timeouts, skipped files |
|
||||
| `info` | Mount/unmount, sync completion, config reload |
|
||||
| `debug` | Cache hits/misses, origin selection, file scans |
|
||||
| `trace` | Individual FUSE operations, chunk I/O |
|
||||
|
||||
Log files rotate daily in `log_dir` (default: `/var/log/musicfs/`). Structured JSON available with `json_output = true`. On Linux, logs forward to journald by default (`journald = true`).
|
||||
|
||||
### Prometheus Metrics
|
||||
|
||||
Metrics are exposed in Prometheus format via the gRPC API:
|
||||
|
||||
```
|
||||
musicfs_fuse_ops_total{op="read"} 152341
|
||||
musicfs_fuse_ops_total{op="readdir"} 8234
|
||||
musicfs_fuse_latency_seconds{op="read",quantile="0.99"} 0.004
|
||||
musicfs_cache_hits_total 142107
|
||||
musicfs_cache_misses_total 10234
|
||||
musicfs_cache_size_bytes 5368709120
|
||||
musicfs_origin_health{origin="local"} 1
|
||||
musicfs_origin_health{origin="s3"} 0
|
||||
musicfs_sync_files_changed{origin="local"} 15
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Performance
|
||||
|
||||
| Operation | Target | Maximum |
|
||||
|-----------|--------|---------|
|
||||
| Mount (any library size) | <100ms | 500ms |
|
||||
| `stat()` cached | <1ms | 5ms |
|
||||
| `readdir()` cached | <10ms | 50ms |
|
||||
| `open()` cached | <5ms | 20ms |
|
||||
| `read()` cached | <1ms | 5ms |
|
||||
| `read()` cache miss, local | <50ms | 200ms |
|
||||
| `read()` cache miss, remote | <200ms | 1000ms |
|
||||
| Search (1M tracks) | <500ms | 1000ms |
|
||||
| Sequential read (cached) | >500 MB/s | — |
|
||||
| Metadata ops | >1000 ops/s | — |
|
||||
|
||||
Memory: <50 MB idle, <200 MB with 1K files active, <500 MB peak.
|
||||
Scales to 10M+ files with O(1) mount and O(log n) lookups.
|
||||
|
||||
---
|
||||
|
||||
## Known Limitations
|
||||
|
||||
These are tracked issues — see `docs/v2/plans/` for details.
|
||||
|
||||
| Issue | Impact | Workaround |
|
||||
|-------|--------|-----------|
|
||||
| **No persistent state on mount** | Every restart does a full origin scan (O(N)). SQLite/search index persist but are not loaded on startup. | — |
|
||||
| **S3 and SFTP origins are stubs** | Only `local`, `nfs`, and `smb` have real implementations. | Use NFS/SMB mount as proxy for remote storage. |
|
||||
| **No write-through for metadata** | Overlaid metadata exists only in MusicFS's database, not in the actual audio files. | Use a tagger (beets, mp3tag) to write back if needed. |
|
||||
| **FUSE↔tokio deadlock risk** | `block_on()` in sync FUSE callbacks can stall under heavy concurrent load. | Keep concurrent open handles below ~500. |
|
||||
| **No background task supervision** | Health monitor, watcher, and indexer are fire-and-forget. A crash silently stops background work. | Restart the daemon periodically in critical deployments. |
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
MusicFS is a workspace of 11 Rust crates:
|
||||
|
||||
```
|
||||
musicfs-cli → binary, CLI parsing, startup wiring
|
||||
musicfs-fuse → FUSE operations (fuser), virtual tree serving
|
||||
musicfs-core → shared types, config, events, errors
|
||||
musicfs-cache → SQLite metadata DB, virtual tree, format handlers
|
||||
musicfs-cas → content-addressable chunk store (sled + xxHash64)
|
||||
musicfs-origins → origin backends (local, NFS, SMB, S3 stub, SFTP stub)
|
||||
musicfs-metadata → audio tag extraction (symphonia)
|
||||
musicfs-sync → delta sync, CDC chunking (FastCDC), inotify watcher
|
||||
musicfs-search → full-text index (tantivy), .search/ virtual dir
|
||||
musicfs-grpc → gRPC server (tonic + prost), proto codegen
|
||||
musicfs-plugins → plugin host, native .so loader, WASM sandbox
|
||||
```
|
||||
|
||||
Data flow on a cache miss: `FUSE read()` → `VirtualPathResolver` → `CAS` (chunk lookup) → `OriginFederation` (fetch missing range) → CDC chunk → store → return.
|
||||
|
||||
Full design: [`docs/v2/architecture.md`](docs/v2/architecture.md)
|
||||
Requirements: [`docs/v2/requirements.md`](docs/v2/requirements.md)
|
||||
Roadmap: [`docs/v2/development-plan.md`](docs/v2/development-plan.md)
|
||||
|
||||
---
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
nix develop # Enter dev shell
|
||||
|
||||
cargo check # Fast compile check
|
||||
cargo test # All 162 tests
|
||||
cargo test -p musicfs-core # Single crate
|
||||
cargo clippy # Lint
|
||||
cargo fmt # Format
|
||||
cargo nextest run # Parallel test runner (faster)
|
||||
cargo watch -x check -x test # Watch mode
|
||||
|
||||
# Cargo aliases
|
||||
cargo t # test
|
||||
cargo c # check
|
||||
cargo b # build
|
||||
|
||||
# gRPC codegen (runs via build.rs automatically)
|
||||
cargo build -p musicfs-grpc
|
||||
```
|
||||
|
||||
Pre-commit hooks (rustfmt + clippy) are installed automatically in the Nix dev shell.
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
MIT OR Apache-2.0 — see [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE).
|
||||
@@ -1,7 +0,0 @@
|
||||
Organising a music library can be a hassle. With the wealth of online stores all providing music tagged in various formats, it can be a nightmare to unify them all.
|
||||
|
||||
This is where beetFs comes in. Derived from beets, beetFs presents a FUSE filesystem that is based on your tags.
|
||||
|
||||
Modifying the tags within the beetFs mountpoint will not change the data on the hard disk, merely update the beet database. When an application requests a music file from within the beetFs mountpoint, beetFs provides tag information from its own database, instead of from the original file, but music data from the on-disk location.
|
||||
|
||||
This enables completely transparent modification of tags within an audio file with no change to the underlying on-disk data.
|
||||
@@ -1,2 +0,0 @@
|
||||
from pkgutil import extend_path
|
||||
__path__ = extend_path(__path__, __name__)
|
||||
-1144
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
||||
# MusicFS Configuration
|
||||
# Copy to /etc/musicfs/config.toml or ~/.config/musicfs/config.toml
|
||||
|
||||
# Required: where to mount the virtual filesystem
|
||||
mount_point = "/mnt/music"
|
||||
|
||||
# Required: directory for cache data (CAS chunks, metadata, search index)
|
||||
cache_dir = "/var/cache/musicfs"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Origins - music sources (at least one required)
|
||||
# Supported types: local, nfs, smb, s3, sftp
|
||||
# Lower priority number = preferred source for failover
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[[origins]]
|
||||
id = "local-music"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
enabled = true
|
||||
path = "/home/user/Music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas-nfs"
|
||||
origin_type = "nfs"
|
||||
priority = 2
|
||||
enabled = true
|
||||
path = "/mnt/nas/music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas-smb"
|
||||
origin_type = "smb"
|
||||
priority = 3
|
||||
enabled = false
|
||||
path = "/mnt/smb/music"
|
||||
|
||||
[[origins]]
|
||||
id = "cloud-backup"
|
||||
origin_type = "s3"
|
||||
priority = 10
|
||||
enabled = false
|
||||
bucket = "my-music-backup"
|
||||
region = "us-east-1"
|
||||
|
||||
[[origins]]
|
||||
id = "remote-server"
|
||||
origin_type = "sftp"
|
||||
priority = 10
|
||||
enabled = false
|
||||
host = "music.example.com"
|
||||
port = 22
|
||||
user = "musicfs"
|
||||
path = "/srv/music"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Cache settings
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[cache]
|
||||
# In-memory metadata cache size (artist/album/track info)
|
||||
metadata_cache_mb = 100
|
||||
|
||||
# On-disk content cache size (audio chunks)
|
||||
content_cache_gb = 10
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Health monitoring for origin failover
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[health]
|
||||
# How often to check origin health
|
||||
check_interval_secs = 30
|
||||
|
||||
# Timeout for health check probes
|
||||
timeout_ms = 5000
|
||||
|
||||
# Consecutive failures before marking origin unhealthy
|
||||
unhealthy_threshold = 3
|
||||
|
||||
# Per-origin type thresholds (overrides unhealthy_threshold)
|
||||
[health.per_origin_thresholds]
|
||||
local = 1
|
||||
nfs = 3
|
||||
smb = 3
|
||||
s3 = 3
|
||||
sftp = 3
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[logging]
|
||||
# Directory for log files
|
||||
log_dir = "/var/log/musicfs"
|
||||
|
||||
# Output logs as JSON (for log aggregators)
|
||||
json_output = false
|
||||
|
||||
# Send logs to systemd journal
|
||||
journald = true
|
||||
|
||||
# Log level filter (tracing format)
|
||||
# Examples: "info", "debug", "musicfs=debug,warn", "musicfs_fuse=trace"
|
||||
level = "musicfs=info,warn"
|
||||
|
||||
# Trace sampling rate for performance tracing (0.0 to 1.0)
|
||||
trace_sample_rate = 1.0
|
||||
@@ -1,12 +1,12 @@
|
||||
mount_point = "/mnt/music"
|
||||
cache_dir = "/var/cache/musicfs"
|
||||
mount_point = "./dev/music"
|
||||
cache_dir = "./dev/cache/musicfs"
|
||||
|
||||
[logging]
|
||||
log_dir = "/var/log/musicfs"
|
||||
json_output = true
|
||||
journald = true
|
||||
level = "musicfs=info,warn"
|
||||
trace_sample_rate = 1.0
|
||||
[[origins]]
|
||||
id = "local-storage"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
enabled = true
|
||||
path = "/home/fujin/.local/share/docker/volumes/containers_downloads/_data"
|
||||
|
||||
[cache]
|
||||
metadata_cache_mb = 100
|
||||
@@ -17,14 +17,9 @@ check_interval_secs = 30
|
||||
timeout_ms = 5000
|
||||
unhealthy_threshold = 3
|
||||
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/srv/music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas"
|
||||
origin_type = "nfs"
|
||||
priority = 2
|
||||
mount_point = "/mnt/nas/music"
|
||||
[logging]
|
||||
log_dir = "./dev/log"
|
||||
json_output = false
|
||||
journald = true
|
||||
level = "musicfs=info,warn"
|
||||
trace_sample_rate = 1.0
|
||||
@@ -7,6 +7,7 @@ edition.workspace = true
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-metadata = { path = "../musicfs-metadata" }
|
||||
bytes.workspace = true
|
||||
rusqlite = { workspace = true, features = ["bundled"] }
|
||||
sled.workspace = true
|
||||
tokio.workspace = true
|
||||
@@ -14,7 +15,9 @@ tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
image.workspace = true
|
||||
lofty = "0.24"
|
||||
parking_lot.workspace = true
|
||||
chrono.workspace = true
|
||||
|
||||
@@ -48,9 +48,18 @@ impl ArtworkCache {
|
||||
}
|
||||
|
||||
pub async fn store(&self, file_id: i64, artwork: &Artwork) -> Result<ChunkHash, ArtworkError> {
|
||||
trace!(file_id = file_id, size_bytes = artwork.data.len(), "Storing artwork");
|
||||
trace!(
|
||||
file_id = file_id,
|
||||
size_bytes = artwork.data.len(),
|
||||
"Storing artwork"
|
||||
);
|
||||
if artwork.data.len() > MAX_ARTWORK_INPUT_SIZE {
|
||||
warn!(file_id = file_id, size = artwork.data.len(), max = MAX_ARTWORK_INPUT_SIZE, "Artwork too large");
|
||||
warn!(
|
||||
file_id = file_id,
|
||||
size = artwork.data.len(),
|
||||
max = MAX_ARTWORK_INPUT_SIZE,
|
||||
"Artwork too large"
|
||||
);
|
||||
return Err(ArtworkError::ImageTooLarge(artwork.data.len()));
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
||||
use musicfs_cas::CasStore;
|
||||
use musicfs_core::ChunkHash;
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::RwLock;
|
||||
use std::time::Instant;
|
||||
use tracing::info;
|
||||
|
||||
@@ -64,8 +64,8 @@ impl Default for LruEviction {
|
||||
impl EvictionPolicy for LruEviction {
|
||||
fn record_access(&self, hash: ChunkHash) {
|
||||
let now = Instant::now();
|
||||
let mut times = self.access_times.write().unwrap();
|
||||
let mut h2t = self.hash_to_time.write().unwrap();
|
||||
let mut times = self.access_times.write();
|
||||
let mut h2t = self.hash_to_time.write();
|
||||
|
||||
if let Some(old_time) = h2t.remove(&hash) {
|
||||
times.remove(&old_time);
|
||||
@@ -76,13 +76,13 @@ impl EvictionPolicy for LruEviction {
|
||||
}
|
||||
|
||||
fn select_victims(&self, count: usize) -> Vec<ChunkHash> {
|
||||
let times = self.access_times.read().unwrap();
|
||||
let times = self.access_times.read();
|
||||
times.values().take(count).copied().collect()
|
||||
}
|
||||
|
||||
fn remove(&self, hash: &ChunkHash) {
|
||||
let mut times = self.access_times.write().unwrap();
|
||||
let mut h2t = self.hash_to_time.write().unwrap();
|
||||
let mut times = self.access_times.write();
|
||||
let mut h2t = self.hash_to_time.write();
|
||||
|
||||
if let Some(time) = h2t.remove(hash) {
|
||||
times.remove(&time);
|
||||
@@ -0,0 +1,103 @@
|
||||
use crate::FormatLayout;
|
||||
use musicfs_core::AudioMeta;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Error types for format handling operations
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FormatError {
|
||||
#[error("Unsupported format")]
|
||||
UnsupportedFormat,
|
||||
|
||||
#[error("Invalid data: {0}")]
|
||||
InvalidData(String),
|
||||
|
||||
#[error("Synthesis failed: {0}")]
|
||||
SynthesisFailed(String),
|
||||
}
|
||||
|
||||
/// Trait for format-specific metadata handling.
|
||||
///
|
||||
/// Implementations handle:
|
||||
/// 1. Analyzing original files to find audio boundaries
|
||||
/// 2. Synthesizing new headers from database metadata
|
||||
pub trait FormatHandler: Send + Sync + 'static {
|
||||
/// Unique identifier for this handler
|
||||
fn id(&self) -> &'static str;
|
||||
|
||||
/// Human-readable name
|
||||
fn name(&self) -> &'static str;
|
||||
|
||||
/// File extensions this handler supports
|
||||
fn extensions(&self) -> &[&'static str];
|
||||
|
||||
/// MIME types this handler supports
|
||||
fn mime_types(&self) -> &[&'static str];
|
||||
|
||||
/// Analyze file bytes to determine audio layout
|
||||
fn analyze(
|
||||
&self,
|
||||
data: &[u8],
|
||||
file_size: u64,
|
||||
) -> std::result::Result<FormatLayout, FormatError>;
|
||||
|
||||
/// Synthesize header bytes from metadata. Called on every read().
|
||||
fn synthesize(
|
||||
&self,
|
||||
metadata: &AudioMeta,
|
||||
layout: &FormatLayout,
|
||||
) -> std::result::Result<Vec<u8>, FormatError>;
|
||||
|
||||
/// Extract metadata from header bytes (for initial ingest)
|
||||
fn extract(&self, data: &[u8]) -> std::result::Result<AudioMeta, FormatError>;
|
||||
|
||||
/// Estimate header size without full synthesis (for getattr)
|
||||
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
|
||||
10 * 1024 // 10KB default
|
||||
}
|
||||
}
|
||||
|
||||
/// Registry for format handlers
|
||||
pub struct FormatHandlerRegistry {
|
||||
handlers: HashMap<String, Arc<dyn FormatHandler>>,
|
||||
extension_map: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl FormatHandlerRegistry {
|
||||
/// Create empty registry
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
handlers: HashMap::new(),
|
||||
extension_map: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a format handler
|
||||
pub fn register(&mut self, handler: Arc<dyn FormatHandler>) {
|
||||
let id = handler.id().to_string();
|
||||
|
||||
// Map extensions to handler ID
|
||||
for ext in handler.extensions() {
|
||||
self.extension_map.insert(ext.to_string(), id.clone());
|
||||
}
|
||||
|
||||
self.handlers.insert(id, handler);
|
||||
}
|
||||
|
||||
/// Get handler by file extension
|
||||
pub fn get_by_extension(&self, ext: &str) -> Option<Arc<dyn FormatHandler>> {
|
||||
let id = self.extension_map.get(ext)?;
|
||||
self.handlers.get(id).cloned()
|
||||
}
|
||||
|
||||
/// Get handler by format ID
|
||||
pub fn get_by_format(&self, format: &str) -> Option<Arc<dyn FormatHandler>> {
|
||||
self.handlers.get(format).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FormatHandlerRegistry {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
use musicfs_core::AudioFormat;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Describes the byte layout of an audio file for overlay splicing.
|
||||
///
|
||||
/// This struct tracks where the audio data begins and ends in the origin file,
|
||||
/// allowing the OverlayReader to splice synthetic headers with original audio.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FormatLayout {
|
||||
/// Byte offset where audio data begins in the origin file
|
||||
pub audio_start: u64,
|
||||
|
||||
/// Byte offset where audio data ends in the origin file
|
||||
pub audio_end: u64,
|
||||
|
||||
/// Audio format (from musicfs-core)
|
||||
pub format: AudioFormat,
|
||||
|
||||
/// Format-specific data (e.g., FLAC STREAMINFO block, MP4 stco offsets)
|
||||
/// Stored as raw bytes, interpreted by format handlers
|
||||
pub format_data: Option<Vec<u8>>,
|
||||
}
|
||||
@@ -0,0 +1,886 @@
|
||||
//! FLAC format handler for metadata synthesis.
|
||||
//!
|
||||
//! FLAC files use Vorbis comments for metadata. The file structure is:
|
||||
//! - "fLaC" marker (4 bytes)
|
||||
//! - STREAMINFO block (mandatory, 38 bytes total: 4 header + 34 data)
|
||||
//! - Optional metadata blocks (VORBIS_COMMENT, PICTURE, PADDING, etc.)
|
||||
//! - Audio frames
|
||||
//!
|
||||
//! CRITICAL: STREAMINFO must be preserved from the original file as it contains
|
||||
//! MD5 checksum, sample count, and audio properties that must match the audio data.
|
||||
|
||||
use crate::{FormatError, FormatHandler, FormatLayout};
|
||||
use lofty::config::ParseOptions;
|
||||
use lofty::file::AudioFile;
|
||||
use lofty::flac::FlacFile;
|
||||
use lofty::ogg::VorbisComments;
|
||||
use lofty::tag::Accessor;
|
||||
use musicfs_core::{AudioFormat, AudioMeta};
|
||||
use std::borrow::Cow;
|
||||
use std::io::Cursor;
|
||||
|
||||
/// FLAC stream marker: "fLaC" in ASCII
|
||||
const FLAC_MARKER: &[u8; 4] = b"fLaC";
|
||||
|
||||
/// FLAC metadata block types
|
||||
const BLOCK_TYPE_STREAMINFO: u8 = 0;
|
||||
const BLOCK_TYPE_VORBIS_COMMENT: u8 = 4;
|
||||
|
||||
/// STREAMINFO block data size (always 34 bytes)
|
||||
const STREAMINFO_DATA_SIZE: usize = 34;
|
||||
|
||||
/// Metadata block header size (1 byte type/flags + 3 bytes length)
|
||||
const BLOCK_HEADER_SIZE: usize = 4;
|
||||
|
||||
/// Full STREAMINFO block size (header + data)
|
||||
const STREAMINFO_BLOCK_SIZE: usize = BLOCK_HEADER_SIZE + STREAMINFO_DATA_SIZE;
|
||||
|
||||
pub struct FlacHandler;
|
||||
|
||||
impl FlacHandler {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
/// Parse FLAC metadata block header.
|
||||
/// Returns (is_last, block_type, block_size).
|
||||
fn parse_block_header(data: &[u8]) -> Option<(bool, u8, usize)> {
|
||||
if data.len() < BLOCK_HEADER_SIZE {
|
||||
return None;
|
||||
}
|
||||
let is_last = (data[0] & 0x80) != 0;
|
||||
let block_type = data[0] & 0x7F;
|
||||
let block_size =
|
||||
((data[1] as usize) << 16) | ((data[2] as usize) << 8) | (data[3] as usize);
|
||||
Some((is_last, block_type, block_size))
|
||||
}
|
||||
|
||||
/// Write a metadata block header.
|
||||
fn write_block_header(is_last: bool, block_type: u8, size: usize) -> [u8; 4] {
|
||||
let type_byte = if is_last {
|
||||
block_type | 0x80
|
||||
} else {
|
||||
block_type
|
||||
};
|
||||
[
|
||||
type_byte,
|
||||
((size >> 16) & 0xFF) as u8,
|
||||
((size >> 8) & 0xFF) as u8,
|
||||
(size & 0xFF) as u8,
|
||||
]
|
||||
}
|
||||
|
||||
/// Build Vorbis comments from AudioMeta.
|
||||
fn build_vorbis_comments(metadata: &AudioMeta) -> VorbisComments {
|
||||
let mut tag = VorbisComments::default();
|
||||
|
||||
// Basic fields (using Accessor trait)
|
||||
if let Some(ref title) = metadata.title {
|
||||
tag.set_title(title.clone());
|
||||
}
|
||||
if let Some(ref artist) = metadata.artist {
|
||||
tag.set_artist(artist.clone());
|
||||
}
|
||||
if let Some(ref album) = metadata.album {
|
||||
tag.set_album(album.clone());
|
||||
}
|
||||
if let Some(ref genre) = metadata.genre {
|
||||
tag.set_genre(genre.clone());
|
||||
}
|
||||
|
||||
// Album artist
|
||||
if let Some(ref album_artist) = metadata.album_artist {
|
||||
tag.insert("ALBUMARTIST".to_string(), album_artist.clone());
|
||||
}
|
||||
|
||||
// Year/Date
|
||||
if let Some(ref date) = metadata.date {
|
||||
tag.insert("DATE".to_string(), date.clone());
|
||||
} else if let Some(year) = metadata.year {
|
||||
tag.insert("DATE".to_string(), year.to_string());
|
||||
}
|
||||
|
||||
// Track/Disc numbers
|
||||
if let Some(track) = metadata.track {
|
||||
tag.insert("TRACKNUMBER".to_string(), track.to_string());
|
||||
}
|
||||
if let Some(track_total) = metadata.track_total {
|
||||
tag.insert("TRACKTOTAL".to_string(), track_total.to_string());
|
||||
}
|
||||
if let Some(disc) = metadata.disc {
|
||||
tag.insert("DISCNUMBER".to_string(), disc.to_string());
|
||||
}
|
||||
if let Some(disc_total) = metadata.disc_total {
|
||||
tag.insert("DISCTOTAL".to_string(), disc_total.to_string());
|
||||
}
|
||||
|
||||
// Extended metadata
|
||||
if let Some(ref composer) = metadata.composer {
|
||||
tag.insert("COMPOSER".to_string(), composer.clone());
|
||||
}
|
||||
if let Some(ref comment) = metadata.comment {
|
||||
tag.insert("COMMENT".to_string(), comment.clone());
|
||||
}
|
||||
if let Some(ref lyrics) = metadata.lyrics {
|
||||
tag.insert("LYRICS".to_string(), lyrics.clone());
|
||||
}
|
||||
if let Some(ref copyright) = metadata.copyright {
|
||||
tag.insert("COPYRIGHT".to_string(), copyright.clone());
|
||||
}
|
||||
if let Some(compilation) = metadata.compilation {
|
||||
tag.insert(
|
||||
"COMPILATION".to_string(),
|
||||
if compilation { "1" } else { "0" }.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Sort fields
|
||||
if let Some(ref title_sort) = metadata.title_sort {
|
||||
tag.insert("TITLESORT".to_string(), title_sort.clone());
|
||||
}
|
||||
if let Some(ref artist_sort) = metadata.artist_sort {
|
||||
tag.insert("ARTISTSORT".to_string(), artist_sort.clone());
|
||||
}
|
||||
if let Some(ref album_sort) = metadata.album_sort {
|
||||
tag.insert("ALBUMSORT".to_string(), album_sort.clone());
|
||||
}
|
||||
if let Some(ref album_artist_sort) = metadata.album_artist_sort {
|
||||
tag.insert("ALBUMARTISTSORT".to_string(), album_artist_sort.clone());
|
||||
}
|
||||
|
||||
// MusicBrainz IDs
|
||||
if let Some(ref mb_recording_id) = metadata.mb_recording_id {
|
||||
tag.insert("MUSICBRAINZ_TRACKID".to_string(), mb_recording_id.clone());
|
||||
}
|
||||
if let Some(ref mb_album_id) = metadata.mb_album_id {
|
||||
tag.insert("MUSICBRAINZ_ALBUMID".to_string(), mb_album_id.clone());
|
||||
}
|
||||
if let Some(ref mb_artist_id) = metadata.mb_artist_id {
|
||||
tag.insert("MUSICBRAINZ_ARTISTID".to_string(), mb_artist_id.clone());
|
||||
}
|
||||
if let Some(ref mb_album_artist_id) = metadata.mb_album_artist_id {
|
||||
tag.insert(
|
||||
"MUSICBRAINZ_ALBUMARTISTID".to_string(),
|
||||
mb_album_artist_id.clone(),
|
||||
);
|
||||
}
|
||||
if let Some(ref mb_release_group_id) = metadata.mb_release_group_id {
|
||||
tag.insert(
|
||||
"MUSICBRAINZ_RELEASEGROUPID".to_string(),
|
||||
mb_release_group_id.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
// ReplayGain
|
||||
if let Some(gain) = metadata.replaygain_track_gain {
|
||||
tag.insert(
|
||||
"REPLAYGAIN_TRACK_GAIN".to_string(),
|
||||
format!("{:.2} dB", gain),
|
||||
);
|
||||
}
|
||||
if let Some(peak) = metadata.replaygain_track_peak {
|
||||
tag.insert("REPLAYGAIN_TRACK_PEAK".to_string(), format!("{:.6}", peak));
|
||||
}
|
||||
if let Some(gain) = metadata.replaygain_album_gain {
|
||||
tag.insert(
|
||||
"REPLAYGAIN_ALBUM_GAIN".to_string(),
|
||||
format!("{:.2} dB", gain),
|
||||
);
|
||||
}
|
||||
if let Some(peak) = metadata.replaygain_album_peak {
|
||||
tag.insert("REPLAYGAIN_ALBUM_PEAK".to_string(), format!("{:.6}", peak));
|
||||
}
|
||||
|
||||
// Encoder
|
||||
if let Some(ref encoder) = metadata.encoder {
|
||||
tag.insert("ENCODER".to_string(), encoder.clone());
|
||||
}
|
||||
|
||||
tag
|
||||
}
|
||||
|
||||
/// Serialize Vorbis comments to bytes (without block header).
|
||||
/// Format: vendor_length (4 LE) + vendor + comment_count (4 LE) + comments
|
||||
fn serialize_vorbis_comments(tag: &VorbisComments) -> Vec<u8> {
|
||||
let vendor = tag.vendor();
|
||||
let vendor = if vendor.is_empty() { "musicfs" } else { vendor };
|
||||
let mut data = Vec::new();
|
||||
|
||||
// Vendor string (little-endian length + UTF-8 string)
|
||||
let vendor_bytes = vendor.as_bytes();
|
||||
data.extend_from_slice(&(vendor_bytes.len() as u32).to_le_bytes());
|
||||
data.extend_from_slice(vendor_bytes);
|
||||
|
||||
// Collect all comments
|
||||
let comments: Vec<_> = tag.items().collect();
|
||||
data.extend_from_slice(&(comments.len() as u32).to_le_bytes());
|
||||
|
||||
for (key, value) in comments {
|
||||
let comment = format!("{}={}", key, value);
|
||||
let comment_bytes = comment.as_bytes();
|
||||
data.extend_from_slice(&(comment_bytes.len() as u32).to_le_bytes());
|
||||
data.extend_from_slice(comment_bytes);
|
||||
}
|
||||
|
||||
data
|
||||
}
|
||||
|
||||
/// Extract metadata from Vorbis comments tag.
|
||||
fn extract_from_vorbis_comments(tag: &VorbisComments) -> AudioMeta {
|
||||
let mut meta = AudioMeta::default();
|
||||
meta.format = AudioFormat::Flac;
|
||||
|
||||
// Basic fields (using Accessor trait)
|
||||
meta.title = tag.title().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.artist = tag.artist().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.album = tag.album().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.genre = tag.genre().map(|c: Cow<'_, str>| c.into_owned());
|
||||
|
||||
// Album artist
|
||||
meta.album_artist = tag.get("ALBUMARTIST").map(String::from);
|
||||
|
||||
// Date/Year
|
||||
meta.date = tag.get("DATE").map(String::from);
|
||||
if let Some(ref date) = meta.date {
|
||||
if let Some(year_str) = date.split('-').next() {
|
||||
meta.year = year_str.parse().ok();
|
||||
}
|
||||
}
|
||||
|
||||
// Track/Disc numbers
|
||||
meta.track = tag.get("TRACKNUMBER").and_then(|s| s.parse().ok());
|
||||
meta.track_total = tag.get("TRACKTOTAL").and_then(|s| s.parse().ok());
|
||||
meta.disc = tag.get("DISCNUMBER").and_then(|s| s.parse().ok());
|
||||
meta.disc_total = tag.get("DISCTOTAL").and_then(|s| s.parse().ok());
|
||||
|
||||
// Extended metadata
|
||||
meta.composer = tag.get("COMPOSER").map(String::from);
|
||||
meta.comment = tag.get("COMMENT").map(String::from);
|
||||
meta.lyrics = tag.get("LYRICS").map(String::from);
|
||||
meta.copyright = tag.get("COPYRIGHT").map(String::from);
|
||||
meta.compilation = tag
|
||||
.get("COMPILATION")
|
||||
.map(|s| s == "1" || s.eq_ignore_ascii_case("true"));
|
||||
|
||||
// Sort fields
|
||||
meta.title_sort = tag.get("TITLESORT").map(String::from);
|
||||
meta.artist_sort = tag.get("ARTISTSORT").map(String::from);
|
||||
meta.album_sort = tag.get("ALBUMSORT").map(String::from);
|
||||
meta.album_artist_sort = tag.get("ALBUMARTISTSORT").map(String::from);
|
||||
|
||||
// MusicBrainz IDs
|
||||
meta.mb_recording_id = tag.get("MUSICBRAINZ_TRACKID").map(String::from);
|
||||
meta.mb_album_id = tag.get("MUSICBRAINZ_ALBUMID").map(String::from);
|
||||
meta.mb_artist_id = tag.get("MUSICBRAINZ_ARTISTID").map(String::from);
|
||||
meta.mb_album_artist_id = tag.get("MUSICBRAINZ_ALBUMARTISTID").map(String::from);
|
||||
meta.mb_release_group_id = tag.get("MUSICBRAINZ_RELEASEGROUPID").map(String::from);
|
||||
|
||||
// ReplayGain
|
||||
meta.replaygain_track_gain = tag
|
||||
.get("REPLAYGAIN_TRACK_GAIN")
|
||||
.and_then(|s| Self::parse_replaygain_value(s));
|
||||
meta.replaygain_track_peak = tag
|
||||
.get("REPLAYGAIN_TRACK_PEAK")
|
||||
.and_then(|s| s.parse().ok());
|
||||
meta.replaygain_album_gain = tag
|
||||
.get("REPLAYGAIN_ALBUM_GAIN")
|
||||
.and_then(|s| Self::parse_replaygain_value(s));
|
||||
meta.replaygain_album_peak = tag
|
||||
.get("REPLAYGAIN_ALBUM_PEAK")
|
||||
.and_then(|s| s.parse().ok());
|
||||
|
||||
// Encoder
|
||||
meta.encoder = tag.get("ENCODER").map(String::from);
|
||||
|
||||
meta
|
||||
}
|
||||
|
||||
/// Parse ReplayGain value, stripping optional "dB" suffix.
|
||||
fn parse_replaygain_value(value: &str) -> Option<f32> {
|
||||
value
|
||||
.trim()
|
||||
.trim_end_matches(" dB")
|
||||
.trim_end_matches("dB")
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FlacHandler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl FormatHandler for FlacHandler {
|
||||
fn id(&self) -> &'static str {
|
||||
"flac"
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"FLAC"
|
||||
}
|
||||
|
||||
fn extensions(&self) -> &[&'static str] {
|
||||
&["flac"]
|
||||
}
|
||||
|
||||
fn mime_types(&self) -> &[&'static str] {
|
||||
&["audio/flac", "audio/x-flac"]
|
||||
}
|
||||
|
||||
fn analyze(&self, data: &[u8], file_size: u64) -> Result<FormatLayout, FormatError> {
|
||||
// Verify FLAC marker
|
||||
if data.len() < FLAC_MARKER.len() || &data[0..4] != FLAC_MARKER {
|
||||
return Err(FormatError::InvalidData("Not a FLAC file".to_string()));
|
||||
}
|
||||
|
||||
let mut offset = FLAC_MARKER.len();
|
||||
let mut streaminfo_data: Option<Vec<u8>> = None;
|
||||
|
||||
// Parse metadata blocks to find audio_start and extract STREAMINFO
|
||||
loop {
|
||||
if offset + BLOCK_HEADER_SIZE > data.len() {
|
||||
return Err(FormatError::InvalidData(
|
||||
"Truncated FLAC metadata".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let (is_last, block_type, block_size) = Self::parse_block_header(&data[offset..])
|
||||
.ok_or_else(|| FormatError::InvalidData("Invalid block header".to_string()))?;
|
||||
|
||||
// Extract STREAMINFO block data (without header)
|
||||
if block_type == BLOCK_TYPE_STREAMINFO {
|
||||
if block_size != STREAMINFO_DATA_SIZE {
|
||||
return Err(FormatError::InvalidData(format!(
|
||||
"Invalid STREAMINFO size: {} (expected {})",
|
||||
block_size, STREAMINFO_DATA_SIZE
|
||||
)));
|
||||
}
|
||||
let data_start = offset + BLOCK_HEADER_SIZE;
|
||||
let data_end = data_start + block_size;
|
||||
if data_end > data.len() {
|
||||
return Err(FormatError::InvalidData(
|
||||
"Truncated STREAMINFO block".to_string(),
|
||||
));
|
||||
}
|
||||
streaminfo_data = Some(data[data_start..data_end].to_vec());
|
||||
}
|
||||
|
||||
offset += BLOCK_HEADER_SIZE + block_size;
|
||||
|
||||
if is_last {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let streaminfo = streaminfo_data
|
||||
.ok_or_else(|| FormatError::InvalidData("Missing STREAMINFO block".to_string()))?;
|
||||
|
||||
Ok(FormatLayout {
|
||||
audio_start: offset as u64,
|
||||
audio_end: file_size,
|
||||
format: AudioFormat::Flac,
|
||||
format_data: Some(streaminfo),
|
||||
})
|
||||
}
|
||||
|
||||
fn synthesize(
|
||||
&self,
|
||||
metadata: &AudioMeta,
|
||||
layout: &FormatLayout,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
// STREAMINFO must be preserved from original
|
||||
let streaminfo_data = layout.format_data.as_ref().ok_or_else(|| {
|
||||
FormatError::SynthesisFailed("Missing STREAMINFO data in layout".to_string())
|
||||
})?;
|
||||
|
||||
if streaminfo_data.len() != STREAMINFO_DATA_SIZE {
|
||||
return Err(FormatError::SynthesisFailed(format!(
|
||||
"Invalid STREAMINFO size: {} (expected {})",
|
||||
streaminfo_data.len(),
|
||||
STREAMINFO_DATA_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
// Build Vorbis comments
|
||||
let vorbis_tag = Self::build_vorbis_comments(metadata);
|
||||
let vorbis_data = Self::serialize_vorbis_comments(&vorbis_tag);
|
||||
|
||||
// Calculate total header size
|
||||
let total_size =
|
||||
FLAC_MARKER.len() + STREAMINFO_BLOCK_SIZE + BLOCK_HEADER_SIZE + vorbis_data.len();
|
||||
let mut buffer = Vec::with_capacity(total_size);
|
||||
|
||||
// Write FLAC marker
|
||||
buffer.extend_from_slice(FLAC_MARKER);
|
||||
|
||||
// Write STREAMINFO block (not last)
|
||||
let streaminfo_header =
|
||||
Self::write_block_header(false, BLOCK_TYPE_STREAMINFO, STREAMINFO_DATA_SIZE);
|
||||
buffer.extend_from_slice(&streaminfo_header);
|
||||
buffer.extend_from_slice(streaminfo_data);
|
||||
|
||||
// Write VORBIS_COMMENT block (last)
|
||||
let vorbis_header =
|
||||
Self::write_block_header(true, BLOCK_TYPE_VORBIS_COMMENT, vorbis_data.len());
|
||||
buffer.extend_from_slice(&vorbis_header);
|
||||
buffer.extend_from_slice(&vorbis_data);
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
fn extract(&self, data: &[u8]) -> Result<AudioMeta, FormatError> {
|
||||
let mut cursor = Cursor::new(data);
|
||||
|
||||
let flac_file = FlacFile::read_from(&mut cursor, ParseOptions::new())
|
||||
.map_err(|e| FormatError::InvalidData(e.to_string()))?;
|
||||
|
||||
let tag = flac_file
|
||||
.vorbis_comments()
|
||||
.ok_or_else(|| FormatError::InvalidData("No Vorbis comments found".to_string()))?;
|
||||
|
||||
Ok(Self::extract_from_vorbis_comments(tag))
|
||||
}
|
||||
|
||||
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
|
||||
// fLaC (4) + STREAMINFO (38) + VORBIS_COMMENT header (4) + typical comments (~4KB)
|
||||
8192
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_test_meta() -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: Some("Test Title".to_string()),
|
||||
artist: Some("Test Artist".to_string()),
|
||||
album: Some("Test Album".to_string()),
|
||||
album_artist: Some("Test Album Artist".to_string()),
|
||||
genre: Some("Rock".to_string()),
|
||||
year: Some(2024),
|
||||
track: Some(5),
|
||||
track_total: Some(12),
|
||||
disc: Some(1),
|
||||
disc_total: Some(2),
|
||||
format: AudioFormat::Flac,
|
||||
date: Some("2024-03-15".to_string()),
|
||||
composer: Some("Test Composer".to_string()),
|
||||
comment: Some("Test Comment".to_string()),
|
||||
lyrics: Some("Test Lyrics\nLine 2".to_string()),
|
||||
copyright: Some("2024 Test Copyright".to_string()),
|
||||
compilation: Some(false),
|
||||
title_sort: Some("Title, Test".to_string()),
|
||||
artist_sort: Some("Artist, Test".to_string()),
|
||||
album_sort: Some("Album, Test".to_string()),
|
||||
album_artist_sort: Some("Album Artist, Test".to_string()),
|
||||
mb_recording_id: Some("rec-12345".to_string()),
|
||||
mb_album_id: Some("alb-12345".to_string()),
|
||||
mb_artist_id: Some("art-12345".to_string()),
|
||||
mb_album_artist_id: Some("albart-12345".to_string()),
|
||||
mb_release_group_id: Some("rg-12345".to_string()),
|
||||
replaygain_track_gain: Some(-6.5),
|
||||
replaygain_track_peak: Some(0.987654),
|
||||
replaygain_album_gain: Some(-5.2),
|
||||
replaygain_album_peak: Some(0.999999),
|
||||
encoder: Some("FLAC 1.4.0".to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a minimal valid FLAC file header for testing.
|
||||
fn make_minimal_flac_header() -> Vec<u8> {
|
||||
let mut data = Vec::new();
|
||||
|
||||
// FLAC marker
|
||||
data.extend_from_slice(b"fLaC");
|
||||
|
||||
// STREAMINFO block (last=true for minimal file)
|
||||
// Header: type=0 (STREAMINFO), last=1, size=34
|
||||
data.push(0x80); // 0x80 = last flag set, type 0
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x22); // 34 bytes
|
||||
|
||||
// STREAMINFO data (34 bytes) - minimal valid values
|
||||
// min_block_size (16 bits) = 4096
|
||||
data.push(0x10);
|
||||
data.push(0x00);
|
||||
// max_block_size (16 bits) = 4096
|
||||
data.push(0x10);
|
||||
data.push(0x00);
|
||||
// min_frame_size (24 bits) = 0 (unknown)
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
// max_frame_size (24 bits) = 0 (unknown)
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
// sample_rate (20 bits) = 44100, channels-1 (3 bits) = 1, bits-1 (5 bits) = 15
|
||||
// 44100 = 0xAC44, channels=2 (1), bits=16 (15)
|
||||
// Packed: SSSS SSSS SSSS SSSS SSSS CCCC CBBB BB
|
||||
// 0xAC44 << 12 | (1 << 9) | (15 << 4) = ...
|
||||
// Let's use simpler encoding:
|
||||
// Byte 0-1: sample_rate high 16 bits of 20
|
||||
// Byte 2: sample_rate low 4 bits | channels 3 bits | bits high 1 bit
|
||||
// Byte 3: bits low 4 bits | total_samples high 4 bits
|
||||
// Actually the format is:
|
||||
// 20 bits sample rate, 3 bits channels-1, 5 bits bits-1, 36 bits total samples
|
||||
// 44100 = 0x0AC44
|
||||
data.push(0x0A); // sample_rate bits 19-12
|
||||
data.push(0xC4); // sample_rate bits 11-4
|
||||
data.push(0x42); // sample_rate bits 3-0 (0x4), channels-1 (0x1=stereo), bits-1 high bit (0)
|
||||
data.push(0xF0); // bits-1 low 4 bits (0xF=15, so 16 bits), total_samples high 4 bits (0)
|
||||
// total_samples (remaining 32 bits) = 0
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
// MD5 signature (128 bits = 16 bytes)
|
||||
data.extend_from_slice(&[0u8; 16]);
|
||||
|
||||
data
|
||||
}
|
||||
|
||||
/// Create a FLAC header with Vorbis comments for testing extract().
|
||||
fn make_flac_with_vorbis_comments() -> Vec<u8> {
|
||||
let mut data = Vec::new();
|
||||
|
||||
// FLAC marker
|
||||
data.extend_from_slice(b"fLaC");
|
||||
|
||||
// STREAMINFO block (not last)
|
||||
data.push(0x00); // type=0, last=0
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x22); // 34 bytes
|
||||
|
||||
// STREAMINFO data (34 bytes)
|
||||
data.push(0x10);
|
||||
data.push(0x00);
|
||||
data.push(0x10);
|
||||
data.push(0x00);
|
||||
data.extend_from_slice(&[0u8; 6]); // frame sizes
|
||||
data.push(0x0A);
|
||||
data.push(0xC4);
|
||||
data.push(0x42);
|
||||
data.push(0xF0);
|
||||
data.extend_from_slice(&[0u8; 4]); // total samples
|
||||
data.extend_from_slice(&[0u8; 16]); // MD5
|
||||
|
||||
// VORBIS_COMMENT block (last)
|
||||
// Vendor: "test"
|
||||
// Comments: TITLE=Test Song, ARTIST=Test Artist
|
||||
let vendor = b"test";
|
||||
let comments = [
|
||||
b"TITLE=Test Song".as_slice(),
|
||||
b"ARTIST=Test Artist".as_slice(),
|
||||
b"ALBUM=Test Album".as_slice(),
|
||||
b"TRACKNUMBER=3".as_slice(),
|
||||
b"REPLAYGAIN_TRACK_GAIN=-5.50 dB".as_slice(),
|
||||
];
|
||||
|
||||
let mut vorbis_data = Vec::new();
|
||||
// Vendor length (LE)
|
||||
vorbis_data.extend_from_slice(&(vendor.len() as u32).to_le_bytes());
|
||||
vorbis_data.extend_from_slice(vendor);
|
||||
// Comment count (LE)
|
||||
vorbis_data.extend_from_slice(&(comments.len() as u32).to_le_bytes());
|
||||
for comment in &comments {
|
||||
vorbis_data.extend_from_slice(&(comment.len() as u32).to_le_bytes());
|
||||
vorbis_data.extend_from_slice(*comment);
|
||||
}
|
||||
|
||||
// VORBIS_COMMENT header
|
||||
data.push(0x84); // type=4, last=1
|
||||
data.push(((vorbis_data.len() >> 16) & 0xFF) as u8);
|
||||
data.push(((vorbis_data.len() >> 8) & 0xFF) as u8);
|
||||
data.push((vorbis_data.len() & 0xFF) as u8);
|
||||
data.extend_from_slice(&vorbis_data);
|
||||
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_id_and_name() {
|
||||
let handler = FlacHandler::new();
|
||||
assert_eq!(handler.id(), "flac");
|
||||
assert_eq!(handler.name(), "FLAC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extensions_and_mime_types() {
|
||||
let handler = FlacHandler::new();
|
||||
assert_eq!(handler.extensions(), &["flac"]);
|
||||
assert_eq!(handler.mime_types(), &["audio/flac", "audio/x-flac"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_header_size() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = AudioMeta::default();
|
||||
assert_eq!(handler.estimate_header_size(&meta), 8192);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_valid_flac() {
|
||||
let handler = FlacHandler::new();
|
||||
let data = make_minimal_flac_header();
|
||||
let file_size = data.len() as u64 + 1000; // Pretend there's audio data
|
||||
|
||||
let result = handler.analyze(&data, file_size);
|
||||
assert!(result.is_ok(), "analyze failed: {:?}", result.err());
|
||||
|
||||
let layout = result.unwrap();
|
||||
assert_eq!(layout.audio_start, 42); // 4 (marker) + 38 (STREAMINFO)
|
||||
assert_eq!(layout.audio_end, file_size);
|
||||
assert_eq!(layout.format, AudioFormat::Flac);
|
||||
assert!(layout.format_data.is_some());
|
||||
assert_eq!(
|
||||
layout.format_data.as_ref().unwrap().len(),
|
||||
STREAMINFO_DATA_SIZE
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_invalid_marker() {
|
||||
let handler = FlacHandler::new();
|
||||
let data = b"ID3\x04\x00\x00"; // MP3 header, not FLAC
|
||||
|
||||
let result = handler.analyze(data, 1000);
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), FormatError::InvalidData(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_truncated() {
|
||||
let handler = FlacHandler::new();
|
||||
let data = b"fLaC"; // Just the marker, no blocks
|
||||
|
||||
let result = handler.analyze(data, 4);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthesize_creates_valid_flac_header() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = make_test_meta();
|
||||
|
||||
// Create layout with STREAMINFO
|
||||
let original_data = make_minimal_flac_header();
|
||||
let layout = handler
|
||||
.analyze(&original_data, original_data.len() as u64)
|
||||
.unwrap();
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok(), "synthesize failed: {:?}", result.err());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
|
||||
// Verify FLAC marker
|
||||
assert!(bytes.len() >= 4);
|
||||
assert_eq!(&bytes[0..4], b"fLaC");
|
||||
|
||||
// Verify STREAMINFO block header
|
||||
assert_eq!(bytes[4] & 0x7F, BLOCK_TYPE_STREAMINFO); // Type 0
|
||||
assert_eq!(bytes[4] & 0x80, 0); // Not last
|
||||
|
||||
// Verify STREAMINFO size
|
||||
let streaminfo_size =
|
||||
((bytes[5] as usize) << 16) | ((bytes[6] as usize) << 8) | (bytes[7] as usize);
|
||||
assert_eq!(streaminfo_size, STREAMINFO_DATA_SIZE);
|
||||
|
||||
// Verify VORBIS_COMMENT block follows
|
||||
let vorbis_offset = 4 + 4 + STREAMINFO_DATA_SIZE;
|
||||
assert_eq!(bytes[vorbis_offset] & 0x7F, BLOCK_TYPE_VORBIS_COMMENT);
|
||||
assert_eq!(bytes[vorbis_offset] & 0x80, 0x80); // Is last
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthesize_preserves_streaminfo() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = AudioMeta::default();
|
||||
|
||||
// Create layout with specific STREAMINFO
|
||||
let original_data = make_minimal_flac_header();
|
||||
let layout = handler
|
||||
.analyze(&original_data, original_data.len() as u64)
|
||||
.unwrap();
|
||||
let original_streaminfo = layout.format_data.as_ref().unwrap().clone();
|
||||
|
||||
let synthesized = handler.synthesize(&meta, &layout).unwrap();
|
||||
|
||||
// Extract STREAMINFO from synthesized header
|
||||
let streaminfo_start = 4 + 4; // After marker and header
|
||||
let streaminfo_end = streaminfo_start + STREAMINFO_DATA_SIZE;
|
||||
let synthesized_streaminfo = &synthesized[streaminfo_start..streaminfo_end];
|
||||
|
||||
assert_eq!(synthesized_streaminfo, original_streaminfo.as_slice());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthesize_missing_streaminfo() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = AudioMeta::default();
|
||||
let layout = FormatLayout {
|
||||
audio_start: 42,
|
||||
audio_end: 1000,
|
||||
format: AudioFormat::Flac,
|
||||
format_data: None, // Missing STREAMINFO
|
||||
};
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(
|
||||
result.unwrap_err(),
|
||||
FormatError::SynthesisFailed(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_from_flac() {
|
||||
let handler = FlacHandler::new();
|
||||
let data = make_flac_with_vorbis_comments();
|
||||
|
||||
let result = handler.extract(&data);
|
||||
assert!(result.is_ok(), "extract failed: {:?}", result.err());
|
||||
|
||||
let meta = result.unwrap();
|
||||
assert_eq!(meta.title, Some("Test Song".to_string()));
|
||||
assert_eq!(meta.artist, Some("Test Artist".to_string()));
|
||||
assert_eq!(meta.album, Some("Test Album".to_string()));
|
||||
assert_eq!(meta.track, Some(3));
|
||||
assert_eq!(meta.format, AudioFormat::Flac);
|
||||
|
||||
// Check ReplayGain parsing
|
||||
let gain = meta.replaygain_track_gain.unwrap();
|
||||
assert!((gain - (-5.5)).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_and_extract_vorbis_comments() {
|
||||
let original_meta = make_test_meta();
|
||||
let tag = FlacHandler::build_vorbis_comments(&original_meta);
|
||||
let extracted = FlacHandler::extract_from_vorbis_comments(&tag);
|
||||
|
||||
assert_eq!(extracted.title, original_meta.title);
|
||||
assert_eq!(extracted.artist, original_meta.artist);
|
||||
assert_eq!(extracted.album, original_meta.album);
|
||||
assert_eq!(extracted.album_artist, original_meta.album_artist);
|
||||
assert_eq!(extracted.genre, original_meta.genre);
|
||||
assert_eq!(extracted.track, original_meta.track);
|
||||
assert_eq!(extracted.track_total, original_meta.track_total);
|
||||
assert_eq!(extracted.disc, original_meta.disc);
|
||||
assert_eq!(extracted.disc_total, original_meta.disc_total);
|
||||
assert_eq!(extracted.composer, original_meta.composer);
|
||||
assert_eq!(extracted.comment, original_meta.comment);
|
||||
assert_eq!(extracted.lyrics, original_meta.lyrics);
|
||||
assert_eq!(extracted.copyright, original_meta.copyright);
|
||||
assert_eq!(extracted.compilation, original_meta.compilation);
|
||||
assert_eq!(extracted.title_sort, original_meta.title_sort);
|
||||
assert_eq!(extracted.artist_sort, original_meta.artist_sort);
|
||||
assert_eq!(extracted.album_sort, original_meta.album_sort);
|
||||
assert_eq!(extracted.album_artist_sort, original_meta.album_artist_sort);
|
||||
assert_eq!(extracted.mb_recording_id, original_meta.mb_recording_id);
|
||||
assert_eq!(extracted.mb_album_id, original_meta.mb_album_id);
|
||||
assert_eq!(extracted.mb_artist_id, original_meta.mb_artist_id);
|
||||
assert_eq!(
|
||||
extracted.mb_album_artist_id,
|
||||
original_meta.mb_album_artist_id
|
||||
);
|
||||
assert_eq!(
|
||||
extracted.mb_release_group_id,
|
||||
original_meta.mb_release_group_id
|
||||
);
|
||||
assert_eq!(extracted.encoder, original_meta.encoder);
|
||||
|
||||
// ReplayGain values (with tolerance for formatting)
|
||||
let orig_track_gain = original_meta.replaygain_track_gain.unwrap();
|
||||
let ext_track_gain = extracted.replaygain_track_gain.unwrap();
|
||||
assert!((orig_track_gain - ext_track_gain).abs() < 0.01);
|
||||
|
||||
let orig_track_peak = original_meta.replaygain_track_peak.unwrap();
|
||||
let ext_track_peak = extracted.replaygain_track_peak.unwrap();
|
||||
assert!((orig_track_peak - ext_track_peak).abs() < 0.0001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_replaygain_value() {
|
||||
assert_eq!(FlacHandler::parse_replaygain_value("-6.50 dB"), Some(-6.50));
|
||||
assert_eq!(FlacHandler::parse_replaygain_value("-6.50dB"), Some(-6.50));
|
||||
assert_eq!(FlacHandler::parse_replaygain_value("-6.50"), Some(-6.50));
|
||||
assert_eq!(FlacHandler::parse_replaygain_value(" 3.2 dB "), Some(3.2));
|
||||
assert_eq!(FlacHandler::parse_replaygain_value("invalid"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_block_header() {
|
||||
// Not last, type 0, size 34
|
||||
let header = [0x00, 0x00, 0x00, 0x22];
|
||||
let (is_last, block_type, size) = FlacHandler::parse_block_header(&header).unwrap();
|
||||
assert!(!is_last);
|
||||
assert_eq!(block_type, 0);
|
||||
assert_eq!(size, 34);
|
||||
|
||||
// Last, type 4, size 256
|
||||
let header = [0x84, 0x00, 0x01, 0x00];
|
||||
let (is_last, block_type, size) = FlacHandler::parse_block_header(&header).unwrap();
|
||||
assert!(is_last);
|
||||
assert_eq!(block_type, 4);
|
||||
assert_eq!(size, 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_block_header() {
|
||||
let header = FlacHandler::write_block_header(false, 0, 34);
|
||||
assert_eq!(header, [0x00, 0x00, 0x00, 0x22]);
|
||||
|
||||
let header = FlacHandler::write_block_header(true, 4, 256);
|
||||
assert_eq!(header, [0x84, 0x00, 0x01, 0x00]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_metadata_produces_minimal_vorbis() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = AudioMeta::default();
|
||||
|
||||
let original_data = make_minimal_flac_header();
|
||||
let layout = handler
|
||||
.analyze(&original_data, original_data.len() as u64)
|
||||
.unwrap();
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
// Should have: fLaC (4) + STREAMINFO (38) + VORBIS_COMMENT (header + minimal data)
|
||||
assert!(bytes.len() >= 42 + 4 + 8); // At least vendor string overhead
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_round_trip_synthesize_analyze() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = make_test_meta();
|
||||
|
||||
// Create initial layout
|
||||
let original_data = make_minimal_flac_header();
|
||||
let layout = handler
|
||||
.analyze(&original_data, original_data.len() as u64)
|
||||
.unwrap();
|
||||
|
||||
// Synthesize new header
|
||||
let synthesized = handler.synthesize(&meta, &layout).unwrap();
|
||||
|
||||
// Analyze synthesized header
|
||||
let new_layout = handler
|
||||
.analyze(&synthesized, synthesized.len() as u64)
|
||||
.unwrap();
|
||||
|
||||
// STREAMINFO should be preserved
|
||||
assert_eq!(new_layout.format_data, layout.format_data);
|
||||
assert_eq!(new_layout.format, AudioFormat::Flac);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,631 @@
|
||||
use crate::{FormatError, FormatHandler, FormatLayout};
|
||||
use lofty::config::{ParseOptions, WriteOptions};
|
||||
use lofty::file::AudioFile;
|
||||
use lofty::id3::v2::{
|
||||
CommentFrame, Frame, FrameId, Id3v2Tag, TextInformationFrame, UnsynchronizedTextFrame,
|
||||
};
|
||||
use lofty::mpeg::MpegFile;
|
||||
use lofty::tag::{Accessor, TagExt};
|
||||
use lofty::TextEncoding;
|
||||
use musicfs_core::{AudioFormat, AudioMeta};
|
||||
use std::borrow::Cow;
|
||||
use std::io::Cursor;
|
||||
|
||||
const ID3V2_HEADER_SIZE: usize = 10;
|
||||
const ID3V1_TAG_SIZE: usize = 128;
|
||||
|
||||
pub struct Id3v2Handler;
|
||||
|
||||
impl Id3v2Handler {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
fn parse_id3v2_header(data: &[u8]) -> Option<usize> {
|
||||
if data.len() < ID3V2_HEADER_SIZE {
|
||||
return None;
|
||||
}
|
||||
|
||||
if &data[0..3] != b"ID3" {
|
||||
return None;
|
||||
}
|
||||
|
||||
let size = syncsafe_decode(&data[6..10]);
|
||||
Some(ID3V2_HEADER_SIZE + size)
|
||||
}
|
||||
|
||||
fn has_id3v1_tag(data: &[u8], file_size: u64) -> bool {
|
||||
if file_size < ID3V1_TAG_SIZE as u64 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let tag_start = (file_size as usize).saturating_sub(ID3V1_TAG_SIZE);
|
||||
if tag_start >= data.len() {
|
||||
return false;
|
||||
}
|
||||
|
||||
&data[tag_start..tag_start + 3] == b"TAG"
|
||||
}
|
||||
|
||||
fn set_text_frame(tag: &mut Id3v2Tag, frame_id: &'static str, value: &str) {
|
||||
let id = FrameId::Valid(Cow::Borrowed(frame_id));
|
||||
let frame = Frame::Text(TextInformationFrame::new(
|
||||
id,
|
||||
TextEncoding::UTF8,
|
||||
value.to_string(),
|
||||
));
|
||||
tag.insert(frame);
|
||||
}
|
||||
|
||||
fn set_track_disc_frame(
|
||||
tag: &mut Id3v2Tag,
|
||||
frame_id: &'static str,
|
||||
num: u32,
|
||||
total: Option<u32>,
|
||||
) {
|
||||
let value = match total {
|
||||
Some(t) => format!("{}/{}", num, t),
|
||||
None => num.to_string(),
|
||||
};
|
||||
Self::set_text_frame(tag, frame_id, &value);
|
||||
}
|
||||
|
||||
fn set_comment_frame(tag: &mut Id3v2Tag, value: &str) {
|
||||
let frame = Frame::Comment(CommentFrame::new(
|
||||
TextEncoding::UTF8,
|
||||
*b"eng",
|
||||
String::new(),
|
||||
value.to_string(),
|
||||
));
|
||||
tag.insert(frame);
|
||||
}
|
||||
|
||||
fn set_lyrics_frame(tag: &mut Id3v2Tag, value: &str) {
|
||||
let frame = Frame::UnsynchronizedText(UnsynchronizedTextFrame::new(
|
||||
TextEncoding::UTF8,
|
||||
*b"eng",
|
||||
String::new(),
|
||||
value.to_string(),
|
||||
));
|
||||
tag.insert(frame);
|
||||
}
|
||||
|
||||
fn build_tag_from_meta(metadata: &AudioMeta) -> Id3v2Tag {
|
||||
let mut tag = Id3v2Tag::new();
|
||||
|
||||
if let Some(ref title) = metadata.title {
|
||||
tag.set_title(title.clone());
|
||||
}
|
||||
if let Some(ref artist) = metadata.artist {
|
||||
tag.set_artist(artist.clone());
|
||||
}
|
||||
if let Some(ref album) = metadata.album {
|
||||
tag.set_album(album.clone());
|
||||
}
|
||||
if let Some(ref album_artist) = metadata.album_artist {
|
||||
Self::set_text_frame(&mut tag, "TPE2", album_artist);
|
||||
}
|
||||
if let Some(year) = metadata.year {
|
||||
Self::set_text_frame(&mut tag, "TDRC", &year.to_string());
|
||||
}
|
||||
if let Some(ref genre) = metadata.genre {
|
||||
tag.set_genre(genre.clone());
|
||||
}
|
||||
|
||||
if let Some(track) = metadata.track {
|
||||
Self::set_track_disc_frame(&mut tag, "TRCK", track, metadata.track_total);
|
||||
}
|
||||
if let Some(disc) = metadata.disc {
|
||||
Self::set_track_disc_frame(&mut tag, "TPOS", disc, metadata.disc_total);
|
||||
}
|
||||
|
||||
if let Some(ref date) = metadata.date {
|
||||
Self::set_text_frame(&mut tag, "TDRC", date);
|
||||
}
|
||||
if let Some(ref composer) = metadata.composer {
|
||||
Self::set_text_frame(&mut tag, "TCOM", composer);
|
||||
}
|
||||
if let Some(ref comment) = metadata.comment {
|
||||
Self::set_comment_frame(&mut tag, comment);
|
||||
}
|
||||
if let Some(ref lyrics) = metadata.lyrics {
|
||||
Self::set_lyrics_frame(&mut tag, lyrics);
|
||||
}
|
||||
if let Some(ref copyright) = metadata.copyright {
|
||||
Self::set_text_frame(&mut tag, "TCOP", copyright);
|
||||
}
|
||||
if let Some(compilation) = metadata.compilation {
|
||||
Self::set_text_frame(&mut tag, "TCMP", if compilation { "1" } else { "0" });
|
||||
}
|
||||
|
||||
if let Some(ref title_sort) = metadata.title_sort {
|
||||
Self::set_text_frame(&mut tag, "TSOT", title_sort);
|
||||
}
|
||||
if let Some(ref artist_sort) = metadata.artist_sort {
|
||||
Self::set_text_frame(&mut tag, "TSOP", artist_sort);
|
||||
}
|
||||
if let Some(ref album_sort) = metadata.album_sort {
|
||||
Self::set_text_frame(&mut tag, "TSOA", album_sort);
|
||||
}
|
||||
if let Some(ref album_artist_sort) = metadata.album_artist_sort {
|
||||
Self::set_text_frame(&mut tag, "TSO2", album_artist_sort);
|
||||
}
|
||||
|
||||
if let Some(ref mb_recording_id) = metadata.mb_recording_id {
|
||||
tag.insert_user_text(
|
||||
"MusicBrainz Recording Id".to_string(),
|
||||
mb_recording_id.clone(),
|
||||
);
|
||||
}
|
||||
if let Some(ref mb_album_id) = metadata.mb_album_id {
|
||||
tag.insert_user_text("MusicBrainz Album Id".to_string(), mb_album_id.clone());
|
||||
}
|
||||
if let Some(ref mb_artist_id) = metadata.mb_artist_id {
|
||||
tag.insert_user_text("MusicBrainz Artist Id".to_string(), mb_artist_id.clone());
|
||||
}
|
||||
if let Some(ref mb_album_artist_id) = metadata.mb_album_artist_id {
|
||||
tag.insert_user_text(
|
||||
"MusicBrainz Album Artist Id".to_string(),
|
||||
mb_album_artist_id.clone(),
|
||||
);
|
||||
}
|
||||
if let Some(ref mb_release_group_id) = metadata.mb_release_group_id {
|
||||
tag.insert_user_text(
|
||||
"MusicBrainz Release Group Id".to_string(),
|
||||
mb_release_group_id.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(gain) = metadata.replaygain_track_gain {
|
||||
tag.insert_user_text(
|
||||
"REPLAYGAIN_TRACK_GAIN".to_string(),
|
||||
format!("{:.2} dB", gain),
|
||||
);
|
||||
}
|
||||
if let Some(peak) = metadata.replaygain_track_peak {
|
||||
tag.insert_user_text("REPLAYGAIN_TRACK_PEAK".to_string(), format!("{:.6}", peak));
|
||||
}
|
||||
if let Some(gain) = metadata.replaygain_album_gain {
|
||||
tag.insert_user_text(
|
||||
"REPLAYGAIN_ALBUM_GAIN".to_string(),
|
||||
format!("{:.2} dB", gain),
|
||||
);
|
||||
}
|
||||
if let Some(peak) = metadata.replaygain_album_peak {
|
||||
tag.insert_user_text("REPLAYGAIN_ALBUM_PEAK".to_string(), format!("{:.6}", peak));
|
||||
}
|
||||
|
||||
if let Some(ref encoder) = metadata.encoder {
|
||||
Self::set_text_frame(&mut tag, "TSSE", encoder);
|
||||
}
|
||||
|
||||
tag
|
||||
}
|
||||
|
||||
fn extract_text_frame(tag: &Id3v2Tag, frame_id: &str) -> Option<String> {
|
||||
let id = FrameId::new(frame_id).ok()?;
|
||||
tag.get_text(&id).map(|s| s.to_string())
|
||||
}
|
||||
|
||||
fn parse_track_disc(value: &str) -> (Option<u32>, Option<u32>) {
|
||||
let parts: Vec<&str> = value.split('/').collect();
|
||||
let num = parts.first().and_then(|s| s.parse().ok());
|
||||
let total = parts.get(1).and_then(|s| s.parse().ok());
|
||||
(num, total)
|
||||
}
|
||||
|
||||
fn parse_replaygain_value(value: &str) -> Option<f32> {
|
||||
value
|
||||
.trim()
|
||||
.trim_end_matches(" dB")
|
||||
.trim_end_matches("dB")
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn extract_from_tag(tag: &Id3v2Tag) -> AudioMeta {
|
||||
let mut meta = AudioMeta::default();
|
||||
meta.format = AudioFormat::Mp3;
|
||||
|
||||
meta.title = tag.title().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.artist = tag.artist().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.album = tag.album().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.album_artist = Self::extract_text_frame(tag, "TPE2");
|
||||
meta.genre = tag.genre().map(|c: Cow<'_, str>| c.into_owned());
|
||||
|
||||
if let Some(track_str) = Self::extract_text_frame(tag, "TRCK") {
|
||||
let (track, track_total) = Self::parse_track_disc(&track_str);
|
||||
meta.track = track;
|
||||
meta.track_total = track_total;
|
||||
} else {
|
||||
meta.track = tag.track();
|
||||
meta.track_total = tag.track_total();
|
||||
}
|
||||
|
||||
if let Some(disc_str) = Self::extract_text_frame(tag, "TPOS") {
|
||||
let (disc, disc_total) = Self::parse_track_disc(&disc_str);
|
||||
meta.disc = disc;
|
||||
meta.disc_total = disc_total;
|
||||
} else {
|
||||
meta.disc = tag.disk();
|
||||
meta.disc_total = tag.disk_total();
|
||||
}
|
||||
|
||||
meta.date = Self::extract_text_frame(tag, "TDRC");
|
||||
if let Some(ref date) = meta.date {
|
||||
if let Some(year_str) = date.split('-').next() {
|
||||
meta.year = year_str.parse().ok();
|
||||
}
|
||||
}
|
||||
|
||||
meta.composer = Self::extract_text_frame(tag, "TCOM");
|
||||
meta.comment = tag.comment().map(|c: Cow<'_, str>| c.into_owned());
|
||||
|
||||
if let Some(uslt) = tag.unsync_text().next() {
|
||||
meta.lyrics = Some(uslt.content.to_string());
|
||||
}
|
||||
|
||||
meta.copyright = Self::extract_text_frame(tag, "TCOP");
|
||||
|
||||
if let Some(tcmp) = Self::extract_text_frame(tag, "TCMP") {
|
||||
meta.compilation = Some(tcmp == "1");
|
||||
}
|
||||
|
||||
meta.title_sort = Self::extract_text_frame(tag, "TSOT");
|
||||
meta.artist_sort = Self::extract_text_frame(tag, "TSOP");
|
||||
meta.album_sort = Self::extract_text_frame(tag, "TSOA");
|
||||
meta.album_artist_sort = Self::extract_text_frame(tag, "TSO2");
|
||||
|
||||
meta.mb_recording_id = tag
|
||||
.get_user_text("MusicBrainz Recording Id")
|
||||
.map(String::from);
|
||||
meta.mb_album_id = tag.get_user_text("MusicBrainz Album Id").map(String::from);
|
||||
meta.mb_artist_id = tag.get_user_text("MusicBrainz Artist Id").map(String::from);
|
||||
meta.mb_album_artist_id = tag
|
||||
.get_user_text("MusicBrainz Album Artist Id")
|
||||
.map(String::from);
|
||||
meta.mb_release_group_id = tag
|
||||
.get_user_text("MusicBrainz Release Group Id")
|
||||
.map(String::from);
|
||||
|
||||
if let Some(gain_str) = tag.get_user_text("REPLAYGAIN_TRACK_GAIN") {
|
||||
meta.replaygain_track_gain = Self::parse_replaygain_value(gain_str);
|
||||
}
|
||||
if let Some(peak_str) = tag.get_user_text("REPLAYGAIN_TRACK_PEAK") {
|
||||
meta.replaygain_track_peak = peak_str.parse::<f32>().ok();
|
||||
}
|
||||
if let Some(gain_str) = tag.get_user_text("REPLAYGAIN_ALBUM_GAIN") {
|
||||
meta.replaygain_album_gain = Self::parse_replaygain_value(gain_str);
|
||||
}
|
||||
if let Some(peak_str) = tag.get_user_text("REPLAYGAIN_ALBUM_PEAK") {
|
||||
meta.replaygain_album_peak = peak_str.parse::<f32>().ok();
|
||||
}
|
||||
|
||||
meta.encoder = Self::extract_text_frame(tag, "TSSE");
|
||||
|
||||
meta
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Id3v2Handler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl FormatHandler for Id3v2Handler {
|
||||
fn id(&self) -> &'static str {
|
||||
"id3v2"
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"ID3v2 (MP3)"
|
||||
}
|
||||
|
||||
fn extensions(&self) -> &[&'static str] {
|
||||
&["mp3"]
|
||||
}
|
||||
|
||||
fn mime_types(&self) -> &[&'static str] {
|
||||
&["audio/mpeg"]
|
||||
}
|
||||
|
||||
fn analyze(&self, data: &[u8], file_size: u64) -> Result<FormatLayout, FormatError> {
|
||||
let audio_start = Self::parse_id3v2_header(data).unwrap_or(0) as u64;
|
||||
|
||||
let audio_end = if Self::has_id3v1_tag(data, file_size) {
|
||||
file_size - ID3V1_TAG_SIZE as u64
|
||||
} else {
|
||||
file_size
|
||||
};
|
||||
|
||||
Ok(FormatLayout {
|
||||
audio_start,
|
||||
audio_end,
|
||||
format: AudioFormat::Mp3,
|
||||
format_data: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn synthesize(
|
||||
&self,
|
||||
metadata: &AudioMeta,
|
||||
_layout: &FormatLayout,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let tag = Self::build_tag_from_meta(metadata);
|
||||
|
||||
let mut buffer = Cursor::new(Vec::new());
|
||||
let write_options = WriteOptions::new().preferred_padding(1024);
|
||||
|
||||
tag.dump_to(&mut buffer, write_options)
|
||||
.map_err(|e| FormatError::SynthesisFailed(e.to_string()))?;
|
||||
|
||||
Ok(buffer.into_inner())
|
||||
}
|
||||
|
||||
fn extract(&self, data: &[u8]) -> Result<AudioMeta, FormatError> {
|
||||
let mut cursor = Cursor::new(data);
|
||||
|
||||
let mpeg_file = MpegFile::read_from(&mut cursor, ParseOptions::new())
|
||||
.map_err(|e| FormatError::InvalidData(e.to_string()))?;
|
||||
|
||||
let tag = mpeg_file
|
||||
.id3v2()
|
||||
.ok_or_else(|| FormatError::InvalidData("No ID3v2 tag found".to_string()))?;
|
||||
|
||||
Ok(Self::extract_from_tag(tag))
|
||||
}
|
||||
|
||||
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
|
||||
4096 + 1024
|
||||
}
|
||||
}
|
||||
|
||||
fn syncsafe_decode(bytes: &[u8]) -> usize {
|
||||
((bytes[0] as usize) << 21)
|
||||
| ((bytes[1] as usize) << 14)
|
||||
| ((bytes[2] as usize) << 7)
|
||||
| (bytes[3] as usize)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_test_meta() -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: Some("Test Title".to_string()),
|
||||
artist: Some("Test Artist".to_string()),
|
||||
album: Some("Test Album".to_string()),
|
||||
album_artist: Some("Test Album Artist".to_string()),
|
||||
genre: Some("Rock".to_string()),
|
||||
year: Some(2024),
|
||||
track: Some(5),
|
||||
track_total: Some(12),
|
||||
disc: Some(1),
|
||||
disc_total: Some(2),
|
||||
format: AudioFormat::Mp3,
|
||||
date: Some("2024-03-15".to_string()),
|
||||
composer: Some("Test Composer".to_string()),
|
||||
comment: Some("Test Comment".to_string()),
|
||||
lyrics: Some("Test Lyrics\nLine 2".to_string()),
|
||||
copyright: Some("2024 Test Copyright".to_string()),
|
||||
compilation: Some(false),
|
||||
title_sort: Some("Title, Test".to_string()),
|
||||
artist_sort: Some("Artist, Test".to_string()),
|
||||
album_sort: Some("Album, Test".to_string()),
|
||||
album_artist_sort: Some("Album Artist, Test".to_string()),
|
||||
mb_recording_id: Some("rec-12345".to_string()),
|
||||
mb_album_id: Some("alb-12345".to_string()),
|
||||
mb_artist_id: Some("art-12345".to_string()),
|
||||
mb_album_artist_id: Some("albart-12345".to_string()),
|
||||
mb_release_group_id: Some("rg-12345".to_string()),
|
||||
replaygain_track_gain: Some(-6.5),
|
||||
replaygain_track_peak: Some(0.987654),
|
||||
replaygain_album_gain: Some(-5.2),
|
||||
replaygain_album_peak: Some(0.999999),
|
||||
encoder: Some("LAME 3.100".to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_id_and_name() {
|
||||
let handler = Id3v2Handler::new();
|
||||
assert_eq!(handler.id(), "id3v2");
|
||||
assert_eq!(handler.name(), "ID3v2 (MP3)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extensions_and_mime_types() {
|
||||
let handler = Id3v2Handler::new();
|
||||
assert_eq!(handler.extensions(), &["mp3"]);
|
||||
assert_eq!(handler.mime_types(), &["audio/mpeg"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_header_size() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let meta = AudioMeta::default();
|
||||
assert_eq!(handler.estimate_header_size(&meta), 5120);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthesize_creates_valid_id3v2() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let meta = make_test_meta();
|
||||
let layout = FormatLayout {
|
||||
audio_start: 0,
|
||||
audio_end: 1000,
|
||||
format: AudioFormat::Mp3,
|
||||
format_data: None,
|
||||
};
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
assert!(bytes.len() >= 10);
|
||||
assert_eq!(&bytes[0..3], b"ID3");
|
||||
assert_eq!(bytes[3], 0x04);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_no_id3v2() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let data = vec![0xFF, 0xFB, 0x90, 0x00];
|
||||
let file_size = 1000;
|
||||
|
||||
let result = handler.analyze(&data, file_size);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let layout = result.unwrap();
|
||||
assert_eq!(layout.audio_start, 0);
|
||||
assert_eq!(layout.audio_end, 1000);
|
||||
assert_eq!(layout.format, AudioFormat::Mp3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_with_id3v2() {
|
||||
let handler = Id3v2Handler::new();
|
||||
|
||||
let mut data = vec![b'I', b'D', b'3', 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64];
|
||||
data.extend(vec![0u8; 100]);
|
||||
let file_size = data.len() as u64;
|
||||
|
||||
let result = handler.analyze(&data, file_size);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let layout = result.unwrap();
|
||||
assert_eq!(layout.audio_start, 110);
|
||||
assert_eq!(layout.audio_end, file_size);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_with_id3v1() {
|
||||
let handler = Id3v2Handler::new();
|
||||
|
||||
let mut data = vec![0xFF, 0xFB, 0x90, 0x00];
|
||||
data.extend(vec![0u8; 100]);
|
||||
data.extend(b"TAG");
|
||||
data.extend(vec![0u8; 125]);
|
||||
let file_size = data.len() as u64;
|
||||
|
||||
let result = handler.analyze(&data, file_size);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let layout = result.unwrap();
|
||||
assert_eq!(layout.audio_start, 0);
|
||||
assert_eq!(layout.audio_end, file_size - 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_syncsafe_decode() {
|
||||
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x00, 0x7F]), 127);
|
||||
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x01, 0x00]), 128);
|
||||
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x00, 0x64]), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_track_disc() {
|
||||
assert_eq!(Id3v2Handler::parse_track_disc("5/12"), (Some(5), Some(12)));
|
||||
assert_eq!(Id3v2Handler::parse_track_disc("5"), (Some(5), None));
|
||||
assert_eq!(Id3v2Handler::parse_track_disc(""), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_replaygain_value() {
|
||||
assert_eq!(
|
||||
Id3v2Handler::parse_replaygain_value("-6.50 dB"),
|
||||
Some(-6.50)
|
||||
);
|
||||
assert_eq!(Id3v2Handler::parse_replaygain_value("-6.50dB"), Some(-6.50));
|
||||
assert_eq!(Id3v2Handler::parse_replaygain_value("-6.50"), Some(-6.50));
|
||||
assert_eq!(Id3v2Handler::parse_replaygain_value("invalid"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_metadata_produces_empty_tag() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let meta = AudioMeta::default();
|
||||
let layout = FormatLayout {
|
||||
audio_start: 0,
|
||||
audio_end: 1000,
|
||||
format: AudioFormat::Mp3,
|
||||
format_data: None,
|
||||
};
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
assert!(bytes.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_minimal_metadata_produces_valid_tag() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let mut meta = AudioMeta::default();
|
||||
meta.title = Some("Test".to_string());
|
||||
let layout = FormatLayout {
|
||||
audio_start: 0,
|
||||
audio_end: 1000,
|
||||
format: AudioFormat::Mp3,
|
||||
format_data: None,
|
||||
};
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
assert!(bytes.len() >= 10);
|
||||
assert_eq!(&bytes[0..3], b"ID3");
|
||||
assert_eq!(bytes[3], 0x04);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_and_extract_tag() {
|
||||
let original_meta = make_test_meta();
|
||||
let tag = Id3v2Handler::build_tag_from_meta(&original_meta);
|
||||
let extracted = Id3v2Handler::extract_from_tag(&tag);
|
||||
|
||||
assert_eq!(extracted.title, original_meta.title);
|
||||
assert_eq!(extracted.artist, original_meta.artist);
|
||||
assert_eq!(extracted.album, original_meta.album);
|
||||
assert_eq!(extracted.album_artist, original_meta.album_artist);
|
||||
assert_eq!(extracted.genre, original_meta.genre);
|
||||
assert_eq!(extracted.track, original_meta.track);
|
||||
assert_eq!(extracted.track_total, original_meta.track_total);
|
||||
assert_eq!(extracted.disc, original_meta.disc);
|
||||
assert_eq!(extracted.disc_total, original_meta.disc_total);
|
||||
assert_eq!(extracted.composer, original_meta.composer);
|
||||
assert_eq!(extracted.comment, original_meta.comment);
|
||||
assert_eq!(extracted.lyrics, original_meta.lyrics);
|
||||
assert_eq!(extracted.copyright, original_meta.copyright);
|
||||
assert_eq!(extracted.compilation, original_meta.compilation);
|
||||
assert_eq!(extracted.title_sort, original_meta.title_sort);
|
||||
assert_eq!(extracted.artist_sort, original_meta.artist_sort);
|
||||
assert_eq!(extracted.album_sort, original_meta.album_sort);
|
||||
assert_eq!(extracted.album_artist_sort, original_meta.album_artist_sort);
|
||||
assert_eq!(extracted.mb_recording_id, original_meta.mb_recording_id);
|
||||
assert_eq!(extracted.mb_album_id, original_meta.mb_album_id);
|
||||
assert_eq!(extracted.mb_artist_id, original_meta.mb_artist_id);
|
||||
assert_eq!(
|
||||
extracted.mb_album_artist_id,
|
||||
original_meta.mb_album_artist_id
|
||||
);
|
||||
assert_eq!(
|
||||
extracted.mb_release_group_id,
|
||||
original_meta.mb_release_group_id
|
||||
);
|
||||
assert_eq!(extracted.encoder, original_meta.encoder);
|
||||
|
||||
let orig_track_gain = original_meta.replaygain_track_gain.unwrap();
|
||||
let ext_track_gain = extracted.replaygain_track_gain.unwrap();
|
||||
assert!((orig_track_gain - ext_track_gain).abs() < 0.01);
|
||||
|
||||
let orig_track_peak = original_meta.replaygain_track_peak.unwrap();
|
||||
let ext_track_peak = extracted.replaygain_track_peak.unwrap();
|
||||
assert!((orig_track_peak - ext_track_peak).abs() < 0.0001);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
//! Format-specific metadata handlers for audio file synthesis.
|
||||
//!
|
||||
//! Each handler implements the `FormatHandler` trait to support:
|
||||
//! - Analyzing original files to find audio boundaries
|
||||
//! - Synthesizing new headers from database metadata
|
||||
//! - Extracting metadata from existing files
|
||||
|
||||
mod flac;
|
||||
mod id3v2;
|
||||
|
||||
pub use flac::FlacHandler;
|
||||
pub use id3v2::Id3v2Handler;
|
||||
@@ -0,0 +1,26 @@
|
||||
mod artwork;
|
||||
mod db;
|
||||
mod eviction;
|
||||
mod format_handler;
|
||||
mod format_layout;
|
||||
pub mod handlers;
|
||||
mod metadata;
|
||||
mod overlay;
|
||||
mod patterns;
|
||||
mod prefetch;
|
||||
mod tree;
|
||||
|
||||
pub use artwork::{ArtworkCache, ArtworkError, CachedArtwork};
|
||||
pub use db::{Database, EnrichmentUpdate, TrashedFile, TrashedFilter};
|
||||
pub use eviction::{EvictionError, EvictionPolicy, LruEviction};
|
||||
pub use format_handler::{FormatError, FormatHandler, FormatHandlerRegistry};
|
||||
pub use format_layout::FormatLayout;
|
||||
pub use handlers::{FlacHandler, Id3v2Handler};
|
||||
pub use metadata::MetadataCache;
|
||||
pub use overlay::{OverlayError, OverlayReader};
|
||||
pub use patterns::{AccessContext, AccessPattern, PatternError, PatternStore};
|
||||
pub use prefetch::{PrefetchConfig, PrefetchEngine, PrefetchHandle};
|
||||
pub use tree::{
|
||||
DirNode, FileNode, Inode, RefreshPolicy, RemoveError, RenameError, TreeBuilder, VirtualNode,
|
||||
VirtualTree, ROOT_INODE,
|
||||
};
|
||||
@@ -94,7 +94,14 @@ mod tests {
|
||||
};
|
||||
|
||||
cache
|
||||
.store(&origin_id, real_path, &virtual_path, &meta, UNIX_EPOCH, 5000)
|
||||
.store(
|
||||
&origin_id,
|
||||
real_path,
|
||||
&virtual_path,
|
||||
&meta,
|
||||
UNIX_EPOCH,
|
||||
5000,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let retrieved = cache.lookup(&virtual_path).unwrap().unwrap();
|
||||
@@ -0,0 +1,467 @@
|
||||
//! OverlayReader: On-the-fly metadata overlay with header/audio splice logic.
|
||||
//!
|
||||
//! This module provides the core read path for metadata overlay. It synthesizes
|
||||
//! headers on-the-fly from database metadata and splices them with original audio
|
||||
//! data from the CAS.
|
||||
|
||||
use crate::{Database, FormatError, FormatHandlerRegistry};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use musicfs_cas::{FileReader, ReaderError};
|
||||
use musicfs_core::{AudioFormat, FileId};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, trace};
|
||||
|
||||
/// Error types for overlay operations
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum OverlayError {
|
||||
#[error("Database error: {0}")]
|
||||
Database(#[from] musicfs_core::Error),
|
||||
|
||||
#[error("Format handler error: {0}")]
|
||||
Handler(#[from] FormatError),
|
||||
|
||||
#[error("CAS error: {0}")]
|
||||
Cas(#[from] ReaderError),
|
||||
|
||||
#[error("File not found: {0:?}")]
|
||||
NotFound(FileId),
|
||||
|
||||
#[error("No handler for format: {0:?}")]
|
||||
NoHandler(AudioFormat),
|
||||
}
|
||||
|
||||
/// OverlayReader provides on-the-fly metadata overlay for audio files.
|
||||
///
|
||||
/// It synthesizes headers from database metadata and splices them with
|
||||
/// original audio data from the CAS, presenting a virtual file that
|
||||
/// reflects the current metadata state.
|
||||
pub struct OverlayReader {
|
||||
db: Arc<Database>,
|
||||
registry: Arc<FormatHandlerRegistry>,
|
||||
cas_reader: Arc<FileReader>,
|
||||
}
|
||||
|
||||
impl OverlayReader {
|
||||
/// Create a new OverlayReader with the given dependencies.
|
||||
pub fn new(
|
||||
db: Arc<Database>,
|
||||
registry: Arc<FormatHandlerRegistry>,
|
||||
cas_reader: Arc<FileReader>,
|
||||
) -> Self {
|
||||
Self {
|
||||
db,
|
||||
registry,
|
||||
cas_reader,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read bytes from a virtual file with metadata overlay.
|
||||
///
|
||||
/// This method implements the three-region splice logic:
|
||||
/// - Region 1: Synthetic header (offset < header_len)
|
||||
/// - Region 2: Audio data from CAS (offset >= header_len)
|
||||
/// - Region 3: Boundary crossing (spans header/audio)
|
||||
///
|
||||
/// If no format_layout exists for the file, delegates directly to CAS reader.
|
||||
pub async fn read(
|
||||
&self,
|
||||
file_id: FileId,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> Result<Bytes, OverlayError> {
|
||||
// Get format layout - if None, passthrough to CAS
|
||||
let layout = match self.db.get_format_layout(file_id)? {
|
||||
Some(layout) => layout,
|
||||
None => {
|
||||
trace!(file_id = ?file_id, "No format_layout, passthrough to CAS");
|
||||
return Ok(self.cas_reader.read(file_id, offset, size).await?);
|
||||
}
|
||||
};
|
||||
|
||||
// Get metadata for synthesis
|
||||
let metadata = self.db.get_file_metadata_row(file_id)?;
|
||||
|
||||
// Get handler for this format (handler IDs are lowercase)
|
||||
let format_id = format!("{:?}", layout.format).to_lowercase();
|
||||
let handler = self
|
||||
.registry
|
||||
.get_by_format(&format_id)
|
||||
.ok_or_else(|| OverlayError::NoHandler(layout.format))?;
|
||||
|
||||
// Synthesize header on-the-fly
|
||||
let header = handler.synthesize(&metadata, &layout)?;
|
||||
let header_len = header.len() as u64;
|
||||
let audio_len = layout.audio_end - layout.audio_start;
|
||||
let virtual_size = header_len + audio_len;
|
||||
|
||||
trace!(
|
||||
file_id = ?file_id,
|
||||
header_len,
|
||||
audio_len,
|
||||
virtual_size,
|
||||
offset,
|
||||
size,
|
||||
"Overlay read"
|
||||
);
|
||||
|
||||
// Handle EOF
|
||||
if offset >= virtual_size {
|
||||
return Ok(Bytes::new());
|
||||
}
|
||||
|
||||
let virtual_end = (offset + size as u64).min(virtual_size);
|
||||
let mut result = BytesMut::with_capacity((virtual_end - offset) as usize);
|
||||
|
||||
// Region 1: Synthetic header
|
||||
if offset < header_len {
|
||||
let end = virtual_end.min(header_len);
|
||||
result.extend_from_slice(&header[offset as usize..end as usize]);
|
||||
trace!(
|
||||
file_id = ?file_id,
|
||||
start = offset,
|
||||
end,
|
||||
bytes = end - offset,
|
||||
"Read from synthetic header"
|
||||
);
|
||||
}
|
||||
|
||||
// Region 2: Origin audio data (from CAS)
|
||||
if virtual_end > header_len {
|
||||
let audio_start_in_virtual = header_len.max(offset);
|
||||
let audio_offset_in_origin = layout.audio_start + (audio_start_in_virtual - header_len);
|
||||
let audio_bytes_needed = (virtual_end - audio_start_in_virtual) as u32;
|
||||
|
||||
trace!(
|
||||
file_id = ?file_id,
|
||||
audio_offset_in_origin,
|
||||
audio_bytes_needed,
|
||||
"Read from CAS audio"
|
||||
);
|
||||
|
||||
let audio = self
|
||||
.cas_reader
|
||||
.read(file_id, audio_offset_in_origin, audio_bytes_needed)
|
||||
.await?;
|
||||
result.extend_from_slice(&audio);
|
||||
}
|
||||
|
||||
debug!(
|
||||
file_id = ?file_id,
|
||||
offset,
|
||||
size,
|
||||
returned = result.len(),
|
||||
"Overlay read complete"
|
||||
);
|
||||
|
||||
Ok(result.freeze())
|
||||
}
|
||||
|
||||
/// Estimate the virtual size of a file for getattr.
|
||||
///
|
||||
/// Returns the estimated size based on format layout. If no layout exists,
|
||||
/// returns None to indicate the caller should use the original file size.
|
||||
pub fn estimate_virtual_size(&self, file_id: FileId) -> Result<Option<u64>, OverlayError> {
|
||||
// Get format layout - if None, return None to indicate passthrough
|
||||
let layout = match self.db.get_format_layout(file_id)? {
|
||||
Some(layout) => layout,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
// Get metadata for header size estimation
|
||||
let metadata = self.db.get_file_metadata_row(file_id)?;
|
||||
|
||||
let format_id = format!("{:?}", layout.format).to_lowercase();
|
||||
let handler = self
|
||||
.registry
|
||||
.get_by_format(&format_id)
|
||||
.ok_or_else(|| OverlayError::NoHandler(layout.format))?;
|
||||
|
||||
// Estimate header size
|
||||
let estimated_header = handler.estimate_header_size(&metadata) as u64;
|
||||
let audio_len = layout.audio_end - layout.audio_start;
|
||||
let virtual_size = estimated_header + audio_len;
|
||||
|
||||
trace!(
|
||||
file_id = ?file_id,
|
||||
estimated_header,
|
||||
audio_len,
|
||||
virtual_size,
|
||||
"Estimated virtual size"
|
||||
);
|
||||
|
||||
Ok(Some(virtual_size))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::handlers::FlacHandler;
|
||||
use crate::FormatLayout;
|
||||
use musicfs_cas::{CasConfig, CasStore, ChunkManifest, ChunkRef};
|
||||
use musicfs_core::{AudioFormat, AudioMeta, OriginId, VirtualPath};
|
||||
use std::path::Path;
|
||||
use std::time::UNIX_EPOCH;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_test_metadata() -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: Some("Test Track".to_string()),
|
||||
artist: Some("Test Artist".to_string()),
|
||||
album: Some("Test Album".to_string()),
|
||||
track: Some(1),
|
||||
format: AudioFormat::Flac,
|
||||
sample_rate: Some(44100),
|
||||
bits_per_sample: Some(16),
|
||||
channels: Some(2),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn make_test_layout() -> FormatLayout {
|
||||
// Simulate a file with minimal FLAC header, audio from 42 to 102442 (100KB audio)
|
||||
// STREAMINFO data (34 bytes) - minimal valid values for FLAC synthesis
|
||||
let streaminfo_data = vec![
|
||||
0x10, 0x00, // min_block_size = 4096
|
||||
0x10, 0x00, // max_block_size = 4096
|
||||
0x00, 0x00, 0x00, // min_frame_size = 0
|
||||
0x00, 0x00, 0x00, // max_frame_size = 0
|
||||
0x0A, 0xC4, 0x42, 0xF0, // sample_rate=44100, channels=2, bits=16
|
||||
0x00, 0x00, 0x00, 0x00, // total_samples
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MD5 (16 bytes)
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
];
|
||||
FormatLayout {
|
||||
audio_start: 42, // fLaC (4) + STREAMINFO block (38)
|
||||
audio_end: 42 + 100 * 1024, // 100KB audio
|
||||
format: AudioFormat::Flac,
|
||||
format_data: Some(streaminfo_data),
|
||||
}
|
||||
}
|
||||
|
||||
async fn setup_test_env() -> (
|
||||
TempDir,
|
||||
Arc<Database>,
|
||||
Arc<FormatHandlerRegistry>,
|
||||
Arc<FileReader>,
|
||||
FileId,
|
||||
) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
// Setup database
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
|
||||
// Setup registry with FLAC handler
|
||||
let mut registry = FormatHandlerRegistry::new();
|
||||
registry.register(Arc::new(FlacHandler::new()));
|
||||
let registry = Arc::new(registry);
|
||||
|
||||
// Setup CAS store and reader
|
||||
let cas_config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
|
||||
|
||||
// Create test audio data (simulating 100KB of audio)
|
||||
let audio_data: Vec<u8> = (0..100 * 1024).map(|i| (i % 256) as u8).collect();
|
||||
let hash = store.put(&audio_data).await.unwrap();
|
||||
|
||||
let reader = Arc::new(FileReader::new(store));
|
||||
|
||||
// Register manifest for the test file
|
||||
// The manifest represents the ORIGINAL file in CAS, with audio starting at offset 42
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: 42 + 100 * 1024, // Original file size (42 byte header + 100KB audio)
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 42, // Audio starts at offset 42 in the original file
|
||||
size: audio_data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let file_id = db
|
||||
.upsert_file_with_layout(
|
||||
&OriginId::from("test"),
|
||||
Path::new("/test.flac"),
|
||||
&VirtualPath::new("/Test Artist/Test Album/01 - Test Track.flac"),
|
||||
&make_test_metadata(),
|
||||
UNIX_EPOCH,
|
||||
42 + 100 * 1024,
|
||||
Some(&make_test_layout()),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
(dir, db, registry, reader, file_id)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_header_region() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
// Read first 100 bytes (should be from synthetic header)
|
||||
let result = overlay.read(file_id, 0, 100).await.unwrap();
|
||||
|
||||
// Should return data (synthetic header)
|
||||
assert!(!result.is_empty());
|
||||
assert!(result.len() <= 100);
|
||||
|
||||
// FLAC files start with "fLaC" magic
|
||||
assert_eq!(&result[0..4], b"fLaC");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_audio_region() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db.clone(), registry.clone(), reader.clone());
|
||||
|
||||
// First, get the actual header size by reading it
|
||||
let _header_result = overlay.read(file_id, 0, 64 * 1024).await.unwrap();
|
||||
|
||||
// Get the layout to know where audio starts in virtual file
|
||||
let layout = db.get_format_layout(file_id).unwrap().unwrap();
|
||||
let metadata = db.get_file_metadata_row(file_id).unwrap();
|
||||
let handler = registry.get_by_format("flac").unwrap();
|
||||
let header = handler.synthesize(&metadata, &layout).unwrap();
|
||||
let header_len = header.len() as u64;
|
||||
|
||||
// Read from well into the audio region
|
||||
let audio_offset = header_len + 1000;
|
||||
let result = overlay.read(file_id, audio_offset, 1000).await.unwrap();
|
||||
|
||||
// Should return audio data
|
||||
assert!(!result.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_boundary() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db.clone(), registry.clone(), reader.clone());
|
||||
|
||||
// Get the actual header size
|
||||
let layout = db.get_format_layout(file_id).unwrap().unwrap();
|
||||
let metadata = db.get_file_metadata_row(file_id).unwrap();
|
||||
let handler = registry.get_by_format("flac").unwrap();
|
||||
let header = handler.synthesize(&metadata, &layout).unwrap();
|
||||
let header_len = header.len() as u64;
|
||||
|
||||
// Read across the header/audio boundary
|
||||
let boundary_offset = header_len - 50;
|
||||
let result = overlay.read(file_id, boundary_offset, 100).await.unwrap();
|
||||
|
||||
// Should return 100 bytes spanning both regions
|
||||
assert_eq!(result.len(), 100);
|
||||
|
||||
// First 50 bytes should be from header
|
||||
assert_eq!(&result[0..50], &header[(header_len - 50) as usize..]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_passthrough() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
let registry = Arc::new(FormatHandlerRegistry::new());
|
||||
|
||||
let cas_config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
|
||||
|
||||
let test_data = b"Hello, World! This is test data.";
|
||||
let hash = store.put(test_data).await.unwrap();
|
||||
|
||||
// Insert file WITHOUT format_layout first to get the file_id
|
||||
let file_id = db
|
||||
.upsert_file(
|
||||
&OriginId::from("test"),
|
||||
Path::new("/test.txt"),
|
||||
&VirtualPath::new("/test.txt"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
test_data.len() as u64,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let reader = Arc::new(FileReader::new(store));
|
||||
// Register manifest with the actual file_id from database
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id,
|
||||
total_size: test_data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 0,
|
||||
size: test_data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
let result = overlay
|
||||
.read(file_id, 0, test_data.len() as u32)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(&result[..], test_data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_estimate_virtual_size() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
// Should return estimated size
|
||||
let size = overlay.estimate_virtual_size(file_id).unwrap();
|
||||
assert!(size.is_some());
|
||||
|
||||
let virtual_size = size.unwrap();
|
||||
// Virtual size should be header + audio (100KB audio)
|
||||
assert!(virtual_size > 100 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_estimate_virtual_size_passthrough() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
let registry = Arc::new(FormatHandlerRegistry::new());
|
||||
let cas_config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
|
||||
let reader = Arc::new(FileReader::new(store));
|
||||
|
||||
// Insert file WITHOUT format_layout
|
||||
let file_id = db
|
||||
.upsert_file(
|
||||
&OriginId::from("test"),
|
||||
Path::new("/test.txt"),
|
||||
&VirtualPath::new("/test.txt"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
1000,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
// Should return None for passthrough
|
||||
let size = overlay.estimate_virtual_size(file_id).unwrap();
|
||||
assert!(size.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_eof() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
// Read past EOF
|
||||
let result = overlay.read(file_id, 1_000_000, 100).await.unwrap();
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -63,13 +63,11 @@ impl PatternStore {
|
||||
|
||||
let sequence_counts = {
|
||||
let mut map = HashMap::new();
|
||||
let mut stmt = db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?;
|
||||
let mut stmt =
|
||||
db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
(
|
||||
FileId(row.get::<_, i64>(0)?),
|
||||
FileId(row.get::<_, i64>(1)?),
|
||||
),
|
||||
(FileId(row.get::<_, i64>(0)?), FileId(row.get::<_, i64>(1)?)),
|
||||
row.get::<_, u32>(2)?,
|
||||
))
|
||||
})?;
|
||||
@@ -154,7 +152,11 @@ impl PatternStore {
|
||||
.take(limit)
|
||||
.map(|(id, _)| id)
|
||||
.collect();
|
||||
debug!(file_id = current.0, predictions = result.len(), "Predicted next files");
|
||||
debug!(
|
||||
file_id = current.0,
|
||||
predictions = result.len(),
|
||||
"Predicted next files"
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -102,13 +102,8 @@ impl PrefetchEngine {
|
||||
pattern_store.predict_next(file_id, config.lookahead);
|
||||
|
||||
for predicted_id in predictions {
|
||||
prefetch_file(
|
||||
predicted_id,
|
||||
&fetcher,
|
||||
&in_flight,
|
||||
&semaphore,
|
||||
)
|
||||
.await;
|
||||
prefetch_file(predicted_id, &fetcher, &in_flight, &semaphore)
|
||||
.await;
|
||||
}
|
||||
|
||||
tokio::time::sleep(config.cooldown).await;
|
||||
@@ -20,6 +20,41 @@ CREATE TABLE IF NOT EXISTS files (
|
||||
bitrate INTEGER,
|
||||
sample_rate INTEGER,
|
||||
format TEXT,
|
||||
track_total INTEGER,
|
||||
disc_total INTEGER,
|
||||
date TEXT,
|
||||
composer TEXT,
|
||||
comment TEXT,
|
||||
lyrics TEXT,
|
||||
copyright TEXT,
|
||||
compilation INTEGER,
|
||||
artist_sort TEXT,
|
||||
album_artist_sort TEXT,
|
||||
album_sort TEXT,
|
||||
title_sort TEXT,
|
||||
mb_recording_id TEXT,
|
||||
mb_album_id TEXT,
|
||||
mb_artist_id TEXT,
|
||||
mb_album_artist_id TEXT,
|
||||
mb_release_group_id TEXT,
|
||||
replaygain_track_gain REAL,
|
||||
replaygain_track_peak REAL,
|
||||
replaygain_album_gain REAL,
|
||||
replaygain_album_peak REAL,
|
||||
channels INTEGER,
|
||||
bits_per_sample INTEGER,
|
||||
encoder TEXT,
|
||||
custom_tags TEXT,
|
||||
format_layout BLOB,
|
||||
|
||||
label TEXT,
|
||||
album_type TEXT,
|
||||
cover_url TEXT,
|
||||
genres_json TEXT,
|
||||
enrichment_source TEXT,
|
||||
enriched_at INTEGER,
|
||||
enrichment_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_enrichment_error TEXT,
|
||||
|
||||
origin_mtime INTEGER NOT NULL,
|
||||
origin_size INTEGER NOT NULL,
|
||||
@@ -27,6 +62,10 @@ CREATE TABLE IF NOT EXISTS files (
|
||||
chunk_manifest BLOB,
|
||||
last_sync INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
|
||||
|
||||
trashed INTEGER NOT NULL DEFAULT 0,
|
||||
original_path TEXT,
|
||||
trashed_at INTEGER,
|
||||
|
||||
UNIQUE(origin_id, real_path)
|
||||
);
|
||||
|
||||
@@ -55,4 +94,18 @@ CREATE INDEX IF NOT EXISTS idx_files_content_hash ON files(content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_real ON files(origin_id, real_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_origin ON files(origin_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_last_sync ON files(last_sync);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_mb_album ON files(mb_album_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_mb_artist ON files(mb_artist_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_genre ON files(genre);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_year ON files(year);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_composer ON files(composer);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_file ON artwork(file_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS directories (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_directories_path ON directories(path);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_trashed ON files(trashed) WHERE trashed = 1;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,12 @@ name = "musicfs-cas"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
failpoints = ["fail/failpoints"]
|
||||
|
||||
[dependencies]
|
||||
fail = { workspace = true, optional = true }
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-origins = { path = "../musicfs-origins" }
|
||||
musicfs-sync = { path = "../musicfs-sync" }
|
||||
@@ -17,6 +22,7 @@ rmp-serde.workspace = true
|
||||
hex.workspace = true
|
||||
dirs.workspace = true
|
||||
thiserror.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -2,9 +2,10 @@ use crate::{CasStore, ChunkManifest, ChunkRef};
|
||||
use musicfs_core::{Event, EventBus, FileId, FileMeta, OriginId};
|
||||
use musicfs_origins::Origin;
|
||||
use musicfs_sync::CdcChunker;
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tracing::{debug, info};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
pub struct ContentFetcher {
|
||||
store: Arc<CasStore>,
|
||||
@@ -37,15 +38,15 @@ impl ContentFetcher {
|
||||
|
||||
pub fn register_origin(&self, origin: Arc<dyn Origin>) {
|
||||
let id = origin.id().clone();
|
||||
self.origins.write().unwrap().insert(id, origin);
|
||||
self.origins.write().insert(id, origin);
|
||||
}
|
||||
|
||||
pub fn register_file(&self, meta: FileMeta) {
|
||||
self.file_meta.write().unwrap().insert(meta.id, meta);
|
||||
self.file_meta.write().insert(meta.id, meta);
|
||||
}
|
||||
|
||||
pub fn register_files(&self, files: impl IntoIterator<Item = FileMeta>) {
|
||||
let mut map = self.file_meta.write().unwrap();
|
||||
let mut map = self.file_meta.write();
|
||||
for meta in files {
|
||||
map.insert(meta.id, meta);
|
||||
}
|
||||
@@ -53,7 +54,7 @@ impl ContentFetcher {
|
||||
|
||||
pub async fn fetch_file(&self, file_id: FileId) -> Result<ChunkManifest, FetchError> {
|
||||
let meta = {
|
||||
let files = self.file_meta.read().unwrap();
|
||||
let files = self.file_meta.read();
|
||||
files
|
||||
.get(&file_id)
|
||||
.cloned()
|
||||
@@ -61,18 +62,14 @@ impl ContentFetcher {
|
||||
};
|
||||
|
||||
let origin = {
|
||||
let origins = self.origins.read().unwrap();
|
||||
let origins = self.origins.read();
|
||||
origins
|
||||
.get(&meta.real_path.origin_id)
|
||||
.cloned()
|
||||
.ok_or_else(|| FetchError::OriginNotFound(meta.real_path.origin_id.clone()))?
|
||||
};
|
||||
|
||||
info!(
|
||||
"Fetching file {:?} from origin {}",
|
||||
file_id,
|
||||
origin.id()
|
||||
);
|
||||
info!("Fetching file {:?} from origin {}", file_id, origin.id());
|
||||
|
||||
let data = origin
|
||||
.read_full(&meta.real_path.path)
|
||||
@@ -91,7 +88,9 @@ impl ContentFetcher {
|
||||
let mut chunk_refs = Vec::with_capacity(chunks.len());
|
||||
for chunk in chunks {
|
||||
if !self.store.exists(&chunk.hash) {
|
||||
self.store.put(chunk.data).await.map_err(FetchError::Store)?;
|
||||
if let Err(e) = self.store.put(chunk.data).await {
|
||||
warn!(hash = %chunk.hash, error = %e, "CAS write failed, continuing in passthrough mode");
|
||||
}
|
||||
}
|
||||
|
||||
chunk_refs.push(ChunkRef {
|
||||
@@ -123,7 +122,7 @@ impl ContentFetcher {
|
||||
}
|
||||
|
||||
pub fn get_file_meta(&self, file_id: FileId) -> Option<FileMeta> {
|
||||
self.file_meta.read().unwrap().get(&file_id).cloned()
|
||||
self.file_meta.read().get(&file_id).cloned()
|
||||
}
|
||||
|
||||
pub fn emit_access_event(&self, meta: &FileMeta, offset: u64, size: u32) {
|
||||
@@ -1,12 +1,13 @@
|
||||
use crate::chunks::ChunkRef;
|
||||
use crate::fetcher::{ContentFetcher, FetchError};
|
||||
use crate::store::CasStore;
|
||||
use crate::store::{CasError, CasStore};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use musicfs_core::FileId;
|
||||
use parking_lot::RwLock;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tracing::{debug, trace};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkManifest {
|
||||
@@ -25,7 +26,12 @@ impl ChunkManifest {
|
||||
rmp_serde::from_slice(data).ok()
|
||||
}
|
||||
|
||||
pub fn from_db(file_id: FileId, total_size: u64, mtime: i64, chunk_blob: &[u8]) -> Option<Self> {
|
||||
pub fn from_db(
|
||||
file_id: FileId,
|
||||
total_size: u64,
|
||||
mtime: i64,
|
||||
chunk_blob: &[u8],
|
||||
) -> Option<Self> {
|
||||
let chunks = Self::chunks_from_bytes(chunk_blob)?;
|
||||
Some(Self {
|
||||
file_id,
|
||||
@@ -60,13 +66,13 @@ impl FileReader {
|
||||
}
|
||||
|
||||
pub fn register_manifest(&self, manifest: ChunkManifest) {
|
||||
let mut manifests = self.manifests.write().unwrap();
|
||||
let mut manifests = self.manifests.write();
|
||||
manifests.insert(manifest.file_id, manifest);
|
||||
}
|
||||
|
||||
async fn get_or_fetch_manifest(&self, file_id: FileId) -> Result<ChunkManifest, ReaderError> {
|
||||
{
|
||||
let manifests = self.manifests.read().unwrap();
|
||||
let manifests = self.manifests.read();
|
||||
if let Some(m) = manifests.get(&file_id) {
|
||||
trace!(file_id = ?file_id, "manifest cache hit");
|
||||
return Ok(m.clone());
|
||||
@@ -79,10 +85,7 @@ impl FileReader {
|
||||
};
|
||||
|
||||
let manifest = fetcher.ensure_cached(file_id).await?;
|
||||
self.manifests
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(file_id, manifest.clone());
|
||||
self.manifests.write().insert(file_id, manifest.clone());
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
@@ -116,7 +119,35 @@ impl FileReader {
|
||||
continue;
|
||||
}
|
||||
|
||||
let chunk_data = self.store.get(&chunk_ref.hash).await?;
|
||||
let chunk_data = match self.store.get(&chunk_ref.hash).await {
|
||||
Ok(data) => data,
|
||||
Err(CasError::IntegrityError { .. }) => {
|
||||
warn!(hash = %chunk_ref.hash, "Chunk corrupt, deleting and re-fetching");
|
||||
let _ = self.store.delete(&chunk_ref.hash).await;
|
||||
if let Some(fetcher) = &self.fetcher {
|
||||
let new_manifest = fetcher.fetch_file(file_id).await?;
|
||||
self.manifests.write().insert(file_id, new_manifest);
|
||||
self.store.get(&chunk_ref.hash).await?
|
||||
} else {
|
||||
return Err(ReaderError::Cas(CasError::NotFound(
|
||||
chunk_ref.hash.as_hex(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(CasError::NotFound(_)) => {
|
||||
warn!(hash = %chunk_ref.hash, "Chunk missing, attempting re-fetch");
|
||||
if let Some(fetcher) = &self.fetcher {
|
||||
let new_manifest = fetcher.fetch_file(file_id).await?;
|
||||
self.manifests.write().insert(file_id, new_manifest);
|
||||
self.store.get(&chunk_ref.hash).await?
|
||||
} else {
|
||||
return Err(ReaderError::Cas(CasError::NotFound(
|
||||
chunk_ref.hash.as_hex(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(ReaderError::Cas(e)),
|
||||
};
|
||||
|
||||
let read_start = if offset > chunk_start {
|
||||
(offset - chunk_start) as usize
|
||||
@@ -4,7 +4,10 @@ use musicfs_core::ChunkHash;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::fs;
|
||||
use tracing::{debug, trace, warn};
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
use fail::fail_point;
|
||||
|
||||
const DEFAULT_MAX_SIZE_10GB: u64 = 10 * 1024 * 1024 * 1024;
|
||||
const DEFAULT_SHARD_LEVELS_256_SUBDIRS: u8 = 2;
|
||||
@@ -42,7 +45,26 @@ impl CasStore {
|
||||
fs::create_dir_all(&config.chunks_dir).await?;
|
||||
|
||||
let index_path = config.chunks_dir.join("index.sled");
|
||||
let index = sled::open(&index_path)?;
|
||||
let index = match sled::open(&index_path) {
|
||||
Ok(db) => db,
|
||||
Err(e) => {
|
||||
warn!(error = %e, path = ?index_path, "sled index corrupted, attempting recovery");
|
||||
|
||||
match sled::Config::new().path(&index_path).open() {
|
||||
Ok(db) => {
|
||||
info!("sled index repaired successfully");
|
||||
db
|
||||
}
|
||||
Err(repair_err) => {
|
||||
warn!(error = %repair_err, "sled repair failed, recreating index");
|
||||
if index_path.exists() {
|
||||
std::fs::remove_dir_all(&index_path).map_err(CasError::Io)?;
|
||||
}
|
||||
sled::open(&index_path)?
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let current_size = Self::calculate_size(&config.chunks_dir).await;
|
||||
|
||||
@@ -54,17 +76,31 @@ impl CasStore {
|
||||
}
|
||||
|
||||
async fn calculate_size(dir: &Path) -> u64 {
|
||||
let mut size = 0u64;
|
||||
if let Ok(mut entries) = fs::read_dir(dir).await {
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
if let Ok(meta) = entry.metadata().await {
|
||||
if meta.is_file() {
|
||||
size += meta.len();
|
||||
Self::calculate_size_recursive(dir).await
|
||||
}
|
||||
|
||||
fn calculate_size_recursive(
|
||||
dir: &Path,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = u64> + Send + '_>> {
|
||||
Box::pin(async move {
|
||||
let mut size = 0u64;
|
||||
if let Ok(mut entries) = fs::read_dir(dir).await {
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
if let Ok(meta) = entry.metadata().await {
|
||||
if meta.is_file() {
|
||||
size += meta.len();
|
||||
} else if meta.is_dir() {
|
||||
// Skip sled index directory
|
||||
let name = entry.file_name();
|
||||
if name != "index.sled" {
|
||||
size += Self::calculate_size_recursive(&entry.path()).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
size
|
||||
size
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
|
||||
@@ -76,12 +112,44 @@ impl CasStore {
|
||||
return Ok(hash);
|
||||
}
|
||||
|
||||
if self.config.max_size > 0 {
|
||||
let new_size = self.current_size.load(Ordering::SeqCst) + data.len() as u64;
|
||||
if new_size > self.config.max_size {
|
||||
warn!(
|
||||
current_size = self.current_size.load(Ordering::SeqCst),
|
||||
chunk_size = data.len(),
|
||||
max_size = self.config.max_size,
|
||||
"CAS store full, rejecting write"
|
||||
);
|
||||
return Err(CasError::StoreFull {
|
||||
current: self.current_size.load(Ordering::SeqCst),
|
||||
max: self.config.max_size,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
fail_point!("cas-put-before-write", |_| {
|
||||
Err(CasError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Failpoint: cas-put-before-write",
|
||||
)))
|
||||
});
|
||||
|
||||
fs::write(&path, data).await?;
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
fail_point!("cas-put-after-write-before-index", |_| {
|
||||
Err(CasError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Failpoint: cas-put-after-write-before-index",
|
||||
)))
|
||||
});
|
||||
|
||||
let location = ChunkLocation {
|
||||
path: path.clone(),
|
||||
size: data.len() as u32,
|
||||
@@ -232,6 +300,9 @@ pub enum CasError {
|
||||
|
||||
#[error("Serialization error: {0}")]
|
||||
Serialization(String),
|
||||
|
||||
#[error("Store full: {current} / {max} bytes")]
|
||||
StoreFull { current: u64, max: u64 },
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
+8
-2
@@ -117,7 +117,10 @@ async fn test_fetcher_cache_miss_flow() {
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let origin_id = OriginId::from("test-origin");
|
||||
let origin = Arc::new(LocalOrigin::new(origin_id.clone(), origin_dir.path().to_path_buf()));
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
origin_id.clone(),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
|
||||
let fetcher = ContentFetcher::new(store.clone());
|
||||
fetcher.register_origin(origin);
|
||||
@@ -163,7 +166,10 @@ async fn test_reader_with_fetcher_integration() {
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let origin_id = OriginId::from("local");
|
||||
let origin = Arc::new(LocalOrigin::new(origin_id.clone(), origin_dir.path().to_path_buf()));
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
origin_id.clone(),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
|
||||
let fetcher = ContentFetcher::new(store.clone());
|
||||
fetcher.register_origin(origin);
|
||||
@@ -14,14 +14,24 @@ musicfs-cache.path = "../musicfs-cache"
|
||||
musicfs-cas.path = "../musicfs-cas"
|
||||
musicfs-fuse.path = "../musicfs-fuse"
|
||||
musicfs-metadata.path = "../musicfs-metadata"
|
||||
musicfs-grpc.path = "../musicfs-grpc"
|
||||
|
||||
clap.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
tonic.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
tracing-appender.workspace = true
|
||||
anyhow.workspace = true
|
||||
dirs.workspace = true
|
||||
toml.workspace = true
|
||||
parking_lot.workspace = true
|
||||
libc.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
tracing-journald.workspace = true
|
||||
sd-notify.workspace = true
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,638 @@
|
||||
//! CLI subcommands for metadata overlay management.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Subcommand;
|
||||
use musicfs_grpc::proto::musicfs::v1::{
|
||||
metadata_service_client::MetadataServiceClient, ClearOverlayRequest, GetMetadataRequest,
|
||||
ImportMetadataRequest, UpdateMetadataRequest,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use tokio_stream::StreamExt;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Metadata overlay management subcommands.
|
||||
#[derive(Subcommand)]
|
||||
pub enum MetadataCommand {
|
||||
/// Get metadata for a file (prints as JSON)
|
||||
Get {
|
||||
/// Virtual path of the file
|
||||
path: String,
|
||||
/// Print only a specific field
|
||||
#[arg(long)]
|
||||
field: Option<String>,
|
||||
},
|
||||
/// Set metadata fields for a file
|
||||
Set {
|
||||
/// Virtual path of the file
|
||||
path: String,
|
||||
/// Track title
|
||||
#[arg(long)]
|
||||
title: Option<String>,
|
||||
/// Artist name
|
||||
#[arg(long)]
|
||||
artist: Option<String>,
|
||||
/// Album name
|
||||
#[arg(long)]
|
||||
album: Option<String>,
|
||||
/// Album artist
|
||||
#[arg(long)]
|
||||
album_artist: Option<String>,
|
||||
/// Track number
|
||||
#[arg(long)]
|
||||
track: Option<u32>,
|
||||
/// Disc number
|
||||
#[arg(long)]
|
||||
disc: Option<u32>,
|
||||
/// Genre
|
||||
#[arg(long)]
|
||||
genre: Option<String>,
|
||||
/// Date (YYYY-MM-DD or YYYY)
|
||||
#[arg(long)]
|
||||
date: Option<String>,
|
||||
/// Composer
|
||||
#[arg(long)]
|
||||
composer: Option<String>,
|
||||
/// Comment
|
||||
#[arg(long)]
|
||||
comment: Option<String>,
|
||||
/// Set metadata from JSON string
|
||||
#[arg(long, conflicts_with_all = ["title", "artist", "album", "album_artist", "track", "disc", "genre", "date", "composer", "comment"])]
|
||||
json: Option<String>,
|
||||
},
|
||||
/// Clear metadata overlay (revert to original)
|
||||
Clear {
|
||||
/// Virtual path of the file
|
||||
path: String,
|
||||
},
|
||||
/// Show difference between current and original metadata
|
||||
Diff {
|
||||
/// Virtual path of the file
|
||||
path: String,
|
||||
},
|
||||
/// Import metadata from CSV or JSON file
|
||||
Import {
|
||||
/// Import file path
|
||||
file: PathBuf,
|
||||
/// File format (csv or json, auto-detected if not specified)
|
||||
#[arg(long)]
|
||||
format: Option<String>,
|
||||
},
|
||||
/// Export metadata to file
|
||||
Export {
|
||||
/// Output file path
|
||||
#[arg(long, short)]
|
||||
output: PathBuf,
|
||||
/// Filter by search query
|
||||
#[arg(long)]
|
||||
query: Option<String>,
|
||||
/// Output format (csv or json, auto-detected from extension)
|
||||
#[arg(long)]
|
||||
format: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Metadata fields for JSON serialization.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct MetadataFields {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub file_id: Option<i64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub artist: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub album: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub album_artist: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub year: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub track: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub disc: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub genre: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub format: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub duration_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bitrate: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub track_total: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub disc_total: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub date: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub composer: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub comment: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lyrics: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub copyright: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub compilation: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub artist_sort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub album_artist_sort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub album_sort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title_sort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_recording_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_album_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_artist_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_album_artist_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_release_group_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub replaygain_track_gain: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub replaygain_track_peak: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub replaygain_album_gain: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub replaygain_album_peak: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub channels: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bits_per_sample: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub encoder: Option<String>,
|
||||
#[serde(skip_serializing_if = "HashMap::is_empty", default)]
|
||||
pub custom_tags: HashMap<String, String>,
|
||||
}
|
||||
|
||||
/// Execute a metadata subcommand.
|
||||
pub async fn run_metadata(command: MetadataCommand, endpoint: &str) -> Result<()> {
|
||||
match command {
|
||||
MetadataCommand::Get { path, field } => run_get(endpoint, &path, field.as_deref()).await,
|
||||
MetadataCommand::Set {
|
||||
path,
|
||||
title,
|
||||
artist,
|
||||
album,
|
||||
album_artist,
|
||||
track,
|
||||
disc,
|
||||
genre,
|
||||
date,
|
||||
composer,
|
||||
comment,
|
||||
json,
|
||||
} => {
|
||||
run_set(
|
||||
endpoint,
|
||||
&path,
|
||||
title,
|
||||
artist,
|
||||
album,
|
||||
album_artist,
|
||||
track,
|
||||
disc,
|
||||
genre,
|
||||
date,
|
||||
composer,
|
||||
comment,
|
||||
json,
|
||||
)
|
||||
.await
|
||||
}
|
||||
MetadataCommand::Clear { path } => run_clear(endpoint, &path).await,
|
||||
MetadataCommand::Diff { path } => run_diff(endpoint, &path).await,
|
||||
MetadataCommand::Import { file, format } => run_import(endpoint, &file, format).await,
|
||||
MetadataCommand::Export {
|
||||
output,
|
||||
query,
|
||||
format,
|
||||
} => run_export(endpoint, &output, query, format).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect(endpoint: &str) -> Result<MetadataServiceClient<Channel>> {
|
||||
MetadataServiceClient::connect(endpoint.to_string())
|
||||
.await
|
||||
.context("Failed to connect to gRPC server")
|
||||
}
|
||||
|
||||
async fn run_get(endpoint: &str, path: &str, field: Option<&str>) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let response = client
|
||||
.get_metadata(GetMetadataRequest {
|
||||
virtual_path: path.to_string(),
|
||||
})
|
||||
.await
|
||||
.context("GetMetadata RPC failed")?;
|
||||
|
||||
let meta = response.into_inner();
|
||||
let fields = MetadataFields {
|
||||
file_id: Some(meta.file_id),
|
||||
title: meta.title,
|
||||
artist: meta.artist,
|
||||
album: meta.album,
|
||||
album_artist: meta.album_artist,
|
||||
year: meta.year,
|
||||
track: meta.track,
|
||||
disc: meta.disc,
|
||||
genre: meta.genre,
|
||||
format: meta.format,
|
||||
duration_ms: meta.duration_ms,
|
||||
bitrate: meta.bitrate,
|
||||
track_total: meta.track_total,
|
||||
disc_total: meta.disc_total,
|
||||
date: meta.date,
|
||||
composer: meta.composer,
|
||||
comment: meta.comment,
|
||||
lyrics: meta.lyrics,
|
||||
copyright: meta.copyright,
|
||||
compilation: meta.compilation,
|
||||
artist_sort: meta.artist_sort,
|
||||
album_artist_sort: meta.album_artist_sort,
|
||||
album_sort: meta.album_sort,
|
||||
title_sort: meta.title_sort,
|
||||
mb_recording_id: meta.mb_recording_id,
|
||||
mb_album_id: meta.mb_album_id,
|
||||
mb_artist_id: meta.mb_artist_id,
|
||||
mb_album_artist_id: meta.mb_album_artist_id,
|
||||
mb_release_group_id: meta.mb_release_group_id,
|
||||
replaygain_track_gain: meta.replaygain_track_gain,
|
||||
replaygain_track_peak: meta.replaygain_track_peak,
|
||||
replaygain_album_gain: meta.replaygain_album_gain,
|
||||
replaygain_album_peak: meta.replaygain_album_peak,
|
||||
channels: meta.channels,
|
||||
bits_per_sample: meta.bits_per_sample,
|
||||
encoder: meta.encoder,
|
||||
custom_tags: meta.custom_tags,
|
||||
};
|
||||
|
||||
if let Some(field_name) = field {
|
||||
let value = get_field_value(&fields, field_name)?;
|
||||
println!("{}", value);
|
||||
} else {
|
||||
let json = serde_json::to_string_pretty(&fields)?;
|
||||
println!("{}", json);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_field_value(fields: &MetadataFields, field_name: &str) -> Result<String> {
|
||||
let value = match field_name {
|
||||
"file_id" => fields.file_id.map(|v| v.to_string()),
|
||||
"title" => fields.title.clone(),
|
||||
"artist" => fields.artist.clone(),
|
||||
"album" => fields.album.clone(),
|
||||
"album_artist" => fields.album_artist.clone(),
|
||||
"year" => fields.year.map(|v| v.to_string()),
|
||||
"track" => fields.track.map(|v| v.to_string()),
|
||||
"disc" => fields.disc.map(|v| v.to_string()),
|
||||
"genre" => fields.genre.clone(),
|
||||
"format" => fields.format.clone(),
|
||||
"duration_ms" => fields.duration_ms.map(|v| v.to_string()),
|
||||
"bitrate" => fields.bitrate.map(|v| v.to_string()),
|
||||
"track_total" => fields.track_total.map(|v| v.to_string()),
|
||||
"disc_total" => fields.disc_total.map(|v| v.to_string()),
|
||||
"date" => fields.date.clone(),
|
||||
"composer" => fields.composer.clone(),
|
||||
"comment" => fields.comment.clone(),
|
||||
"lyrics" => fields.lyrics.clone(),
|
||||
"copyright" => fields.copyright.clone(),
|
||||
"compilation" => fields.compilation.map(|v| v.to_string()),
|
||||
"artist_sort" => fields.artist_sort.clone(),
|
||||
"album_artist_sort" => fields.album_artist_sort.clone(),
|
||||
"album_sort" => fields.album_sort.clone(),
|
||||
"title_sort" => fields.title_sort.clone(),
|
||||
"mb_recording_id" => fields.mb_recording_id.clone(),
|
||||
"mb_album_id" => fields.mb_album_id.clone(),
|
||||
"mb_artist_id" => fields.mb_artist_id.clone(),
|
||||
"mb_album_artist_id" => fields.mb_album_artist_id.clone(),
|
||||
"mb_release_group_id" => fields.mb_release_group_id.clone(),
|
||||
"replaygain_track_gain" => fields.replaygain_track_gain.map(|v| v.to_string()),
|
||||
"replaygain_track_peak" => fields.replaygain_track_peak.map(|v| v.to_string()),
|
||||
"replaygain_album_gain" => fields.replaygain_album_gain.map(|v| v.to_string()),
|
||||
"replaygain_album_peak" => fields.replaygain_album_peak.map(|v| v.to_string()),
|
||||
"channels" => fields.channels.map(|v| v.to_string()),
|
||||
"bits_per_sample" => fields.bits_per_sample.map(|v| v.to_string()),
|
||||
"encoder" => fields.encoder.clone(),
|
||||
_ => return Err(anyhow::anyhow!("Unknown field: {}", field_name)),
|
||||
};
|
||||
|
||||
Ok(value.unwrap_or_else(|| "null".to_string()))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_set(
|
||||
endpoint: &str,
|
||||
path: &str,
|
||||
title: Option<String>,
|
||||
artist: Option<String>,
|
||||
album: Option<String>,
|
||||
album_artist: Option<String>,
|
||||
track: Option<u32>,
|
||||
disc: Option<u32>,
|
||||
genre: Option<String>,
|
||||
date: Option<String>,
|
||||
composer: Option<String>,
|
||||
comment: Option<String>,
|
||||
json: Option<String>,
|
||||
) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let get_response = client
|
||||
.get_metadata(GetMetadataRequest {
|
||||
virtual_path: path.to_string(),
|
||||
})
|
||||
.await
|
||||
.context("Failed to get file metadata")?;
|
||||
|
||||
let file_id = get_response.into_inner().file_id;
|
||||
|
||||
let request = if let Some(json_str) = json {
|
||||
let fields: MetadataFields =
|
||||
serde_json::from_str(&json_str).context("Failed to parse JSON metadata")?;
|
||||
UpdateMetadataRequest {
|
||||
file_id,
|
||||
title: fields.title,
|
||||
artist: fields.artist,
|
||||
album: fields.album,
|
||||
album_artist: fields.album_artist,
|
||||
track_number: fields.track,
|
||||
disc_number: fields.disc,
|
||||
genre: fields.genre,
|
||||
date: fields.date,
|
||||
composer: fields.composer,
|
||||
comment: fields.comment,
|
||||
lyrics: fields.lyrics,
|
||||
copyright: fields.copyright,
|
||||
compilation: fields.compilation,
|
||||
artist_sort: fields.artist_sort,
|
||||
album_artist_sort: fields.album_artist_sort,
|
||||
album_sort: fields.album_sort,
|
||||
title_sort: fields.title_sort,
|
||||
mb_recording_id: fields.mb_recording_id,
|
||||
mb_album_id: fields.mb_album_id,
|
||||
mb_artist_id: fields.mb_artist_id,
|
||||
replaygain_track_gain: fields.replaygain_track_gain,
|
||||
replaygain_track_peak: fields.replaygain_track_peak,
|
||||
replaygain_album_gain: fields.replaygain_album_gain,
|
||||
replaygain_album_peak: fields.replaygain_album_peak,
|
||||
label: None,
|
||||
album_type: None,
|
||||
cover_url: None,
|
||||
custom_tags: fields.custom_tags,
|
||||
}
|
||||
} else {
|
||||
UpdateMetadataRequest {
|
||||
file_id,
|
||||
title,
|
||||
artist,
|
||||
album,
|
||||
album_artist,
|
||||
track_number: track,
|
||||
disc_number: disc,
|
||||
genre,
|
||||
date,
|
||||
composer,
|
||||
comment,
|
||||
lyrics: None,
|
||||
copyright: None,
|
||||
compilation: None,
|
||||
artist_sort: None,
|
||||
album_artist_sort: None,
|
||||
album_sort: None,
|
||||
title_sort: None,
|
||||
mb_recording_id: None,
|
||||
mb_album_id: None,
|
||||
mb_artist_id: None,
|
||||
replaygain_track_gain: None,
|
||||
replaygain_track_peak: None,
|
||||
replaygain_album_gain: None,
|
||||
replaygain_album_peak: None,
|
||||
label: None,
|
||||
album_type: None,
|
||||
cover_url: None,
|
||||
custom_tags: HashMap::new(),
|
||||
}
|
||||
};
|
||||
|
||||
let response = client
|
||||
.update_metadata(request)
|
||||
.await
|
||||
.context("UpdateMetadata RPC failed")?;
|
||||
|
||||
let result = response.into_inner();
|
||||
if result.success {
|
||||
info!(file_id = result.file_id, "Metadata updated successfully");
|
||||
println!("Metadata updated for file_id={}", result.file_id);
|
||||
} else {
|
||||
let msg = result
|
||||
.error_message
|
||||
.unwrap_or_else(|| "Unknown error".to_string());
|
||||
anyhow::bail!("Failed to update metadata: {}", msg);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_clear(endpoint: &str, path: &str) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let get_response = client
|
||||
.get_metadata(GetMetadataRequest {
|
||||
virtual_path: path.to_string(),
|
||||
})
|
||||
.await
|
||||
.context("Failed to get file metadata")?;
|
||||
|
||||
let file_id = get_response.into_inner().file_id;
|
||||
|
||||
let response = client
|
||||
.clear_overlay(ClearOverlayRequest { file_id })
|
||||
.await
|
||||
.context("ClearOverlay RPC failed")?;
|
||||
|
||||
let result = response.into_inner();
|
||||
if result.success {
|
||||
info!(file_id = result.file_id, "Overlay cleared successfully");
|
||||
println!("Metadata overlay cleared for file_id={}", result.file_id);
|
||||
} else {
|
||||
let msg = result
|
||||
.error_message
|
||||
.unwrap_or_else(|| "Unknown error".to_string());
|
||||
anyhow::bail!("Failed to clear overlay: {}", msg);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_diff(endpoint: &str, path: &str) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let response = client
|
||||
.get_metadata(GetMetadataRequest {
|
||||
virtual_path: path.to_string(),
|
||||
})
|
||||
.await
|
||||
.context("GetMetadata RPC failed")?;
|
||||
|
||||
let meta = response.into_inner();
|
||||
debug!(file_id = meta.file_id, "Retrieved metadata for diff");
|
||||
|
||||
println!("Current metadata for: {}", path);
|
||||
println!("---");
|
||||
|
||||
let fields = MetadataFields {
|
||||
file_id: Some(meta.file_id),
|
||||
title: meta.title,
|
||||
artist: meta.artist,
|
||||
album: meta.album,
|
||||
album_artist: meta.album_artist,
|
||||
year: meta.year,
|
||||
track: meta.track,
|
||||
disc: meta.disc,
|
||||
genre: meta.genre,
|
||||
format: meta.format,
|
||||
duration_ms: meta.duration_ms,
|
||||
bitrate: meta.bitrate,
|
||||
track_total: meta.track_total,
|
||||
disc_total: meta.disc_total,
|
||||
date: meta.date,
|
||||
composer: meta.composer,
|
||||
comment: meta.comment,
|
||||
lyrics: meta.lyrics,
|
||||
copyright: meta.copyright,
|
||||
compilation: meta.compilation,
|
||||
artist_sort: meta.artist_sort,
|
||||
album_artist_sort: meta.album_artist_sort,
|
||||
album_sort: meta.album_sort,
|
||||
title_sort: meta.title_sort,
|
||||
mb_recording_id: meta.mb_recording_id,
|
||||
mb_album_id: meta.mb_album_id,
|
||||
mb_artist_id: meta.mb_artist_id,
|
||||
mb_album_artist_id: meta.mb_album_artist_id,
|
||||
mb_release_group_id: meta.mb_release_group_id,
|
||||
replaygain_track_gain: meta.replaygain_track_gain,
|
||||
replaygain_track_peak: meta.replaygain_track_peak,
|
||||
replaygain_album_gain: meta.replaygain_album_gain,
|
||||
replaygain_album_peak: meta.replaygain_album_peak,
|
||||
channels: meta.channels,
|
||||
bits_per_sample: meta.bits_per_sample,
|
||||
encoder: meta.encoder,
|
||||
custom_tags: meta.custom_tags,
|
||||
};
|
||||
|
||||
let json = serde_json::to_string_pretty(&fields)?;
|
||||
println!("{}", json);
|
||||
println!("---");
|
||||
println!("Note: Original metadata comparison requires re-parsing the source file.");
|
||||
println!("Use 'musicfs metadata clear <path>' to revert to original metadata.");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_import(endpoint: &str, file: &PathBuf, format: Option<String>) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let file_format = format.or_else(|| {
|
||||
file.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|s| s.to_lowercase())
|
||||
});
|
||||
|
||||
let source_path = file
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| file.clone())
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
|
||||
info!(source_path = %source_path, format = ?file_format, "Starting metadata import");
|
||||
|
||||
let response = client
|
||||
.import_metadata(ImportMetadataRequest {
|
||||
source_path,
|
||||
format: file_format,
|
||||
})
|
||||
.await
|
||||
.context("ImportMetadata RPC failed")?;
|
||||
|
||||
let mut stream = response.into_inner();
|
||||
let mut last_imported = 0u32;
|
||||
let mut last_total = 0u32;
|
||||
let mut errors = Vec::new();
|
||||
|
||||
while let Some(progress) = stream.next().await {
|
||||
let progress = progress.context("Stream error")?;
|
||||
last_imported = progress.imported;
|
||||
last_total = progress.total;
|
||||
|
||||
if let Some(ref err) = progress.error_message {
|
||||
let file = progress.current_file.as_deref().unwrap_or("unknown");
|
||||
errors.push(format!("{}: {}", file, err));
|
||||
}
|
||||
|
||||
if let Some(ref current) = progress.current_file {
|
||||
print!(
|
||||
"\rImporting: {}/{} - {}",
|
||||
progress.imported, progress.total, current
|
||||
);
|
||||
std::io::Write::flush(&mut std::io::stdout())?;
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
println!(
|
||||
"Import complete: {}/{} files imported",
|
||||
last_imported, last_total
|
||||
);
|
||||
|
||||
if !errors.is_empty() {
|
||||
println!("\nErrors ({}):", errors.len());
|
||||
for err in errors.iter().take(10) {
|
||||
println!(" - {}", err);
|
||||
}
|
||||
if errors.len() > 10 {
|
||||
println!(" ... and {} more", errors.len() - 10);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_export(
|
||||
_endpoint: &str,
|
||||
output: &PathBuf,
|
||||
query: Option<String>,
|
||||
format: Option<String>,
|
||||
) -> Result<()> {
|
||||
let output_format = format.or_else(|| {
|
||||
output
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|s| s.to_lowercase())
|
||||
});
|
||||
|
||||
println!("Export metadata to: {}", output.display());
|
||||
if let Some(ref q) = query {
|
||||
println!("Filter query: {}", q);
|
||||
}
|
||||
println!("Format: {}", output_format.as_deref().unwrap_or("json"));
|
||||
println!();
|
||||
println!("Note: Export requires file listing capability.");
|
||||
println!("This feature requires integration with the Search service.");
|
||||
println!(
|
||||
"Use 'musicfs search <query>' to find files, then 'musicfs metadata get <path>' for each."
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -12,6 +12,7 @@ tokio = { workspace = true, features = ["sync"] }
|
||||
tracing.workspace = true
|
||||
xxhash-rust.workspace = true
|
||||
hex.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -23,6 +23,9 @@ pub enum Error {
|
||||
#[error("Database error: {0}")]
|
||||
Database(String),
|
||||
|
||||
#[error("Database corrupted: {0}")]
|
||||
DatabaseCorrupted(String),
|
||||
|
||||
#[error("NFS stale file handle")]
|
||||
NfsStaleHandle,
|
||||
|
||||
@@ -16,7 +16,10 @@ impl EventBus {
|
||||
trace!(event = ?event, "Publishing event");
|
||||
let receiver_count = self.sender.receiver_count();
|
||||
if self.sender.send(event).is_err() && receiver_count > 0 {
|
||||
debug!(receiver_count = receiver_count, "Event dropped, no active receivers");
|
||||
debug!(
|
||||
receiver_count = receiver_count,
|
||||
"Event dropped, no active receivers"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
pub mod config;
|
||||
pub mod credentials;
|
||||
pub mod error;
|
||||
pub mod events;
|
||||
pub mod metrics;
|
||||
pub mod resolver;
|
||||
pub mod supervisor;
|
||||
pub mod types;
|
||||
|
||||
pub use config::{
|
||||
CacheConfig, Config, ConfigError, HealthConfig, LoggingConfig, OriginConfig, OriginType,
|
||||
};
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub fn sanitize_path(path: &Path) -> String {
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
path.to_string_lossy().replace(&home, "~")
|
||||
} else {
|
||||
path.to_string_lossy().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Install a custom panic hook that logs panics via tracing before the default behavior.
|
||||
/// This ensures panics are captured in log files and journald.
|
||||
pub fn install_panic_hook() {
|
||||
let default_hook = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(move |info| {
|
||||
let thread = std::thread::current();
|
||||
let thread_name = thread.name().unwrap_or("<unnamed>");
|
||||
|
||||
let message = if let Some(s) = info.payload().downcast_ref::<&str>() {
|
||||
(*s).to_string()
|
||||
} else if let Some(s) = info.payload().downcast_ref::<String>() {
|
||||
s.clone()
|
||||
} else {
|
||||
"unknown panic".to_string()
|
||||
};
|
||||
|
||||
let location = info
|
||||
.location()
|
||||
.map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
|
||||
.unwrap_or_else(|| "unknown location".to_string());
|
||||
|
||||
tracing::error!(
|
||||
thread = thread_name,
|
||||
location = %location,
|
||||
"PANIC: {}",
|
||||
message
|
||||
);
|
||||
|
||||
default_hook(info);
|
||||
}));
|
||||
}
|
||||
pub use credentials::{Credential, CredentialConfig, CredentialError, CredentialStore};
|
||||
pub use error::{Error, Result};
|
||||
pub use events::{Event, EventBus};
|
||||
pub use metrics::{CacheMetrics, FuseOpsMetrics, Metrics, OriginsMetrics};
|
||||
pub use resolver::{PathResolver, PathTemplate};
|
||||
pub use types::*;
|
||||
@@ -1,6 +1,6 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::RwLock;
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -22,9 +22,7 @@ impl Metrics {
|
||||
}
|
||||
|
||||
pub fn uptime_secs(&self) -> u64 {
|
||||
self.start_time
|
||||
.map(|t| t.elapsed().as_secs())
|
||||
.unwrap_or(0)
|
||||
self.start_time.map(|t| t.elapsed().as_secs()).unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn to_prometheus(&self) -> String {
|
||||
@@ -45,7 +43,7 @@ impl Metrics {
|
||||
self.fuse_ops.open.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
for (op, histogram) in self.fuse_latency.histograms.read().unwrap().iter() {
|
||||
for (op, histogram) in self.fuse_latency.histograms.read().iter() {
|
||||
let quantiles = histogram.quantiles();
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_fuse_latency_seconds FUSE operation latency\n\
|
||||
@@ -55,11 +53,16 @@ impl Metrics {
|
||||
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.99\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds_sum{{op=\"{}\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds_count{{op=\"{}\"}} {}\n",
|
||||
op, quantiles.p50,
|
||||
op, quantiles.p95,
|
||||
op, quantiles.p99,
|
||||
op, histogram.sum_secs(),
|
||||
op, histogram.count(),
|
||||
op,
|
||||
quantiles.p50,
|
||||
op,
|
||||
quantiles.p95,
|
||||
op,
|
||||
quantiles.p99,
|
||||
op,
|
||||
histogram.sum_secs(),
|
||||
op,
|
||||
histogram.count(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -95,7 +98,7 @@ impl Metrics {
|
||||
"# HELP musicfs_origin_health Origin health status (1=healthy, 0=unhealthy)\n\
|
||||
# TYPE musicfs_origin_health gauge\n",
|
||||
);
|
||||
for (origin_id, healthy) in self.origin_health.status.read().unwrap().iter() {
|
||||
for (origin_id, healthy) in self.origin_health.status.read().iter() {
|
||||
output.push_str(&format!(
|
||||
"musicfs_origin_health{{origin=\"{}\"}} {}\n",
|
||||
origin_id,
|
||||
@@ -203,7 +206,7 @@ pub struct FuseLatencyMetrics {
|
||||
|
||||
impl FuseLatencyMetrics {
|
||||
pub fn record(&self, op: &str, latency_secs: f64) {
|
||||
let mut histograms = self.histograms.write().unwrap();
|
||||
let mut histograms = self.histograms.write();
|
||||
histograms
|
||||
.entry(op.to_string())
|
||||
.or_default()
|
||||
@@ -266,10 +269,7 @@ pub struct OriginHealthMetrics {
|
||||
|
||||
impl OriginHealthMetrics {
|
||||
pub fn set_health(&self, origin_id: &str, healthy: bool) {
|
||||
self.status
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(origin_id.to_string(), healthy);
|
||||
self.status.write().insert(origin_id.to_string(), healthy);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{error, warn};
|
||||
|
||||
pub struct TaskSupervisor {
|
||||
tasks: Arc<RwLock<HashMap<String, TaskEntry>>>,
|
||||
}
|
||||
|
||||
struct TaskEntry {
|
||||
handle: JoinHandle<()>,
|
||||
status: TaskStatus,
|
||||
restart_count: u32,
|
||||
last_restart: Option<Instant>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum TaskStatus {
|
||||
Running,
|
||||
Failed { error: String, at: Instant },
|
||||
Restarting { attempt: u32 },
|
||||
Stopped,
|
||||
}
|
||||
|
||||
impl Default for TaskSupervisor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl TaskSupervisor {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tasks: Arc::new(RwLock::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn spawn_supervised<F>(&self, name: &str, future: F)
|
||||
where
|
||||
F: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let name_owned = name.to_string();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
future.await;
|
||||
});
|
||||
|
||||
self.tasks.write().insert(
|
||||
name_owned,
|
||||
TaskEntry {
|
||||
handle,
|
||||
status: TaskStatus::Running,
|
||||
restart_count: 0,
|
||||
last_restart: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn spawn_critical<F, Fut>(&self, name: &str, factory: F)
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync + 'static,
|
||||
Fut: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let tasks = self.tasks.clone();
|
||||
let name_owned = name.to_string();
|
||||
|
||||
let monitor_handle = tokio::spawn(async move {
|
||||
let mut restart_count = 0u32;
|
||||
let max_restarts = 5u32;
|
||||
let backoff_durations = [
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(30),
|
||||
];
|
||||
|
||||
loop {
|
||||
let handle = tokio::spawn(factory());
|
||||
|
||||
{
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Running;
|
||||
}
|
||||
}
|
||||
|
||||
match handle.await {
|
||||
Ok(()) => {
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Stopped;
|
||||
}
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
restart_count += 1;
|
||||
|
||||
if restart_count > max_restarts {
|
||||
error!(task = %name_owned, "Task exceeded max restarts ({}), giving up", max_restarts);
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: format!("Exceeded max restarts: {}", e),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
let backoff_idx =
|
||||
(restart_count as usize - 1).min(backoff_durations.len() - 1);
|
||||
let backoff = backoff_durations[backoff_idx];
|
||||
|
||||
warn!(
|
||||
task = %name_owned,
|
||||
error = %e,
|
||||
attempt = restart_count,
|
||||
backoff_ms = backoff.as_millis() as u64,
|
||||
"Critical task failed, restarting with backoff"
|
||||
);
|
||||
|
||||
{
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Restarting {
|
||||
attempt: restart_count,
|
||||
};
|
||||
entry.restart_count = restart_count;
|
||||
entry.last_restart = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::sleep(backoff).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
self.tasks.write().insert(
|
||||
name.to_string(),
|
||||
TaskEntry {
|
||||
handle: monitor_handle,
|
||||
status: TaskStatus::Running,
|
||||
restart_count: 0,
|
||||
last_restart: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn task_status(&self, name: &str) -> TaskStatus {
|
||||
let mut tasks = self.tasks.write();
|
||||
if let Some(entry) = tasks.get_mut(name) {
|
||||
if entry.handle.is_finished() {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: "Task exited".into(),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
entry.status.clone()
|
||||
} else {
|
||||
TaskStatus::Stopped
|
||||
}
|
||||
}
|
||||
|
||||
pub fn check_all(&self) -> Vec<(String, TaskStatus)> {
|
||||
let mut tasks = self.tasks.write();
|
||||
tasks
|
||||
.iter_mut()
|
||||
.map(|(name, entry)| {
|
||||
if entry.handle.is_finished() {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: "Task exited".into(),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
(name.clone(), entry.status.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
@@ -132,6 +132,30 @@ pub struct AudioMeta {
|
||||
pub bitrate: Option<u32>,
|
||||
pub sample_rate: Option<u32>,
|
||||
pub format: AudioFormat,
|
||||
pub track_total: Option<u32>,
|
||||
pub disc_total: Option<u32>,
|
||||
pub date: Option<String>,
|
||||
pub composer: Option<String>,
|
||||
pub comment: Option<String>,
|
||||
pub lyrics: Option<String>,
|
||||
pub copyright: Option<String>,
|
||||
pub compilation: Option<bool>,
|
||||
pub artist_sort: Option<String>,
|
||||
pub album_artist_sort: Option<String>,
|
||||
pub album_sort: Option<String>,
|
||||
pub title_sort: Option<String>,
|
||||
pub mb_recording_id: Option<String>,
|
||||
pub mb_album_id: Option<String>,
|
||||
pub mb_artist_id: Option<String>,
|
||||
pub mb_album_artist_id: Option<String>,
|
||||
pub mb_release_group_id: Option<String>,
|
||||
pub replaygain_track_gain: Option<f32>,
|
||||
pub replaygain_track_peak: Option<f32>,
|
||||
pub replaygain_album_gain: Option<f32>,
|
||||
pub replaygain_album_peak: Option<f32>,
|
||||
pub channels: Option<u32>,
|
||||
pub bits_per_sample: Option<u32>,
|
||||
pub encoder: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -0,0 +1,954 @@
|
||||
use crate::ops::SearchOps;
|
||||
use fuser::{
|
||||
FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, ReplyOpen,
|
||||
Request,
|
||||
};
|
||||
use musicfs_cache::{
|
||||
Database, OverlayError, OverlayReader, RemoveError, RenameError, VirtualNode, VirtualTree,
|
||||
ROOT_INODE,
|
||||
};
|
||||
use musicfs_cas::FileReader;
|
||||
use musicfs_core::{Result, VirtualPath};
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::OsStr;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::runtime::Handle;
|
||||
use tracing::{debug, info, instrument, trace, warn};
|
||||
|
||||
const TTL: Duration = Duration::from_secs(1);
|
||||
const BLOCK_SIZE: u32 = 512;
|
||||
const SEARCH_QUERY_INODE_BASE: u64 = 0xFFFF_FFFF_0000_0100;
|
||||
|
||||
pub struct MusicFs {
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
reader: Option<Arc<FileReader>>,
|
||||
db: Option<Arc<Database>>,
|
||||
overlay_reader: Option<Arc<OverlayReader>>,
|
||||
runtime_handle: Handle,
|
||||
search_ops: Option<SearchOps>,
|
||||
query_inodes: RwLock<HashMap<String, u64>>,
|
||||
inode_queries: RwLock<HashMap<u64, String>>,
|
||||
next_query_inode: RwLock<u64>,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
}
|
||||
|
||||
impl MusicFs {
|
||||
pub fn new(tree: Arc<RwLock<VirtualTree>>, runtime_handle: Handle) -> Self {
|
||||
Self {
|
||||
tree,
|
||||
reader: None,
|
||||
db: None,
|
||||
overlay_reader: None,
|
||||
runtime_handle,
|
||||
search_ops: None,
|
||||
query_inodes: RwLock::new(HashMap::new()),
|
||||
inode_queries: RwLock::new(HashMap::new()),
|
||||
next_query_inode: RwLock::new(SEARCH_QUERY_INODE_BASE),
|
||||
uid: unsafe { libc::getuid() },
|
||||
gid: unsafe { libc::getgid() },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_reader(
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
reader: Arc<FileReader>,
|
||||
runtime_handle: Handle,
|
||||
) -> Self {
|
||||
Self {
|
||||
tree,
|
||||
reader: Some(reader),
|
||||
db: None,
|
||||
overlay_reader: None,
|
||||
runtime_handle,
|
||||
search_ops: None,
|
||||
query_inodes: RwLock::new(HashMap::new()),
|
||||
inode_queries: RwLock::new(HashMap::new()),
|
||||
next_query_inode: RwLock::new(SEARCH_QUERY_INODE_BASE),
|
||||
uid: unsafe { libc::getuid() },
|
||||
gid: unsafe { libc::getgid() },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_db(mut self, db: Arc<Database>) -> Self {
|
||||
self.db = Some(db);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_overlay(mut self, overlay: Arc<OverlayReader>) -> Self {
|
||||
self.overlay_reader = Some(overlay);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_search(mut self, search_ops: SearchOps) -> Self {
|
||||
self.search_ops = Some(search_ops);
|
||||
self
|
||||
}
|
||||
|
||||
fn resolve_path(&self, parent_inode: u64, name: &OsStr) -> Option<VirtualPath> {
|
||||
let tree = self.tree.read();
|
||||
let parent_path = self.inode_to_path_inner(&tree, parent_inode)?;
|
||||
let name_str = name.to_string_lossy();
|
||||
let full_path = if parent_path == "/" {
|
||||
format!("/{}", name_str)
|
||||
} else {
|
||||
format!("{}/{}", parent_path, name_str)
|
||||
};
|
||||
Some(VirtualPath::new(full_path))
|
||||
}
|
||||
|
||||
fn inode_to_path_inner(&self, tree: &VirtualTree, inode: u64) -> Option<String> {
|
||||
for (path, &ino) in tree.path_to_inode_iter() {
|
||||
if ino == inode {
|
||||
return Some(path.as_str().to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn get_or_create_query_inode(&self, query: &str) -> u64 {
|
||||
let query_inodes = self.query_inodes.read();
|
||||
if let Some(&inode) = query_inodes.get(query) {
|
||||
return inode;
|
||||
}
|
||||
drop(query_inodes);
|
||||
|
||||
let mut query_inodes = self.query_inodes.write();
|
||||
let mut inode_queries = self.inode_queries.write();
|
||||
let mut next_inode = self.next_query_inode.write();
|
||||
|
||||
if let Some(&inode) = query_inodes.get(query) {
|
||||
return inode;
|
||||
}
|
||||
|
||||
let inode = *next_inode;
|
||||
*next_inode += 1;
|
||||
query_inodes.insert(query.to_string(), inode);
|
||||
inode_queries.insert(inode, query.to_string());
|
||||
inode
|
||||
}
|
||||
|
||||
fn get_query_for_inode(&self, inode: u64) -> Option<String> {
|
||||
self.inode_queries.read().get(&inode).cloned()
|
||||
}
|
||||
|
||||
pub fn mount(self, mountpoint: &Path) -> Result<()> {
|
||||
info!("Mounting MusicFS at {:?}", mountpoint);
|
||||
|
||||
let options = vec![
|
||||
fuser::MountOption::FSName("musicfs".to_string()),
|
||||
fuser::MountOption::AutoUnmount,
|
||||
fuser::MountOption::AllowOther,
|
||||
];
|
||||
|
||||
fuser::mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn spawn_mount(self, mountpoint: &Path) -> Result<fuser::BackgroundSession> {
|
||||
info!("Mounting MusicFS at {:?}", mountpoint);
|
||||
|
||||
let options = vec![
|
||||
fuser::MountOption::FSName("musicfs".to_string()),
|
||||
fuser::MountOption::AutoUnmount,
|
||||
fuser::MountOption::AllowOther,
|
||||
];
|
||||
|
||||
let session =
|
||||
fuser::spawn_mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
|
||||
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
fn node_to_attr(&self, node: &VirtualNode) -> FileAttr {
|
||||
match node {
|
||||
VirtualNode::Directory(dir) => FileAttr {
|
||||
ino: dir.inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: dir.mtime,
|
||||
mtime: dir.mtime,
|
||||
ctime: dir.mtime,
|
||||
crtime: dir.mtime,
|
||||
kind: FileType::Directory,
|
||||
perm: 0o755,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: BLOCK_SIZE,
|
||||
flags: 0,
|
||||
},
|
||||
VirtualNode::File(file) => FileAttr {
|
||||
ino: file.inode,
|
||||
size: file.size,
|
||||
blocks: (file.size + BLOCK_SIZE as u64 - 1) / BLOCK_SIZE as u64,
|
||||
atime: file.mtime,
|
||||
mtime: file.mtime,
|
||||
ctime: file.mtime,
|
||||
crtime: file.mtime,
|
||||
kind: FileType::RegularFile,
|
||||
perm: 0o644,
|
||||
nlink: 1,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: BLOCK_SIZE,
|
||||
flags: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Filesystem for MusicFs {
|
||||
fn init(
|
||||
&mut self,
|
||||
_req: &Request<'_>,
|
||||
_config: &mut fuser::KernelConfig,
|
||||
) -> std::result::Result<(), libc::c_int> {
|
||||
info!("MusicFS initialized");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn destroy(&mut self) {
|
||||
info!("MusicFS destroyed");
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) {
|
||||
let name_str = name.to_string_lossy();
|
||||
|
||||
if parent == ROOT_INODE && SearchOps::is_search_dir_name(&name_str) {
|
||||
trace!(parent, name = %name_str, "search_dir_name matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.lookup_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if parent == SearchOps::search_dir_inode() {
|
||||
trace!(parent, name = %name_str, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
let inode = self.get_or_create_query_inode(&name_str);
|
||||
search_ops.lookup_query_dir(&name_str, inode, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(query) = self.get_query_for_inode(parent) {
|
||||
trace!(parent, name = %name_str, query = %query, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
let inode = self.get_or_create_query_inode(&format!("{}:{}", query, name_str));
|
||||
search_ops.lookup_result(inode, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(inode) = tree.lookup(parent, name) {
|
||||
trace!(parent, name = %name_str, ino = inode, "file found in tree");
|
||||
if let Some(node) = tree.get(inode) {
|
||||
let attr = self.node_to_attr(node);
|
||||
reply.entry(&TTL, &attr, 0);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
trace!(parent, name = %name_str, "file not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) {
|
||||
if ino == SearchOps::search_dir_inode() {
|
||||
trace!(ino, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if SearchOps::is_search_inode(ino) {
|
||||
trace!(ino, "search_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_result(ino, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if self.get_query_for_inode(ino).is_some() {
|
||||
trace!(ino, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(node) = tree.get(ino) {
|
||||
trace!(ino, "inode found in tree");
|
||||
let mut attr = self.node_to_attr(node);
|
||||
|
||||
if let VirtualNode::File(file) = node {
|
||||
if let Some(ref overlay) = self.overlay_reader {
|
||||
match overlay.estimate_virtual_size(file.file_id) {
|
||||
Ok(Some(virtual_size)) => {
|
||||
trace!(ino, file_id = ?file.file_id, virtual_size, "using overlay virtual size");
|
||||
attr.size = virtual_size;
|
||||
attr.blocks =
|
||||
(virtual_size + BLOCK_SIZE as u64 - 1) / BLOCK_SIZE as u64;
|
||||
}
|
||||
Ok(None) => {
|
||||
trace!(ino, file_id = ?file.file_id, "no overlay, using original size");
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(ino, file_id = ?file.file_id, error = %e, "overlay size estimation failed, using original");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
reply.attr(&TTL, &attr);
|
||||
} else {
|
||||
trace!(ino, "inode not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn readdir(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
offset: i64,
|
||||
mut reply: ReplyDirectory,
|
||||
) {
|
||||
if ino == SearchOps::search_dir_inode() {
|
||||
trace!(ino, offset, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readdir_search_root(offset, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(query) = self.get_query_for_inode(ino) {
|
||||
trace!(ino, offset, query = %query, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readdir_query(&query, offset, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(children) = tree.readdir(ino) {
|
||||
trace!(
|
||||
ino,
|
||||
offset,
|
||||
children_count = children.len(),
|
||||
"directory found"
|
||||
);
|
||||
let parent_ino = tree.get_parent(ino).unwrap_or(ROOT_INODE);
|
||||
|
||||
let entries: Vec<(u64, FileType, &str)> = vec![
|
||||
(ino, FileType::Directory, "."),
|
||||
(parent_ino, FileType::Directory, ".."),
|
||||
];
|
||||
|
||||
let child_entries: Vec<(u64, FileType, String)> = children
|
||||
.iter()
|
||||
.map(|(name, child_ino, is_dir)| {
|
||||
let kind = if *is_dir {
|
||||
FileType::Directory
|
||||
} else {
|
||||
FileType::RegularFile
|
||||
};
|
||||
(*child_ino, kind, name.to_string_lossy().to_string())
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, name) {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let base_offset = entries.len();
|
||||
for (i, (inode, kind, name)) in child_entries.iter().enumerate() {
|
||||
let entry_offset = base_offset + i;
|
||||
if entry_offset < offset as usize {
|
||||
continue;
|
||||
}
|
||||
if reply.add(*inode, (entry_offset + 1) as i64, *kind, name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
reply.ok();
|
||||
} else {
|
||||
trace!(ino, offset, "directory not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn open(&mut self, _req: &Request, ino: u64, flags: i32, reply: ReplyOpen) {
|
||||
let write_flags = libc::O_WRONLY | libc::O_RDWR | libc::O_APPEND | libc::O_TRUNC;
|
||||
if flags & write_flags != 0 {
|
||||
trace!(ino, flags, "write flags detected");
|
||||
reply.error(libc::EROFS);
|
||||
return;
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if tree.get(ino).is_some() {
|
||||
trace!(ino, "inode found");
|
||||
reply.opened(0, 0);
|
||||
} else {
|
||||
trace!(ino, "inode not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn read(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
offset: i64,
|
||||
size: u32,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
reply: ReplyData,
|
||||
) {
|
||||
let file_id = {
|
||||
let tree = self.tree.read();
|
||||
if let Some(VirtualNode::File(file)) = tree.get(ino) {
|
||||
trace!(ino, "file found in tree");
|
||||
file.file_id
|
||||
} else {
|
||||
trace!(ino, "file not found");
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let handle = self.runtime_handle.clone();
|
||||
|
||||
if let Some(ref overlay) = self.overlay_reader {
|
||||
let overlay = overlay.clone();
|
||||
let result = std::thread::scope(|_| {
|
||||
handle.block_on(async {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
overlay.read(file_id, offset as u64, size),
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(Ok(data)) => {
|
||||
trace!(
|
||||
ino,
|
||||
offset,
|
||||
size_bytes = size,
|
||||
bytes_read = data.len(),
|
||||
"overlay read successful"
|
||||
);
|
||||
reply.data(&data);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
let errno = match &e {
|
||||
OverlayError::NotFound(_) => libc::ENOENT,
|
||||
OverlayError::Database(_) => libc::EIO,
|
||||
OverlayError::Handler(_) => libc::EIO,
|
||||
OverlayError::Cas(_) => libc::EIO,
|
||||
OverlayError::NoHandler(_) => libc::EIO,
|
||||
};
|
||||
warn!(ino, offset, size_bytes = size, error = %e, "overlay read failed");
|
||||
reply.error(errno);
|
||||
}
|
||||
Err(_timeout) => {
|
||||
warn!(
|
||||
ino,
|
||||
offset,
|
||||
size_bytes = size,
|
||||
"overlay read timed out after 30s"
|
||||
);
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let Some(reader) = &self.reader else {
|
||||
trace!(ino, "no reader available");
|
||||
reply.data(&[]);
|
||||
return;
|
||||
};
|
||||
|
||||
let reader = reader.clone();
|
||||
let result = std::thread::scope(|_| {
|
||||
handle.block_on(async {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
reader.read(file_id, offset as u64, size),
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(Ok(data)) => {
|
||||
trace!(
|
||||
ino,
|
||||
offset,
|
||||
size_bytes = size,
|
||||
bytes_read = data.len(),
|
||||
"read successful"
|
||||
);
|
||||
reply.data(&data);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!(ino, offset, size_bytes = size, error = %e, "read failed");
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
Err(_timeout) => {
|
||||
warn!(ino, offset, size_bytes = size, "read timed out after 30s");
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn release(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
_flush: bool,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
trace!(ino, "releasing file handle");
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn readlink(&mut self, _req: &Request, ino: u64, reply: ReplyData) {
|
||||
debug!("readlink(ino={})", ino);
|
||||
|
||||
if SearchOps::is_search_inode(ino) {
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readlink(ino, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
reply.error(libc::EINVAL);
|
||||
}
|
||||
|
||||
fn write(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_fh: u64,
|
||||
_offset: i64,
|
||||
_data: &[u8],
|
||||
_write_flags: u32,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
reply: fuser::ReplyWrite,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn mkdir(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
parent: u64,
|
||||
name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
let path = match self.resolve_path(parent, name) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut tree = self.tree.write();
|
||||
match tree.mkdir(&path) {
|
||||
Ok(inode) => {
|
||||
if let Some(ref db) = self.db {
|
||||
if let Err(e) = db.insert_directory(&path) {
|
||||
warn!(error = %e, "failed to persist directory to database");
|
||||
}
|
||||
}
|
||||
let attr = FileAttr {
|
||||
ino: inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: SystemTime::now(),
|
||||
mtime: SystemTime::now(),
|
||||
ctime: SystemTime::now(),
|
||||
crtime: SystemTime::now(),
|
||||
kind: FileType::Directory,
|
||||
perm: 0o755,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: BLOCK_SIZE,
|
||||
flags: 0,
|
||||
};
|
||||
debug!(path = %path.as_str(), inode, "mkdir successful");
|
||||
reply.entry(&TTL, &attr, 0);
|
||||
}
|
||||
Err(RenameError::TargetExists) => reply.error(libc::EEXIST),
|
||||
Err(RenameError::ParentNotFound) => reply.error(libc::ENOENT),
|
||||
Err(_) => reply.error(libc::EIO),
|
||||
}
|
||||
}
|
||||
|
||||
fn unlink(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: fuser::ReplyEmpty) {
|
||||
let path = match self.resolve_path(parent, name) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let (file_id, is_dir) = {
|
||||
let tree = self.tree.read();
|
||||
match tree.get_by_path(&path) {
|
||||
Some(VirtualNode::File(f)) => (Some(f.file_id), false),
|
||||
Some(VirtualNode::Directory(_)) => (None, true),
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if is_dir {
|
||||
reply.error(libc::EISDIR);
|
||||
return;
|
||||
}
|
||||
|
||||
let trash_path = VirtualPath::new(format!("/.trash{}", path.as_str()));
|
||||
|
||||
{
|
||||
let mut tree = self.tree.write();
|
||||
tree.ensure_trash_dir();
|
||||
|
||||
let trash_parent = std::path::Path::new(trash_path.as_str())
|
||||
.parent()
|
||||
.map(|p| VirtualPath::new(p.to_string_lossy().into_owned()))
|
||||
.unwrap_or_else(|| VirtualPath::new("/.trash"));
|
||||
|
||||
if let Err(e) = tree.mkdir_p(&trash_parent) {
|
||||
if !matches!(e, RenameError::TargetExists) {
|
||||
warn!(error = ?e, "failed to create trash parent directories");
|
||||
reply.error(libc::EIO);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = tree.rename_file(&path, &trash_path) {
|
||||
match e {
|
||||
RenameError::SourceNotFound => reply.error(libc::ENOENT),
|
||||
RenameError::TargetExists => reply.error(libc::EEXIST),
|
||||
_ => reply.error(libc::EIO),
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let (Some(ref db), Some(id)) = (&self.db, file_id) {
|
||||
if let Err(e) = db.update_virtual_path(id, &trash_path) {
|
||||
warn!(error = %e, "failed to update virtual path in database");
|
||||
}
|
||||
if let Err(e) = db.mark_trashed(id, &path) {
|
||||
warn!(error = %e, "failed to mark file as trashed in database");
|
||||
}
|
||||
}
|
||||
|
||||
debug!(path = %path.as_str(), trash = %trash_path.as_str(), "file moved to trash");
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn rmdir(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: fuser::ReplyEmpty) {
|
||||
let path = match self.resolve_path(parent, name) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if VirtualTree::is_trash_path(&path) {
|
||||
reply.error(libc::EPERM);
|
||||
return;
|
||||
}
|
||||
|
||||
{
|
||||
let mut tree = self.tree.write();
|
||||
match tree.remove_directory(&path) {
|
||||
Ok(()) => {}
|
||||
Err(RemoveError::NotFound) => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
Err(RemoveError::NotEmpty) => {
|
||||
reply.error(libc::ENOTEMPTY);
|
||||
return;
|
||||
}
|
||||
Err(RemoveError::NotDirectory) => {
|
||||
reply.error(libc::ENOTDIR);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref db) = self.db {
|
||||
if let Err(e) = db.delete_directory(&path) {
|
||||
warn!(error = %e, "failed to delete directory from database");
|
||||
}
|
||||
}
|
||||
|
||||
debug!(path = %path.as_str(), "directory removed");
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn rename(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
parent: u64,
|
||||
name: &OsStr,
|
||||
newparent: u64,
|
||||
newname: &OsStr,
|
||||
_flags: u32,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
let old_path = match self.resolve_path(parent, name) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let new_path = match self.resolve_path(newparent, newname) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if old_path.as_str() == new_path.as_str() {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
|
||||
let is_dir = {
|
||||
let tree = self.tree.read();
|
||||
tree.get_by_path(&old_path)
|
||||
.map(|n| n.is_dir())
|
||||
.unwrap_or(false)
|
||||
};
|
||||
|
||||
let result = if is_dir {
|
||||
let mut tree = self.tree.write();
|
||||
match tree.rename_directory(&old_path, &new_path) {
|
||||
Ok(count) => {
|
||||
if let Some(ref db) = self.db {
|
||||
let old_prefix = if old_path.as_str().ends_with('/') {
|
||||
old_path.as_str().to_string()
|
||||
} else {
|
||||
format!("{}/", old_path.as_str())
|
||||
};
|
||||
let new_prefix = if new_path.as_str().ends_with('/') {
|
||||
new_path.as_str().to_string()
|
||||
} else {
|
||||
format!("{}/", new_path.as_str())
|
||||
};
|
||||
if let Err(e) = db.rename_directory(&old_prefix, &new_prefix) {
|
||||
warn!(error = %e, "failed to persist file path rename to database");
|
||||
}
|
||||
if let Err(e) = db.rename_directories(&old_prefix, &new_prefix) {
|
||||
warn!(error = %e, "failed to persist directory rename to database");
|
||||
}
|
||||
}
|
||||
debug!(old = %old_path.as_str(), new = %new_path.as_str(), count, "directory renamed");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
} else {
|
||||
let file_id = {
|
||||
let tree = self.tree.read();
|
||||
match tree.get_by_path(&old_path) {
|
||||
Some(VirtualNode::File(f)) => Some(f.file_id),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
let mut tree = self.tree.write();
|
||||
match tree.rename_file(&old_path, &new_path) {
|
||||
Ok(()) => {
|
||||
if let (Some(ref db), Some(id)) = (&self.db, file_id) {
|
||||
if let Err(e) = db.update_virtual_path(id, &new_path) {
|
||||
warn!(error = %e, "failed to persist file rename to database");
|
||||
}
|
||||
let was_in_trash = VirtualTree::is_trash_path(&old_path);
|
||||
let now_in_trash = VirtualTree::is_trash_path(&new_path);
|
||||
if was_in_trash && !now_in_trash {
|
||||
if let Err(e) = db.unmark_trashed(id) {
|
||||
warn!(error = %e, "failed to unmark trashed after restore");
|
||||
}
|
||||
debug!(path = %new_path.as_str(), "file restored from trash");
|
||||
}
|
||||
}
|
||||
debug!(old = %old_path.as_str(), new = %new_path.as_str(), "file renamed");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => reply.ok(),
|
||||
Err(RenameError::SourceNotFound) => reply.error(libc::ENOENT),
|
||||
Err(RenameError::TargetExists) => reply.error(libc::EEXIST),
|
||||
Err(RenameError::ParentNotFound) => reply.error(libc::ENOENT),
|
||||
Err(RenameError::IsDirectory) => reply.error(libc::EISDIR),
|
||||
Err(RenameError::NotDirectory) => reply.error(libc::ENOTDIR),
|
||||
}
|
||||
}
|
||||
|
||||
fn create(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
_flags: i32,
|
||||
reply: fuser::ReplyCreate,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn setattr(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_mode: Option<u32>,
|
||||
_uid: Option<u32>,
|
||||
_gid: Option<u32>,
|
||||
_size: Option<u64>,
|
||||
_atime: Option<fuser::TimeOrNow>,
|
||||
_mtime: Option<fuser::TimeOrNow>,
|
||||
_ctime: Option<SystemTime>,
|
||||
_fh: Option<u64>,
|
||||
_crtime: Option<SystemTime>,
|
||||
_chgtime: Option<SystemTime>,
|
||||
_bkuptime: Option<SystemTime>,
|
||||
_flags: Option<u32>,
|
||||
reply: ReplyAttr,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn symlink(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_link: &Path,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn link(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_newparent: u64,
|
||||
_newname: &OsStr,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn mknod(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
_rdev: u32,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from("/test"),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tree_integration() {
|
||||
let runtime = tokio::runtime::Runtime::new().unwrap();
|
||||
let handle = runtime.handle().clone();
|
||||
|
||||
let mut builder = TreeBuilder::new();
|
||||
builder.add_file(&make_file_meta(1, "/Artist/Album/Track.flac", 30_000_000));
|
||||
let tree = Arc::new(RwLock::new(builder.build()));
|
||||
|
||||
let _fs = MusicFs::new(tree.clone(), handle);
|
||||
|
||||
let tree_read = tree.read();
|
||||
assert!(tree_read.get(ROOT_INODE).is_some());
|
||||
assert!(tree_read
|
||||
.get_by_path(&VirtualPath::new("/Artist"))
|
||||
.is_some());
|
||||
}
|
||||
}
|
||||
+13
-8
@@ -43,10 +43,7 @@ impl PrefetchOps {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_engine(
|
||||
&self,
|
||||
event_bus: Arc<EventBus>,
|
||||
) -> Option<musicfs_cache::PrefetchHandle> {
|
||||
pub fn start_engine(&self, event_bus: Arc<EventBus>) -> Option<musicfs_cache::PrefetchHandle> {
|
||||
self.engine
|
||||
.as_ref()
|
||||
.map(|e| e.clone().start(event_bus, self.pattern_store.clone()))
|
||||
@@ -266,7 +263,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_prefetch_ops_new() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let pattern_store = Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
|
||||
let pattern_store =
|
||||
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
|
||||
let _ops = PrefetchOps::new(pattern_store, 1000, 1000);
|
||||
}
|
||||
|
||||
@@ -283,11 +281,18 @@ mod tests {
|
||||
#[test]
|
||||
fn test_hint_name_to_inode() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let pattern_store = Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
|
||||
let pattern_store =
|
||||
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
|
||||
let ops = PrefetchOps::new(pattern_store, 1000, 1000);
|
||||
|
||||
assert_eq!(ops.hint_name_to_inode("hint_0001"), Some(PREFETCH_HINTS_BASE + 1));
|
||||
assert_eq!(ops.hint_name_to_inode("hint_9999"), Some(PREFETCH_HINTS_BASE + 9999));
|
||||
assert_eq!(
|
||||
ops.hint_name_to_inode("hint_0001"),
|
||||
Some(PREFETCH_HINTS_BASE + 1)
|
||||
);
|
||||
assert_eq!(
|
||||
ops.hint_name_to_inode("hint_9999"),
|
||||
Some(PREFETCH_HINTS_BASE + 9999)
|
||||
);
|
||||
assert_eq!(ops.hint_name_to_inode("invalid"), None);
|
||||
}
|
||||
}
|
||||
+8
-5
@@ -160,16 +160,17 @@ impl SearchOps {
|
||||
}
|
||||
|
||||
fn safe_symlink_target(&self, virtual_path: &str) -> Option<String> {
|
||||
let normalized = Path::new(virtual_path)
|
||||
.components()
|
||||
.fold(std::path::PathBuf::new(), |mut acc, comp| {
|
||||
let normalized = Path::new(virtual_path).components().fold(
|
||||
std::path::PathBuf::new(),
|
||||
|mut acc, comp| {
|
||||
match comp {
|
||||
std::path::Component::Normal(s) => acc.push(s),
|
||||
std::path::Component::RootDir => acc.push("/"),
|
||||
_ => {}
|
||||
}
|
||||
acc
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
let path_str = normalized.to_string_lossy();
|
||||
if path_str.contains("..") {
|
||||
@@ -198,7 +199,9 @@ impl SearchOps {
|
||||
fn result_filename(&self, hit: &SearchHit, index: usize) -> String {
|
||||
let artist = hit.artist.as_deref().unwrap_or("Unknown");
|
||||
let title = hit.title.as_deref().unwrap_or("Unknown");
|
||||
let ext = hit.virtual_path.as_str()
|
||||
let ext = hit
|
||||
.virtual_path
|
||||
.as_str()
|
||||
.rsplit('.')
|
||||
.next()
|
||||
.unwrap_or("flac");
|
||||
@@ -4,8 +4,12 @@ version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-cache = { path = "../musicfs-cache" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-metadata = { path = "../musicfs-metadata" }
|
||||
musicfs-search = { path = "../musicfs-search" }
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
parking_lot.workspace = true
|
||||
tonic.workspace = true
|
||||
prost.workspace = true
|
||||
tokio.workspace = true
|
||||
@@ -15,6 +19,7 @@ thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
chrono.workspace = true
|
||||
csv = "1.3"
|
||||
reqwest = { version = "0.11", features = ["json"] }
|
||||
hmac = "0.12"
|
||||
sha2 = "0.10"
|
||||
@@ -0,0 +1,322 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package musicfs.v1;
|
||||
|
||||
option go_package = "homelab.lan/music-agregator/gen/musicfs/v1;musicfsv1";
|
||||
|
||||
service MusicFS {
|
||||
rpc Search(SearchRequest) returns (SearchResponse);
|
||||
rpc SearchStream(SearchRequest) returns (stream SearchResult);
|
||||
rpc GetStatus(Empty) returns (StatusResponse);
|
||||
rpc Shutdown(ShutdownRequest) returns (Empty);
|
||||
rpc GetCacheStats(Empty) returns (CacheStats);
|
||||
rpc ClearCache(ClearCacheRequest) returns (ClearCacheResponse);
|
||||
rpc Prefetch(PrefetchRequest) returns (stream PrefetchProgress);
|
||||
rpc ListOrigins(Empty) returns (OriginsResponse);
|
||||
rpc GetOriginHealth(OriginRequest) returns (OriginHealthResponse);
|
||||
rpc RescanOrigin(OriginRequest) returns (stream SyncProgress);
|
||||
rpc SubscribeEvents(EventFilter) returns (stream Event);
|
||||
}
|
||||
|
||||
service MetadataService {
|
||||
rpc GetMetadata(GetMetadataRequest) returns (MetadataResponse);
|
||||
rpc UpdateMetadata(UpdateMetadataRequest) returns (UpdateMetadataResponse);
|
||||
rpc ClearOverlay(ClearOverlayRequest) returns (ClearOverlayResponse);
|
||||
rpc BatchUpdateMetadata(BatchUpdateRequest) returns (stream BatchUpdateProgress);
|
||||
rpc ImportMetadata(ImportMetadataRequest) returns (stream ImportProgress);
|
||||
}
|
||||
|
||||
message Empty {}
|
||||
|
||||
message SearchRequest {
|
||||
string query = 1;
|
||||
optional uint32 limit = 2;
|
||||
optional uint32 offset = 3;
|
||||
optional string origin_id = 4;
|
||||
}
|
||||
|
||||
message SearchResponse {
|
||||
repeated SearchResult results = 1;
|
||||
uint64 total_matches = 2;
|
||||
uint32 query_time_ms = 3;
|
||||
}
|
||||
|
||||
message SearchResult {
|
||||
int64 file_id = 1;
|
||||
string virtual_path = 2;
|
||||
optional string artist = 3;
|
||||
optional string album = 4;
|
||||
optional string title = 5;
|
||||
float score = 6;
|
||||
map<string, string> highlights = 7;
|
||||
}
|
||||
|
||||
enum MountState {
|
||||
MOUNT_UNKNOWN = 0;
|
||||
MOUNT_MOUNTING = 1;
|
||||
MOUNT_READY = 2;
|
||||
MOUNT_SYNCING = 3;
|
||||
MOUNT_DEGRADED = 4;
|
||||
MOUNT_UNMOUNTING = 5;
|
||||
}
|
||||
|
||||
message StatusResponse {
|
||||
string version = 1;
|
||||
uint64 uptime_secs = 2;
|
||||
string mount_point = 3;
|
||||
MountState state = 4;
|
||||
uint32 open_file_handles = 5;
|
||||
uint64 fuse_ops_total = 6;
|
||||
uint64 files_indexed = 7;
|
||||
uint64 cache_size_bytes = 8;
|
||||
repeated OriginStatus origins = 9;
|
||||
}
|
||||
|
||||
message OriginStatus {
|
||||
string id = 1;
|
||||
string origin_type = 2;
|
||||
HealthStatus health = 3;
|
||||
uint64 files_count = 4;
|
||||
}
|
||||
|
||||
enum HealthStatus {
|
||||
HEALTH_UNKNOWN = 0;
|
||||
HEALTH_HEALTHY = 1;
|
||||
HEALTH_DEGRADED = 2;
|
||||
HEALTH_UNHEALTHY = 3;
|
||||
}
|
||||
|
||||
message ShutdownRequest {
|
||||
bool graceful = 1;
|
||||
uint32 timeout_secs = 2;
|
||||
}
|
||||
|
||||
message TierStats {
|
||||
uint64 entries = 1;
|
||||
uint64 size_bytes = 2;
|
||||
uint64 hits = 3;
|
||||
uint64 misses = 4;
|
||||
}
|
||||
|
||||
message CacheStats {
|
||||
uint64 total_size_bytes = 1;
|
||||
uint64 used_size_bytes = 2;
|
||||
uint64 size_limit_bytes = 3;
|
||||
uint64 chunk_count = 4;
|
||||
uint64 chunks_unique = 5;
|
||||
double dedup_ratio = 6;
|
||||
uint64 hit_count = 7;
|
||||
uint64 miss_count = 8;
|
||||
double hit_ratio = 9;
|
||||
uint64 metadata_entries = 10;
|
||||
uint64 metadata_bytes = 11;
|
||||
TierStats l1_metadata = 12;
|
||||
TierStats l2_headers = 13;
|
||||
TierStats l3_chunks = 14;
|
||||
}
|
||||
|
||||
message ClearCacheRequest {
|
||||
optional string origin_id = 1;
|
||||
bool clear_metadata = 2;
|
||||
bool clear_chunks = 3;
|
||||
}
|
||||
|
||||
message ClearCacheResponse {
|
||||
uint64 bytes_cleared = 1;
|
||||
uint64 chunks_cleared = 2;
|
||||
}
|
||||
|
||||
message PrefetchRequest {
|
||||
repeated string paths = 1;
|
||||
optional string origin_id = 2;
|
||||
}
|
||||
|
||||
message PrefetchProgress {
|
||||
string current_path = 1;
|
||||
uint32 completed = 2;
|
||||
uint32 total = 3;
|
||||
uint64 bytes_fetched = 4;
|
||||
}
|
||||
|
||||
message OriginsResponse {
|
||||
repeated OriginInfo origins = 1;
|
||||
}
|
||||
|
||||
message OriginInfo {
|
||||
string id = 1;
|
||||
string origin_type = 2;
|
||||
string display_name = 3;
|
||||
string root_path = 4;
|
||||
HealthStatus health = 5;
|
||||
uint64 files_count = 6;
|
||||
uint64 total_size_bytes = 7;
|
||||
}
|
||||
|
||||
message OriginRequest {
|
||||
string origin_id = 1;
|
||||
// Optional subdirectory to scope the scan (relative to origin root).
|
||||
// If empty, scans the entire origin.
|
||||
// Example: "Metallica - Master of Puppets (1986) [FLAC]"
|
||||
optional string subdir = 2;
|
||||
}
|
||||
|
||||
message OriginHealthResponse {
|
||||
string origin_id = 1;
|
||||
HealthStatus status = 2;
|
||||
optional string message = 3;
|
||||
uint64 last_check_secs = 4;
|
||||
}
|
||||
|
||||
message SyncProgress {
|
||||
string phase = 1;
|
||||
uint32 current = 2;
|
||||
uint32 total = 3;
|
||||
string current_path = 4;
|
||||
uint64 bytes_synced = 5;
|
||||
repeated SyncedFile new_files = 6;
|
||||
}
|
||||
|
||||
message SyncedFile {
|
||||
string path = 1;
|
||||
int64 file_id = 2;
|
||||
string virtual_path = 3;
|
||||
}
|
||||
|
||||
message EventFilter {
|
||||
repeated string event_types = 1;
|
||||
optional string origin_id = 2;
|
||||
}
|
||||
|
||||
message Event {
|
||||
string event_type = 1;
|
||||
int64 timestamp_ms = 2;
|
||||
optional string origin_id = 3;
|
||||
optional string path = 4;
|
||||
optional int64 file_id = 5;
|
||||
map<string, string> metadata = 6;
|
||||
}
|
||||
|
||||
// MetadataService messages
|
||||
|
||||
message GetMetadataRequest {
|
||||
string virtual_path = 1;
|
||||
}
|
||||
|
||||
message MetadataResponse {
|
||||
int64 file_id = 1;
|
||||
optional string title = 2;
|
||||
optional string artist = 3;
|
||||
optional string album = 4;
|
||||
optional string album_artist = 5;
|
||||
optional uint32 year = 6;
|
||||
optional uint32 track = 7;
|
||||
optional uint32 disc = 8;
|
||||
optional string genre = 9;
|
||||
optional string format = 10;
|
||||
optional uint64 duration_ms = 11;
|
||||
optional uint64 bitrate = 12;
|
||||
optional uint32 track_total = 13;
|
||||
optional uint32 disc_total = 14;
|
||||
optional string date = 15;
|
||||
optional string composer = 16;
|
||||
optional string comment = 17;
|
||||
optional string lyrics = 18;
|
||||
optional string copyright = 19;
|
||||
optional bool compilation = 20;
|
||||
optional string artist_sort = 21;
|
||||
optional string album_artist_sort = 22;
|
||||
optional string album_sort = 23;
|
||||
optional string title_sort = 24;
|
||||
optional string mb_recording_id = 25;
|
||||
optional string mb_album_id = 26;
|
||||
optional string mb_artist_id = 27;
|
||||
optional string mb_album_artist_id = 28;
|
||||
optional string mb_release_group_id = 29;
|
||||
optional float replaygain_track_gain = 30;
|
||||
optional float replaygain_track_peak = 31;
|
||||
optional float replaygain_album_gain = 32;
|
||||
optional float replaygain_album_peak = 33;
|
||||
optional uint32 channels = 34;
|
||||
optional uint32 bits_per_sample = 35;
|
||||
optional string encoder = 36;
|
||||
optional string label = 40;
|
||||
optional string album_type = 41;
|
||||
optional string cover_url = 42;
|
||||
map<string, string> custom_tags = 50;
|
||||
}
|
||||
|
||||
message UpdateMetadataRequest {
|
||||
int64 file_id = 1;
|
||||
optional string title = 2;
|
||||
optional string artist = 3;
|
||||
optional string album = 4;
|
||||
optional string album_artist = 5;
|
||||
optional uint32 track_number = 6;
|
||||
optional uint32 disc_number = 7;
|
||||
optional string date = 8;
|
||||
optional string genre = 9;
|
||||
optional string composer = 10;
|
||||
optional string comment = 11;
|
||||
optional string lyrics = 12;
|
||||
optional string copyright = 13;
|
||||
optional bool compilation = 14;
|
||||
optional string artist_sort = 15;
|
||||
optional string album_artist_sort = 16;
|
||||
optional string album_sort = 17;
|
||||
optional string title_sort = 18;
|
||||
optional string mb_recording_id = 20;
|
||||
optional string mb_album_id = 21;
|
||||
optional string mb_artist_id = 22;
|
||||
optional float replaygain_track_gain = 30;
|
||||
optional float replaygain_track_peak = 31;
|
||||
optional float replaygain_album_gain = 32;
|
||||
optional float replaygain_album_peak = 33;
|
||||
optional string label = 40;
|
||||
optional string album_type = 41;
|
||||
optional string cover_url = 42;
|
||||
map<string, string> custom_tags = 50;
|
||||
}
|
||||
|
||||
message UpdateMetadataResponse {
|
||||
int64 file_id = 1;
|
||||
bool success = 2;
|
||||
optional string error_message = 3;
|
||||
}
|
||||
|
||||
message ClearOverlayRequest {
|
||||
int64 file_id = 1;
|
||||
}
|
||||
|
||||
message ClearOverlayResponse {
|
||||
int64 file_id = 1;
|
||||
bool success = 2;
|
||||
optional string error_message = 3;
|
||||
}
|
||||
|
||||
message BatchUpdateRequest {
|
||||
repeated BatchUpdateItem items = 1;
|
||||
}
|
||||
|
||||
message BatchUpdateItem {
|
||||
int64 file_id = 1;
|
||||
UpdateMetadataRequest metadata = 2;
|
||||
}
|
||||
|
||||
message BatchUpdateProgress {
|
||||
uint32 completed = 1;
|
||||
uint32 total = 2;
|
||||
optional int64 current_file_id = 3;
|
||||
optional string error_message = 4;
|
||||
}
|
||||
|
||||
message ImportMetadataRequest {
|
||||
string source_path = 1;
|
||||
optional string format = 2;
|
||||
}
|
||||
|
||||
message ImportProgress {
|
||||
uint32 imported = 1;
|
||||
uint32 total = 2;
|
||||
optional string current_file = 3;
|
||||
optional string error_message = 4;
|
||||
}
|
||||
@@ -6,10 +6,14 @@ pub mod proto {
|
||||
}
|
||||
}
|
||||
|
||||
mod metadata;
|
||||
pub mod scanner;
|
||||
mod search_service;
|
||||
mod server;
|
||||
mod webhook;
|
||||
|
||||
pub use metadata::MetadataServiceImpl;
|
||||
pub use proto::musicfs::v1::metadata_service_server::MetadataServiceServer;
|
||||
pub use proto::musicfs::v1::music_fs_server::{MusicFs, MusicFsServer as MusicFsGrpcServer};
|
||||
pub use proto::musicfs::v1::*;
|
||||
pub use search_service::SearchService;
|
||||
@@ -0,0 +1,794 @@
|
||||
//! MetadataService gRPC handlers for metadata overlay operations.
|
||||
|
||||
use crate::proto::musicfs::v1::{
|
||||
metadata_service_server::MetadataService, BatchUpdateProgress, BatchUpdateRequest,
|
||||
ClearOverlayRequest, ClearOverlayResponse, GetMetadataRequest, ImportMetadataRequest,
|
||||
ImportProgress, MetadataResponse, UpdateMetadataRequest, UpdateMetadataResponse,
|
||||
};
|
||||
use musicfs_cache::{Database, EnrichmentUpdate};
|
||||
use musicfs_core::{AudioMeta, FileId, VirtualPath};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::{debug, info, instrument, warn};
|
||||
|
||||
/// gRPC service implementation for metadata operations.
|
||||
pub struct MetadataServiceImpl {
|
||||
db: Arc<Database>,
|
||||
}
|
||||
|
||||
impl MetadataServiceImpl {
|
||||
/// Create a new MetadataServiceImpl with the given database.
|
||||
pub fn new(db: Arc<Database>) -> Self {
|
||||
Self { db }
|
||||
}
|
||||
|
||||
/// Convert AudioMeta to MetadataResponse proto message.
|
||||
fn audio_meta_to_response(file_id: FileId, meta: &AudioMeta) -> MetadataResponse {
|
||||
MetadataResponse {
|
||||
file_id: file_id.0,
|
||||
title: meta.title.clone(),
|
||||
artist: meta.artist.clone(),
|
||||
album: meta.album.clone(),
|
||||
album_artist: meta.album_artist.clone(),
|
||||
year: meta.year,
|
||||
track: meta.track,
|
||||
disc: meta.disc,
|
||||
genre: meta.genre.clone(),
|
||||
format: Some(format!("{:?}", meta.format)),
|
||||
duration_ms: meta.duration_ms,
|
||||
bitrate: meta.bitrate.map(|b| b as u64),
|
||||
track_total: meta.track_total,
|
||||
disc_total: meta.disc_total,
|
||||
date: meta.date.clone(),
|
||||
composer: meta.composer.clone(),
|
||||
comment: meta.comment.clone(),
|
||||
lyrics: meta.lyrics.clone(),
|
||||
copyright: meta.copyright.clone(),
|
||||
compilation: meta.compilation,
|
||||
artist_sort: meta.artist_sort.clone(),
|
||||
album_artist_sort: meta.album_artist_sort.clone(),
|
||||
album_sort: meta.album_sort.clone(),
|
||||
title_sort: meta.title_sort.clone(),
|
||||
mb_recording_id: meta.mb_recording_id.clone(),
|
||||
mb_album_id: meta.mb_album_id.clone(),
|
||||
mb_artist_id: meta.mb_artist_id.clone(),
|
||||
mb_album_artist_id: meta.mb_album_artist_id.clone(),
|
||||
mb_release_group_id: meta.mb_release_group_id.clone(),
|
||||
replaygain_track_gain: meta.replaygain_track_gain,
|
||||
replaygain_track_peak: meta.replaygain_track_peak,
|
||||
replaygain_album_gain: meta.replaygain_album_gain,
|
||||
replaygain_album_peak: meta.replaygain_album_peak,
|
||||
channels: meta.channels,
|
||||
bits_per_sample: meta.bits_per_sample,
|
||||
encoder: meta.encoder.clone(),
|
||||
label: None,
|
||||
album_type: None,
|
||||
cover_url: None,
|
||||
custom_tags: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert UpdateMetadataRequest to AudioMeta for database update.
|
||||
fn request_to_audio_meta(req: &UpdateMetadataRequest) -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: req.title.clone(),
|
||||
artist: req.artist.clone(),
|
||||
album: req.album.clone(),
|
||||
album_artist: req.album_artist.clone(),
|
||||
genre: req.genre.clone(),
|
||||
year: None,
|
||||
track: req.track_number,
|
||||
disc: req.disc_number,
|
||||
duration_ms: None,
|
||||
bitrate: None,
|
||||
sample_rate: None,
|
||||
format: musicfs_core::AudioFormat::Unknown,
|
||||
track_total: None,
|
||||
disc_total: None,
|
||||
date: req.date.clone(),
|
||||
composer: req.composer.clone(),
|
||||
comment: req.comment.clone(),
|
||||
lyrics: req.lyrics.clone(),
|
||||
copyright: req.copyright.clone(),
|
||||
compilation: req.compilation,
|
||||
artist_sort: req.artist_sort.clone(),
|
||||
album_artist_sort: req.album_artist_sort.clone(),
|
||||
album_sort: req.album_sort.clone(),
|
||||
title_sort: req.title_sort.clone(),
|
||||
mb_recording_id: req.mb_recording_id.clone(),
|
||||
mb_album_id: req.mb_album_id.clone(),
|
||||
mb_artist_id: req.mb_artist_id.clone(),
|
||||
mb_album_artist_id: None,
|
||||
mb_release_group_id: None,
|
||||
replaygain_track_gain: req.replaygain_track_gain,
|
||||
replaygain_track_peak: req.replaygain_track_peak,
|
||||
replaygain_album_gain: req.replaygain_album_gain,
|
||||
replaygain_album_peak: req.replaygain_album_peak,
|
||||
channels: None,
|
||||
bits_per_sample: None,
|
||||
encoder: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl MetadataService for MetadataServiceImpl {
|
||||
#[instrument(level = "debug", skip(self, request), fields(method = "get_metadata"))]
|
||||
async fn get_metadata(
|
||||
&self,
|
||||
request: Request<GetMetadataRequest>,
|
||||
) -> Result<Response<MetadataResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
debug!(virtual_path = %req.virtual_path, "GetMetadata request");
|
||||
|
||||
if req.virtual_path.is_empty() {
|
||||
return Err(Status::invalid_argument("virtual_path cannot be empty"));
|
||||
}
|
||||
|
||||
let vpath = VirtualPath::new(&req.virtual_path);
|
||||
|
||||
let file_meta = self
|
||||
.db
|
||||
.get_file_by_virtual_path(&vpath)
|
||||
.map_err(|e| Status::internal(format!("Database error: {}", e)))?
|
||||
.ok_or_else(|| Status::not_found(format!("File not found: {}", req.virtual_path)))?;
|
||||
|
||||
let audio_meta = self
|
||||
.db
|
||||
.get_file_metadata_row(file_meta.id)
|
||||
.map_err(|e| Status::internal(format!("Failed to get metadata: {}", e)))?;
|
||||
|
||||
let response = Self::audio_meta_to_response(file_meta.id, &audio_meta);
|
||||
Ok(Response::new(response))
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "update_metadata")
|
||||
)]
|
||||
async fn update_metadata(
|
||||
&self,
|
||||
request: Request<UpdateMetadataRequest>,
|
||||
) -> Result<Response<UpdateMetadataResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let file_id = FileId(req.file_id);
|
||||
info!(file_id = req.file_id, "UpdateMetadata request");
|
||||
|
||||
if req.file_id <= 0 {
|
||||
return Err(Status::invalid_argument("file_id must be positive"));
|
||||
}
|
||||
|
||||
let audio_meta = Self::request_to_audio_meta(&req);
|
||||
|
||||
if let Err(e) = self.db.update_metadata(file_id, &audio_meta) {
|
||||
warn!(file_id = req.file_id, error = %e, "Failed to update metadata");
|
||||
return Ok(Response::new(UpdateMetadataResponse {
|
||||
file_id: req.file_id,
|
||||
success: false,
|
||||
error_message: Some(e.to_string()),
|
||||
}));
|
||||
}
|
||||
|
||||
if req.label.is_some() || req.album_type.is_some() || req.cover_url.is_some() {
|
||||
let enrichment = EnrichmentUpdate {
|
||||
label: req.label.clone(),
|
||||
album_type: req.album_type.clone(),
|
||||
cover_url: req.cover_url.clone(),
|
||||
genres_json: None,
|
||||
primary_genre: None,
|
||||
source: "orchestrator".to_string(),
|
||||
};
|
||||
if let Err(e) = self.db.update_enrichment(file_id, &enrichment) {
|
||||
warn!(file_id = req.file_id, error = %e, "Failed to update enrichment");
|
||||
return Ok(Response::new(UpdateMetadataResponse {
|
||||
file_id: req.file_id,
|
||||
success: false,
|
||||
error_message: Some(e.to_string()),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
debug!(file_id = req.file_id, "Metadata updated successfully");
|
||||
Ok(Response::new(UpdateMetadataResponse {
|
||||
file_id: req.file_id,
|
||||
success: true,
|
||||
error_message: None,
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "clear_overlay"))]
|
||||
async fn clear_overlay(
|
||||
&self,
|
||||
request: Request<ClearOverlayRequest>,
|
||||
) -> Result<Response<ClearOverlayResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let file_id = FileId(req.file_id);
|
||||
info!(file_id = req.file_id, "ClearOverlay request");
|
||||
|
||||
if req.file_id <= 0 {
|
||||
return Err(Status::invalid_argument("file_id must be positive"));
|
||||
}
|
||||
|
||||
match self.db.clear_overlay(file_id) {
|
||||
Ok(()) => {
|
||||
debug!(file_id = req.file_id, "Overlay cleared successfully");
|
||||
Ok(Response::new(ClearOverlayResponse {
|
||||
file_id: req.file_id,
|
||||
success: true,
|
||||
error_message: None,
|
||||
}))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(file_id = req.file_id, error = %e, "Failed to clear overlay");
|
||||
Ok(Response::new(ClearOverlayResponse {
|
||||
file_id: req.file_id,
|
||||
success: false,
|
||||
error_message: Some(e.to_string()),
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type BatchUpdateMetadataStream = ReceiverStream<Result<BatchUpdateProgress, Status>>;
|
||||
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "batch_update_metadata")
|
||||
)]
|
||||
async fn batch_update_metadata(
|
||||
&self,
|
||||
request: Request<BatchUpdateRequest>,
|
||||
) -> Result<Response<Self::BatchUpdateMetadataStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
let total = req.items.len() as u32;
|
||||
info!(item_count = total, "BatchUpdateMetadata request");
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
let db = Arc::clone(&self.db);
|
||||
|
||||
tokio::spawn(async move {
|
||||
for (i, item) in req.items.into_iter().enumerate() {
|
||||
let file_id = FileId(item.file_id);
|
||||
let completed = (i + 1) as u32;
|
||||
|
||||
let error_message = if let Some(ref metadata_req) = item.metadata {
|
||||
let audio_meta = MetadataServiceImpl::request_to_audio_meta(metadata_req);
|
||||
match db.update_metadata(file_id, &audio_meta) {
|
||||
Ok(()) => {
|
||||
if metadata_req.label.is_some()
|
||||
|| metadata_req.album_type.is_some()
|
||||
|| metadata_req.cover_url.is_some()
|
||||
{
|
||||
let enrichment = EnrichmentUpdate {
|
||||
label: metadata_req.label.clone(),
|
||||
album_type: metadata_req.album_type.clone(),
|
||||
cover_url: metadata_req.cover_url.clone(),
|
||||
genres_json: None,
|
||||
primary_genre: None,
|
||||
source: "orchestrator".to_string(),
|
||||
};
|
||||
if let Err(e) = db.update_enrichment(file_id, &enrichment) {
|
||||
Some(e.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(e) => Some(e.to_string()),
|
||||
}
|
||||
} else {
|
||||
Some("Missing metadata in batch item".to_string())
|
||||
};
|
||||
|
||||
let progress = BatchUpdateProgress {
|
||||
completed,
|
||||
total,
|
||||
current_file_id: Some(item.file_id),
|
||||
error_message,
|
||||
};
|
||||
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
type ImportMetadataStream = ReceiverStream<Result<ImportProgress, Status>>;
|
||||
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "import_metadata")
|
||||
)]
|
||||
async fn import_metadata(
|
||||
&self,
|
||||
request: Request<ImportMetadataRequest>,
|
||||
) -> Result<Response<Self::ImportMetadataStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(source_path = %req.source_path, format = ?req.format, "ImportMetadata request");
|
||||
|
||||
if req.source_path.is_empty() {
|
||||
return Err(Status::invalid_argument("source_path cannot be empty"));
|
||||
}
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
let db = Arc::clone(&self.db);
|
||||
let source_path = req.source_path.clone();
|
||||
let format = req.format.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let file_format = format.as_deref().unwrap_or_else(|| {
|
||||
if source_path.ends_with(".csv") {
|
||||
"csv"
|
||||
} else if source_path.ends_with(".json") {
|
||||
"json"
|
||||
} else {
|
||||
"unknown"
|
||||
}
|
||||
});
|
||||
|
||||
let content = match tokio::fs::read_to_string(&source_path).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Ok(ImportProgress {
|
||||
imported: 0,
|
||||
total: 0,
|
||||
current_file: None,
|
||||
error_message: Some(format!("Failed to read file: {}", e)),
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let entries: Vec<ImportEntry> = match file_format {
|
||||
"json" => match serde_json::from_str::<Vec<ImportEntry>>(&content) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Ok(ImportProgress {
|
||||
imported: 0,
|
||||
total: 0,
|
||||
current_file: None,
|
||||
error_message: Some(format!("Failed to parse JSON: {}", e)),
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
"csv" => match parse_csv_entries(&content) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Ok(ImportProgress {
|
||||
imported: 0,
|
||||
total: 0,
|
||||
current_file: None,
|
||||
error_message: Some(format!("Failed to parse CSV: {}", e)),
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
let _ = tx
|
||||
.send(Ok(ImportProgress {
|
||||
imported: 0,
|
||||
total: 0,
|
||||
current_file: None,
|
||||
error_message: Some(format!("Unsupported format: {}", file_format)),
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let total = entries.len() as u32;
|
||||
let mut imported = 0u32;
|
||||
|
||||
for entry in entries {
|
||||
let vpath = VirtualPath::new(&entry.virtual_path);
|
||||
|
||||
let file_meta = match db.get_file_by_virtual_path(&vpath) {
|
||||
Ok(Some(f)) => f,
|
||||
Ok(None) => {
|
||||
let progress = ImportProgress {
|
||||
imported,
|
||||
total,
|
||||
current_file: Some(entry.virtual_path.clone()),
|
||||
error_message: Some(format!("File not found: {}", entry.virtual_path)),
|
||||
};
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
let progress = ImportProgress {
|
||||
imported,
|
||||
total,
|
||||
current_file: Some(entry.virtual_path.clone()),
|
||||
error_message: Some(format!("Database error: {}", e)),
|
||||
};
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let audio_meta = entry.to_audio_meta();
|
||||
let error_message = match db.update_metadata(file_meta.id, &audio_meta) {
|
||||
Ok(()) => {
|
||||
imported += 1;
|
||||
None
|
||||
}
|
||||
Err(e) => Some(e.to_string()),
|
||||
};
|
||||
|
||||
let progress = ImportProgress {
|
||||
imported,
|
||||
total,
|
||||
current_file: Some(entry.virtual_path),
|
||||
error_message,
|
||||
};
|
||||
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Entry from import file (CSV or JSON).
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
struct ImportEntry {
|
||||
virtual_path: String,
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
artist: Option<String>,
|
||||
#[serde(default)]
|
||||
album: Option<String>,
|
||||
#[serde(default)]
|
||||
album_artist: Option<String>,
|
||||
#[serde(default)]
|
||||
genre: Option<String>,
|
||||
#[serde(default)]
|
||||
year: Option<u32>,
|
||||
#[serde(default)]
|
||||
track: Option<u32>,
|
||||
#[serde(default)]
|
||||
disc: Option<u32>,
|
||||
#[serde(default)]
|
||||
date: Option<String>,
|
||||
#[serde(default)]
|
||||
composer: Option<String>,
|
||||
#[serde(default)]
|
||||
comment: Option<String>,
|
||||
}
|
||||
|
||||
impl ImportEntry {
|
||||
fn to_audio_meta(&self) -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: self.title.clone(),
|
||||
artist: self.artist.clone(),
|
||||
album: self.album.clone(),
|
||||
album_artist: self.album_artist.clone(),
|
||||
genre: self.genre.clone(),
|
||||
year: self.year,
|
||||
track: self.track,
|
||||
disc: self.disc,
|
||||
duration_ms: None,
|
||||
bitrate: None,
|
||||
sample_rate: None,
|
||||
format: musicfs_core::AudioFormat::Unknown,
|
||||
track_total: None,
|
||||
disc_total: None,
|
||||
date: self.date.clone(),
|
||||
composer: self.composer.clone(),
|
||||
comment: self.comment.clone(),
|
||||
lyrics: None,
|
||||
copyright: None,
|
||||
compilation: None,
|
||||
artist_sort: None,
|
||||
album_artist_sort: None,
|
||||
album_sort: None,
|
||||
title_sort: None,
|
||||
mb_recording_id: None,
|
||||
mb_album_id: None,
|
||||
mb_artist_id: None,
|
||||
mb_album_artist_id: None,
|
||||
mb_release_group_id: None,
|
||||
replaygain_track_gain: None,
|
||||
replaygain_track_peak: None,
|
||||
replaygain_album_gain: None,
|
||||
replaygain_album_peak: None,
|
||||
channels: None,
|
||||
bits_per_sample: None,
|
||||
encoder: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse CSV content into ImportEntry list.
|
||||
fn parse_csv_entries(content: &str) -> Result<Vec<ImportEntry>, String> {
|
||||
let mut reader = csv::Reader::from_reader(content.as_bytes());
|
||||
let mut entries = Vec::new();
|
||||
|
||||
for result in reader.deserialize() {
|
||||
let entry: ImportEntry = result.map_err(|e| format!("CSV parse error: {}", e))?;
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::proto::musicfs::v1::BatchUpdateItem;
|
||||
use musicfs_core::{AudioFormat, OriginId};
|
||||
use std::path::Path;
|
||||
use std::time::UNIX_EPOCH;
|
||||
use tempfile::TempDir;
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
fn create_test_db() -> (TempDir, Arc<Database>) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
(dir, db)
|
||||
}
|
||||
|
||||
fn insert_test_file(db: &Database, vpath: &str) -> FileId {
|
||||
let real_path = format!("/music{}", vpath);
|
||||
db.upsert_file(
|
||||
&OriginId::from("local"),
|
||||
Path::new(&real_path),
|
||||
&VirtualPath::new(vpath),
|
||||
&AudioMeta {
|
||||
title: Some("Test Track".to_string()),
|
||||
artist: Some("Test Artist".to_string()),
|
||||
album: Some("Test Album".to_string()),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
},
|
||||
UNIX_EPOCH,
|
||||
1000,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_metadata_success() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let vpath = "/Artist/Album/Track.flac";
|
||||
insert_test_file(&db, vpath);
|
||||
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
let request = Request::new(GetMetadataRequest {
|
||||
virtual_path: vpath.to_string(),
|
||||
});
|
||||
|
||||
let response = service.get_metadata(request).await.unwrap();
|
||||
let meta = response.into_inner();
|
||||
|
||||
assert_eq!(meta.title, Some("Test Track".to_string()));
|
||||
assert_eq!(meta.artist, Some("Test Artist".to_string()));
|
||||
assert_eq!(meta.album, Some("Test Album".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_metadata_not_found() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(GetMetadataRequest {
|
||||
virtual_path: "/nonexistent.flac".to_string(),
|
||||
});
|
||||
|
||||
let result = service.get_metadata(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_metadata_empty_path() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(GetMetadataRequest {
|
||||
virtual_path: String::new(),
|
||||
});
|
||||
|
||||
let result = service.get_metadata(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_update_metadata_success() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let vpath = "/Artist/Album/Track.flac";
|
||||
let file_id = insert_test_file(&db, vpath);
|
||||
|
||||
let service = MetadataServiceImpl::new(db.clone());
|
||||
let request = Request::new(UpdateMetadataRequest {
|
||||
file_id: file_id.0,
|
||||
title: Some("Updated Title".to_string()),
|
||||
artist: Some("Updated Artist".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let response = service.update_metadata(request).await.unwrap();
|
||||
let result = response.into_inner();
|
||||
|
||||
assert!(result.success);
|
||||
assert!(result.error_message.is_none());
|
||||
|
||||
let meta = db.get_file_metadata_row(file_id).unwrap();
|
||||
assert_eq!(meta.title, Some("Updated Title".to_string()));
|
||||
assert_eq!(meta.artist, Some("Updated Artist".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_update_metadata_invalid_id() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(UpdateMetadataRequest {
|
||||
file_id: 0,
|
||||
title: Some("Title".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let result = service.update_metadata(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clear_overlay_success() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let vpath = "/Artist/Album/Track.flac";
|
||||
let file_id = insert_test_file(&db, vpath);
|
||||
|
||||
let service = MetadataServiceImpl::new(db.clone());
|
||||
let request = Request::new(ClearOverlayRequest { file_id: file_id.0 });
|
||||
|
||||
let response = service.clear_overlay(request).await.unwrap();
|
||||
let result = response.into_inner();
|
||||
|
||||
assert!(result.success);
|
||||
assert!(result.error_message.is_none());
|
||||
|
||||
let meta = db.get_file_metadata_row(file_id).unwrap();
|
||||
assert!(meta.title.is_none());
|
||||
assert!(meta.artist.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clear_overlay_invalid_id() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(ClearOverlayRequest { file_id: -1 });
|
||||
|
||||
let result = service.clear_overlay(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_batch_update_metadata() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let file_id1 = insert_test_file(&db, "/Track1.flac");
|
||||
let file_id2 = insert_test_file(&db, "/Track2.flac");
|
||||
|
||||
let service = MetadataServiceImpl::new(db.clone());
|
||||
let request = Request::new(BatchUpdateRequest {
|
||||
items: vec![
|
||||
BatchUpdateItem {
|
||||
file_id: file_id1.0,
|
||||
metadata: Some(UpdateMetadataRequest {
|
||||
file_id: file_id1.0,
|
||||
title: Some("Batch Title 1".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
BatchUpdateItem {
|
||||
file_id: file_id2.0,
|
||||
metadata: Some(UpdateMetadataRequest {
|
||||
file_id: file_id2.0,
|
||||
title: Some("Batch Title 2".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
let response = service.batch_update_metadata(request).await.unwrap();
|
||||
let mut stream = response.into_inner();
|
||||
|
||||
let mut progress_count = 0;
|
||||
while let Some(Ok(result)) = stream.next().await {
|
||||
progress_count += 1;
|
||||
assert!(result.error_message.is_none());
|
||||
}
|
||||
|
||||
assert_eq!(progress_count, 2);
|
||||
|
||||
let meta1 = db.get_file_metadata_row(file_id1).unwrap();
|
||||
assert_eq!(meta1.title, Some("Batch Title 1".to_string()));
|
||||
|
||||
let meta2 = db.get_file_metadata_row(file_id2).unwrap();
|
||||
assert_eq!(meta2.title, Some("Batch Title 2".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_import_metadata_empty_path() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(ImportMetadataRequest {
|
||||
source_path: String::new(),
|
||||
format: None,
|
||||
});
|
||||
|
||||
let result = service.import_metadata(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_csv_entries() {
|
||||
let csv_content = r#"virtual_path,title,artist,album
|
||||
/Track1.flac,Title 1,Artist 1,Album 1
|
||||
/Track2.flac,Title 2,Artist 2,Album 2"#;
|
||||
|
||||
let entries = parse_csv_entries(csv_content).unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(entries[0].virtual_path, "/Track1.flac");
|
||||
assert_eq!(entries[0].title, Some("Title 1".to_string()));
|
||||
assert_eq!(entries[1].virtual_path, "/Track2.flac");
|
||||
assert_eq!(entries[1].artist, Some("Artist 2".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_import_entry_to_audio_meta() {
|
||||
let entry = ImportEntry {
|
||||
virtual_path: "/test.flac".to_string(),
|
||||
title: Some("Test".to_string()),
|
||||
artist: Some("Artist".to_string()),
|
||||
album: None,
|
||||
album_artist: None,
|
||||
genre: Some("Rock".to_string()),
|
||||
year: Some(2024),
|
||||
track: Some(1),
|
||||
disc: None,
|
||||
date: None,
|
||||
composer: None,
|
||||
comment: None,
|
||||
};
|
||||
|
||||
let meta = entry.to_audio_meta();
|
||||
assert_eq!(meta.title, Some("Test".to_string()));
|
||||
assert_eq!(meta.artist, Some("Artist".to_string()));
|
||||
assert_eq!(meta.genre, Some("Rock".to_string()));
|
||||
assert_eq!(meta.year, Some(2024));
|
||||
assert_eq!(meta.track, Some(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
use musicfs_cache::{Database, VirtualTree};
|
||||
use musicfs_cas::ContentFetcher;
|
||||
use musicfs_core::{
|
||||
AudioMeta, Error, Event, EventBus, FileId, FileMeta, OriginId, RealPath, Result, VirtualPath,
|
||||
};
|
||||
use musicfs_metadata::MetadataParser;
|
||||
use parking_lot::RwLock;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::UNIX_EPOCH;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub struct ScanResult {
|
||||
pub new_files: Vec<SyncedFileInfo>,
|
||||
pub changed: u32,
|
||||
pub deleted: u32,
|
||||
pub unchanged: u32,
|
||||
pub bytes_synced: u64,
|
||||
}
|
||||
|
||||
pub struct SyncedFileInfo {
|
||||
pub path: String,
|
||||
pub file_id: FileId,
|
||||
pub virtual_path: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScanProgress {
|
||||
pub phase: String,
|
||||
pub current: u32,
|
||||
pub total: u32,
|
||||
pub current_path: String,
|
||||
pub bytes_synced: u64,
|
||||
}
|
||||
|
||||
pub struct OriginScanner {
|
||||
db: Arc<Database>,
|
||||
event_bus: Arc<EventBus>,
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
parser: MetadataParser,
|
||||
}
|
||||
|
||||
impl OriginScanner {
|
||||
pub fn new(
|
||||
db: Arc<Database>,
|
||||
event_bus: Arc<EventBus>,
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
) -> Self {
|
||||
Self {
|
||||
db,
|
||||
event_bus,
|
||||
tree,
|
||||
fetcher,
|
||||
parser: MetadataParser,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn scan(
|
||||
&self,
|
||||
origin_id: &OriginId,
|
||||
origin_root: &Path,
|
||||
subdir: Option<&str>,
|
||||
progress_tx: mpsc::Sender<ScanProgress>,
|
||||
) -> Result<ScanResult> {
|
||||
let scan_root = match subdir {
|
||||
Some(sub) if !sub.is_empty() => origin_root.join(sub),
|
||||
_ => origin_root.to_path_buf(),
|
||||
};
|
||||
|
||||
if !scan_root.exists() {
|
||||
return Err(Error::Origin(format!(
|
||||
"scan path does not exist: {}",
|
||||
scan_root.display()
|
||||
)));
|
||||
}
|
||||
|
||||
// Phase 1: Scanning
|
||||
let audio_files = self.collect_audio_files(&scan_root, &progress_tx)?;
|
||||
let total_files = audio_files.len() as u32;
|
||||
info!(files = total_files, "scan phase complete");
|
||||
|
||||
// Phase 2: Hashing + categorization
|
||||
let mut new_files = Vec::new();
|
||||
let mut unchanged = 0u32;
|
||||
|
||||
for (i, abs_path) in audio_files.iter().enumerate() {
|
||||
let _ = progress_tx.try_send(ScanProgress {
|
||||
phase: "hashing".to_string(),
|
||||
current: i as u32 + 1,
|
||||
total: total_files,
|
||||
current_path: abs_path.display().to_string(),
|
||||
bytes_synced: 0,
|
||||
});
|
||||
|
||||
let rel_path = abs_path.strip_prefix(origin_root).unwrap_or(abs_path);
|
||||
|
||||
let existing = self.db.get_file_by_real_path(origin_id, rel_path)?;
|
||||
if existing.is_some() {
|
||||
unchanged += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let size = std::fs::metadata(abs_path).map(|m| m.len()).unwrap_or(0);
|
||||
|
||||
new_files.push(DiscoveredFile {
|
||||
abs_path: abs_path.clone(),
|
||||
rel_path: rel_path.to_path_buf(),
|
||||
size,
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
new = new_files.len(),
|
||||
unchanged = unchanged,
|
||||
"hash phase complete"
|
||||
);
|
||||
|
||||
// Phase 3: Indexing
|
||||
let mut synced = Vec::new();
|
||||
let mut bytes_synced = 0u64;
|
||||
let ingest_total = new_files.len() as u32;
|
||||
|
||||
for (i, file) in new_files.iter().enumerate() {
|
||||
let _ = progress_tx.try_send(ScanProgress {
|
||||
phase: "indexing".to_string(),
|
||||
current: i as u32 + 1,
|
||||
total: ingest_total,
|
||||
current_path: file.abs_path.display().to_string(),
|
||||
bytes_synced,
|
||||
});
|
||||
|
||||
let audio_meta = match self.parser.parse_file(&file.abs_path) {
|
||||
Ok(meta) => meta,
|
||||
Err(e) => {
|
||||
warn!(path = %file.abs_path.display(), error = %e, "parse failed, using defaults");
|
||||
AudioMeta::default()
|
||||
}
|
||||
};
|
||||
|
||||
let virtual_path = derive_virtual_path(&audio_meta, &file.rel_path);
|
||||
|
||||
let file_id = self.db.upsert_file(
|
||||
origin_id,
|
||||
&file.rel_path,
|
||||
&virtual_path,
|
||||
&audio_meta,
|
||||
UNIX_EPOCH,
|
||||
file.size,
|
||||
)?;
|
||||
|
||||
let file_meta = FileMeta {
|
||||
id: file_id,
|
||||
virtual_path: virtual_path.clone(),
|
||||
real_path: RealPath {
|
||||
origin_id: origin_id.clone(),
|
||||
path: file.rel_path.clone(),
|
||||
},
|
||||
size: file.size,
|
||||
mtime: UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: Some(audio_meta),
|
||||
};
|
||||
|
||||
{
|
||||
let mut tree = self.tree.write();
|
||||
tree.insert_file(&file_meta);
|
||||
}
|
||||
|
||||
self.fetcher.register_file(file_meta.clone());
|
||||
|
||||
self.event_bus.publish(Event::FileAdded {
|
||||
path: virtual_path.clone(),
|
||||
origin_id: origin_id.clone(),
|
||||
});
|
||||
|
||||
bytes_synced += file.size;
|
||||
|
||||
synced.push(SyncedFileInfo {
|
||||
path: file.abs_path.display().to_string(),
|
||||
file_id,
|
||||
virtual_path: virtual_path.as_str().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(ScanResult {
|
||||
new_files: synced,
|
||||
changed: 0,
|
||||
deleted: 0,
|
||||
unchanged,
|
||||
bytes_synced,
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_audio_files(
|
||||
&self,
|
||||
scan_root: &Path,
|
||||
progress_tx: &mpsc::Sender<ScanProgress>,
|
||||
) -> Result<Vec<PathBuf>> {
|
||||
let mut files = Vec::new();
|
||||
self.walk_dir(scan_root, &mut files, progress_tx)?;
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn walk_dir(
|
||||
&self,
|
||||
dir: &Path,
|
||||
files: &mut Vec<PathBuf>,
|
||||
progress_tx: &mpsc::Sender<ScanProgress>,
|
||||
) -> Result<()> {
|
||||
let entries = std::fs::read_dir(dir)
|
||||
.map_err(|e| Error::Origin(format!("read_dir {}: {}", dir.display(), e)))?;
|
||||
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
self.walk_dir(&path, files, progress_tx)?;
|
||||
} else if is_audio_file(&path) {
|
||||
files.push(path.clone());
|
||||
let _ = progress_tx.try_send(ScanProgress {
|
||||
phase: "scanning".to_string(),
|
||||
current: files.len() as u32,
|
||||
total: 0,
|
||||
current_path: path.display().to_string(),
|
||||
bytes_synced: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn derive_virtual_path(meta: &AudioMeta, rel_path: &Path) -> VirtualPath {
|
||||
let artist = meta.artist.as_deref().unwrap_or("Unknown Artist");
|
||||
let album = meta.album.as_deref().unwrap_or("Unknown Album");
|
||||
let filename = rel_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("unknown");
|
||||
|
||||
VirtualPath::new(format!("/{}/{}/{}", artist, album, filename))
|
||||
}
|
||||
|
||||
fn is_audio_file(path: &Path) -> bool {
|
||||
matches!(
|
||||
path.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.to_lowercase())
|
||||
.as_deref(),
|
||||
Some("flac" | "mp3" | "ogg" | "wav" | "m4a" | "aac" | "opus")
|
||||
)
|
||||
}
|
||||
|
||||
struct DiscoveredFile {
|
||||
abs_path: PathBuf,
|
||||
rel_path: PathBuf,
|
||||
size: u64,
|
||||
}
|
||||
+3
-1
@@ -35,7 +35,9 @@ impl MusicFs for SearchService {
|
||||
}
|
||||
|
||||
if req.query.len() > 256 {
|
||||
return Err(Status::invalid_argument("Query exceeds maximum length (256)"));
|
||||
return Err(Status::invalid_argument(
|
||||
"Query exceeds maximum length (256)",
|
||||
));
|
||||
}
|
||||
|
||||
let limit = req.limit.unwrap_or(100).min(10000) as usize;
|
||||
@@ -2,11 +2,11 @@ use crate::proto::musicfs::v1::{
|
||||
music_fs_server::MusicFs, CacheStats, ClearCacheRequest, ClearCacheResponse, Empty, Event,
|
||||
EventFilter, HealthStatus, MountState, OriginHealthResponse, OriginRequest, OriginsResponse,
|
||||
PrefetchProgress, PrefetchRequest, SearchRequest, SearchResponse, SearchResult,
|
||||
ShutdownRequest, StatusResponse, SyncProgress, TierStats,
|
||||
ShutdownRequest, StatusResponse, SyncProgress, SyncedFile, TierStats,
|
||||
};
|
||||
use musicfs_core::{Event as CoreEvent, EventBus};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::time::Instant;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
@@ -16,14 +16,30 @@ pub struct MusicFsServer {
|
||||
start_time: Instant,
|
||||
event_bus: Arc<EventBus>,
|
||||
version: String,
|
||||
scanner: Arc<crate::scanner::OriginScanner>,
|
||||
origin_root: std::path::PathBuf,
|
||||
}
|
||||
|
||||
impl MusicFsServer {
|
||||
pub fn new(event_bus: Arc<EventBus>) -> Self {
|
||||
pub fn new(
|
||||
event_bus: Arc<EventBus>,
|
||||
db: Arc<musicfs_cache::Database>,
|
||||
tree: Arc<parking_lot::RwLock<musicfs_cache::VirtualTree>>,
|
||||
fetcher: Arc<musicfs_cas::ContentFetcher>,
|
||||
origin_root: std::path::PathBuf,
|
||||
) -> Self {
|
||||
let scanner = Arc::new(crate::scanner::OriginScanner::new(
|
||||
db,
|
||||
event_bus.clone(),
|
||||
tree,
|
||||
fetcher,
|
||||
));
|
||||
Self {
|
||||
start_time: Instant::now(),
|
||||
event_bus,
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
scanner,
|
||||
origin_root,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,10 +244,7 @@ impl MusicFs for MusicFsServer {
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "shutdown"))]
|
||||
async fn shutdown(
|
||||
&self,
|
||||
request: Request<ShutdownRequest>,
|
||||
) -> Result<Response<Empty>, Status> {
|
||||
async fn shutdown(&self, request: Request<ShutdownRequest>) -> Result<Response<Empty>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(
|
||||
graceful = req.graceful,
|
||||
@@ -242,7 +255,11 @@ impl MusicFs for MusicFsServer {
|
||||
Ok(Response::new(Empty {}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _request), fields(method = "get_cache_stats"))]
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip(self, _request),
|
||||
fields(method = "get_cache_stats")
|
||||
)]
|
||||
async fn get_cache_stats(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
@@ -339,7 +356,11 @@ impl MusicFs for MusicFsServer {
|
||||
Ok(Response::new(OriginsResponse { origins: vec![] }))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, request), fields(method = "get_origin_health"))]
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip(self, request),
|
||||
fields(method = "get_origin_health")
|
||||
)]
|
||||
async fn get_origin_health(
|
||||
&self,
|
||||
request: Request<OriginRequest>,
|
||||
@@ -363,24 +384,85 @@ impl MusicFs for MusicFsServer {
|
||||
request: Request<OriginRequest>,
|
||||
) -> Result<Response<Self::RescanOriginStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(origin_id = %req.origin_id, "gRPC rescan_origin started");
|
||||
let subdir = req.subdir.as_deref().filter(|s| !s.is_empty());
|
||||
info!(
|
||||
origin_id = %req.origin_id,
|
||||
subdir = ?subdir,
|
||||
"gRPC rescan_origin started"
|
||||
);
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
let (progress_tx, mut progress_rx) = mpsc::channel::<crate::scanner::ScanProgress>(64);
|
||||
|
||||
let origin_id = musicfs_core::OriginId::from(req.origin_id.as_str());
|
||||
let scanner = self.scanner.clone();
|
||||
let origin_root = self.origin_root.clone();
|
||||
let subdir_owned = subdir.map(|s| s.to_string());
|
||||
|
||||
tokio::spawn(async move {
|
||||
let phases = ["scanning", "indexing", "complete"];
|
||||
for (i, phase) in phases.iter().enumerate() {
|
||||
let progress = SyncProgress {
|
||||
phase: phase.to_string(),
|
||||
current: i as u32 + 1,
|
||||
total: phases.len() as u32,
|
||||
current_path: String::new(),
|
||||
bytes_synced: 0,
|
||||
};
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
let forward_handle = {
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some(progress) = progress_rx.recv().await {
|
||||
let proto = SyncProgress {
|
||||
phase: progress.phase,
|
||||
current: progress.current,
|
||||
total: progress.total,
|
||||
current_path: progress.current_path,
|
||||
bytes_synced: progress.bytes_synced,
|
||||
new_files: vec![],
|
||||
};
|
||||
if tx.send(Ok(proto)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let result = scanner
|
||||
.scan(
|
||||
&origin_id,
|
||||
&origin_root,
|
||||
subdir_owned.as_deref(),
|
||||
progress_tx,
|
||||
)
|
||||
.await;
|
||||
|
||||
forward_handle.abort();
|
||||
|
||||
match result {
|
||||
Ok(scan_result) => {
|
||||
let synced_files: Vec<SyncedFile> = scan_result
|
||||
.new_files
|
||||
.iter()
|
||||
.map(|f| SyncedFile {
|
||||
path: f.path.clone(),
|
||||
file_id: f.file_id.0,
|
||||
virtual_path: f.virtual_path.clone(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let _ = tx
|
||||
.send(Ok(SyncProgress {
|
||||
phase: "complete".to_string(),
|
||||
current: scan_result.new_files.len() as u32
|
||||
+ scan_result.changed
|
||||
+ scan_result.deleted,
|
||||
total: scan_result.new_files.len() as u32
|
||||
+ scan_result.changed
|
||||
+ scan_result.deleted
|
||||
+ scan_result.unchanged,
|
||||
current_path: String::new(),
|
||||
bytes_synced: scan_result.bytes_synced,
|
||||
new_files: synced_files,
|
||||
}))
|
||||
.await;
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Err(Status::internal(format!("rescan failed: {}", e))))
|
||||
.await;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -389,7 +471,11 @@ impl MusicFs for MusicFsServer {
|
||||
|
||||
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "subscribe_events"))]
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "subscribe_events")
|
||||
)]
|
||||
async fn subscribe_events(
|
||||
&self,
|
||||
request: Request<EventFilter>,
|
||||
@@ -429,10 +515,29 @@ impl MusicFs for MusicFsServer {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
async fn make_test_server() -> (MusicFsServer, tempfile::TempDir) {
|
||||
let event_bus = Arc::new(EventBus::new(16));
|
||||
let db = Arc::new(musicfs_cache::Database::open_memory().unwrap());
|
||||
let tree = Arc::new(parking_lot::RwLock::new(
|
||||
musicfs_cache::TreeBuilder::new().build(),
|
||||
));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cfg = musicfs_cas::CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(musicfs_cas::CasStore::open(cfg).await.unwrap());
|
||||
let fetcher = Arc::new(musicfs_cas::ContentFetcher::new(store));
|
||||
let origin_root = std::path::PathBuf::from("/tmp/test-origin");
|
||||
(
|
||||
MusicFsServer::new(event_bus, db, tree, fetcher, origin_root),
|
||||
dir,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_status() {
|
||||
let event_bus = Arc::new(EventBus::new(16));
|
||||
let server = MusicFsServer::new(event_bus);
|
||||
let (server, _dir) = make_test_server().await;
|
||||
|
||||
let response = server.get_status(Request::new(Empty {})).await.unwrap();
|
||||
let status = response.into_inner();
|
||||
@@ -443,8 +548,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_cache_stats() {
|
||||
let event_bus = Arc::new(EventBus::new(16));
|
||||
let server = MusicFsServer::new(event_bus);
|
||||
let (server, _dir) = make_test_server().await;
|
||||
|
||||
let response = server
|
||||
.get_cache_stats(Request::new(Empty {}))
|
||||
@@ -277,7 +277,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_event_type_name() {
|
||||
let handler = WebhookHandler::new(vec![]);
|
||||
let handler = WebhookHandler::new(vec![]).unwrap();
|
||||
|
||||
let event = Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
@@ -287,7 +287,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_matches_filter_empty() {
|
||||
let handler = WebhookHandler::new(vec![]);
|
||||
let handler = WebhookHandler::new(vec![]).unwrap();
|
||||
let config = WebhookConfig {
|
||||
url: "http://example.com".to_string(),
|
||||
secret: None,
|
||||
@@ -304,7 +304,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_matches_filter_specific() {
|
||||
let handler = WebhookHandler::new(vec![]);
|
||||
let handler = WebhookHandler::new(vec![]).unwrap();
|
||||
let config = WebhookConfig {
|
||||
url: "http://example.com".to_string(),
|
||||
secret: None,
|
||||
@@ -0,0 +1,209 @@
|
||||
use musicfs_core::{AudioFormat, AudioMeta, Error, Result};
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use symphonia::core::codecs::CODEC_TYPE_NULL;
|
||||
use symphonia::core::formats::FormatOptions;
|
||||
use symphonia::core::io::MediaSourceStream;
|
||||
use symphonia::core::meta::MetadataOptions;
|
||||
use symphonia::core::probe::Hint;
|
||||
use tracing::debug;
|
||||
|
||||
pub struct MetadataParser;
|
||||
|
||||
impl MetadataParser {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
pub fn parse_file(&self, path: &Path) -> Result<AudioMeta> {
|
||||
let file = File::open(path)?;
|
||||
let mss = MediaSourceStream::new(Box::new(file), Default::default());
|
||||
|
||||
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
|
||||
|
||||
let mut hint = Hint::new();
|
||||
if !ext.is_empty() {
|
||||
hint.with_extension(ext);
|
||||
}
|
||||
|
||||
let fmt_opts = FormatOptions::default();
|
||||
let meta_opts = MetadataOptions::default();
|
||||
|
||||
let probed = symphonia::default::get_probe()
|
||||
.format(&hint, mss, &fmt_opts, &meta_opts)
|
||||
.map_err(|e| Error::Metadata(format!("Failed to probe format: {}", e)))?;
|
||||
let mut format = probed.format;
|
||||
|
||||
let mut audio_meta = AudioMeta {
|
||||
format: AudioFormat::from_extension(ext),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Some(metadata) = format.metadata().current() {
|
||||
self.extract_tags(&mut audio_meta, metadata);
|
||||
}
|
||||
|
||||
if let Some(track) = format
|
||||
.tracks()
|
||||
.iter()
|
||||
.find(|t| t.codec_params.codec != CODEC_TYPE_NULL)
|
||||
{
|
||||
let params = &track.codec_params;
|
||||
|
||||
if let Some(n_frames) = params.n_frames {
|
||||
if let Some(sample_rate) = params.sample_rate {
|
||||
audio_meta.duration_ms = Some((n_frames as u64 * 1000) / sample_rate as u64);
|
||||
audio_meta.sample_rate = Some(sample_rate);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(channels) = params.channels {
|
||||
audio_meta.channels = Some(channels.count() as u32);
|
||||
}
|
||||
|
||||
if let Some(bits_per_sample) = params.bits_per_sample {
|
||||
audio_meta.bits_per_sample = Some(bits_per_sample);
|
||||
if let Some(sample_rate) = params.sample_rate {
|
||||
if let Some(channels) = params.channels {
|
||||
audio_meta.bitrate =
|
||||
Some(bits_per_sample * sample_rate * channels.count() as u32 / 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!(?audio_meta, "Parsed metadata");
|
||||
Ok(audio_meta)
|
||||
}
|
||||
|
||||
fn extract_tags(
|
||||
&self,
|
||||
meta: &mut AudioMeta,
|
||||
metadata: &symphonia::core::meta::MetadataRevision,
|
||||
) {
|
||||
use symphonia::core::meta::StandardTagKey;
|
||||
|
||||
for tag in metadata.tags() {
|
||||
if let Some(std_key) = tag.std_key {
|
||||
let value = tag.value.to_string();
|
||||
match std_key {
|
||||
// Basic metadata
|
||||
StandardTagKey::TrackTitle => meta.title = Some(value),
|
||||
StandardTagKey::Artist => meta.artist = Some(value),
|
||||
StandardTagKey::Album => meta.album = Some(value),
|
||||
StandardTagKey::AlbumArtist => meta.album_artist = Some(value),
|
||||
StandardTagKey::Genre => meta.genre = Some(value),
|
||||
|
||||
// Track/disc with totals (parse "X/Y" format)
|
||||
StandardTagKey::TrackNumber => {
|
||||
let parts: Vec<&str> = value.split('/').collect();
|
||||
meta.track = parts.first().and_then(|s| s.trim().parse().ok());
|
||||
if parts.len() > 1 {
|
||||
meta.track_total = parts.get(1).and_then(|s| s.trim().parse().ok());
|
||||
}
|
||||
}
|
||||
StandardTagKey::DiscNumber => {
|
||||
let parts: Vec<&str> = value.split('/').collect();
|
||||
meta.disc = parts.first().and_then(|s| s.trim().parse().ok());
|
||||
if parts.len() > 1 {
|
||||
meta.disc_total = parts.get(1).and_then(|s| s.trim().parse().ok());
|
||||
}
|
||||
}
|
||||
StandardTagKey::TrackTotal => {
|
||||
meta.track_total = value.trim().parse().ok();
|
||||
}
|
||||
StandardTagKey::DiscTotal => {
|
||||
meta.disc_total = value.trim().parse().ok();
|
||||
}
|
||||
|
||||
// Date handling: store full date string, extract year
|
||||
StandardTagKey::Date | StandardTagKey::ReleaseDate => {
|
||||
meta.date = Some(value.clone());
|
||||
meta.year = value.chars().take(4).collect::<String>().parse().ok();
|
||||
}
|
||||
|
||||
// Additional metadata
|
||||
StandardTagKey::Composer => meta.composer = Some(value),
|
||||
StandardTagKey::Comment => meta.comment = Some(value),
|
||||
StandardTagKey::Lyrics => meta.lyrics = Some(value),
|
||||
StandardTagKey::Copyright => meta.copyright = Some(value),
|
||||
StandardTagKey::Compilation => {
|
||||
meta.compilation = Some(value == "1" || value.eq_ignore_ascii_case("true"));
|
||||
}
|
||||
StandardTagKey::Encoder => meta.encoder = Some(value),
|
||||
|
||||
// Sort keys
|
||||
StandardTagKey::SortTrackTitle => meta.title_sort = Some(value),
|
||||
StandardTagKey::SortArtist => meta.artist_sort = Some(value),
|
||||
StandardTagKey::SortAlbum => meta.album_sort = Some(value),
|
||||
StandardTagKey::SortAlbumArtist => meta.album_artist_sort = Some(value),
|
||||
|
||||
// MusicBrainz IDs
|
||||
StandardTagKey::MusicBrainzRecordingId => meta.mb_recording_id = Some(value),
|
||||
StandardTagKey::MusicBrainzAlbumId => meta.mb_album_id = Some(value),
|
||||
StandardTagKey::MusicBrainzArtistId => meta.mb_artist_id = Some(value),
|
||||
StandardTagKey::MusicBrainzAlbumArtistId => {
|
||||
meta.mb_album_artist_id = Some(value)
|
||||
}
|
||||
StandardTagKey::MusicBrainzReleaseGroupId => {
|
||||
meta.mb_release_group_id = Some(value)
|
||||
}
|
||||
|
||||
// ReplayGain (parse as f32, values may have "dB" suffix)
|
||||
StandardTagKey::ReplayGainTrackGain => {
|
||||
meta.replaygain_track_gain = parse_replaygain(&value);
|
||||
}
|
||||
StandardTagKey::ReplayGainTrackPeak => {
|
||||
meta.replaygain_track_peak = value.trim().parse().ok();
|
||||
}
|
||||
StandardTagKey::ReplayGainAlbumGain => {
|
||||
meta.replaygain_album_gain = parse_replaygain(&value);
|
||||
}
|
||||
StandardTagKey::ReplayGainAlbumPeak => {
|
||||
meta.replaygain_album_peak = value.trim().parse().ok();
|
||||
}
|
||||
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse ReplayGain value, stripping optional "dB" suffix
|
||||
fn parse_replaygain(value: &str) -> Option<f32> {
|
||||
let trimmed = value.trim();
|
||||
let without_db = trimmed
|
||||
.strip_suffix("dB")
|
||||
.or_else(|| trimmed.strip_suffix(" dB"))
|
||||
.unwrap_or(trimmed);
|
||||
without_db.trim().parse().ok()
|
||||
}
|
||||
|
||||
impl Default for MetadataParser {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_audio_format_detection() {
|
||||
assert_eq!(AudioFormat::from_extension("flac"), AudioFormat::Flac);
|
||||
assert_eq!(AudioFormat::from_extension("mp3"), AudioFormat::Mp3);
|
||||
assert_eq!(AudioFormat::from_extension("opus"), AudioFormat::Opus);
|
||||
assert_eq!(AudioFormat::from_extension("ogg"), AudioFormat::Vorbis);
|
||||
assert_eq!(AudioFormat::from_extension("m4a"), AudioFormat::Aac);
|
||||
assert_eq!(AudioFormat::from_extension("wav"), AudioFormat::Wav);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parser_creation() {
|
||||
let parser = MetadataParser::new();
|
||||
let default_parser = MetadataParser::default();
|
||||
assert!(std::mem::size_of_val(&parser) == std::mem::size_of_val(&default_parser));
|
||||
}
|
||||
}
|
||||
@@ -12,10 +12,12 @@ sftp = []
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
async-trait.workspace = true
|
||||
dashmap.workspace = true
|
||||
futures.workspace = true
|
||||
libc.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "sync", "time"] }
|
||||
tracing.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
+10
-10
@@ -67,11 +67,10 @@ impl FailoverExecutor {
|
||||
|
||||
if origins.is_empty() {
|
||||
if let Some(origin) = self.registry.route_with_fallback(path) {
|
||||
warn!(
|
||||
"No healthy origins, using fallback origin {}",
|
||||
origin.id()
|
||||
);
|
||||
return self.read_with_retry(&origin, &path.path, offset, size).await;
|
||||
warn!("No healthy origins, using fallback origin {}", origin.id());
|
||||
return self
|
||||
.read_with_retry(&origin, &path.path, offset, size)
|
||||
.await;
|
||||
}
|
||||
return Err(Error::NoOriginAvailable);
|
||||
}
|
||||
@@ -81,7 +80,10 @@ impl FailoverExecutor {
|
||||
for origin in origins {
|
||||
trace!(origin_id = %origin.id(), "Attempting read from origin");
|
||||
let start = std::time::Instant::now();
|
||||
match self.read_with_retry(&origin, &path.path, offset, size).await {
|
||||
match self
|
||||
.read_with_retry(&origin, &path.path, offset, size)
|
||||
.await
|
||||
{
|
||||
Ok(data) => {
|
||||
let latency = start.elapsed().as_millis() as u64;
|
||||
self.registry.record_latency(origin.id(), latency);
|
||||
@@ -214,10 +216,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_custom_delays() {
|
||||
let config = RetryConfig::with_delays(vec![
|
||||
Duration::from_millis(50),
|
||||
Duration::from_millis(100),
|
||||
]);
|
||||
let config =
|
||||
RetryConfig::with_delays(vec![Duration::from_millis(50), Duration::from_millis(100)]);
|
||||
|
||||
assert_eq!(config.max_attempts, 2);
|
||||
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(50));
|
||||
+33
-10
@@ -1,11 +1,12 @@
|
||||
use crate::traits::Origin;
|
||||
use dashmap::DashMap;
|
||||
use futures::future::join_all;
|
||||
use musicfs_core::{Event, EventBus, HealthStatus, OriginId, OriginType};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, info_span, Instrument};
|
||||
use tracing::{debug, info, info_span, warn, Instrument};
|
||||
|
||||
pub struct HealthMonitor {
|
||||
origins: DashMap<OriginId, Arc<dyn Origin>>,
|
||||
@@ -180,21 +181,37 @@ impl HealthMonitor {
|
||||
HealthCheckHandle { stop_tx }
|
||||
}
|
||||
|
||||
async fn check_all(&self) {
|
||||
pub async fn check_all(&self) {
|
||||
let origins: Vec<_> = self
|
||||
.origins
|
||||
.iter()
|
||||
.map(|e| (e.key().clone(), e.value().clone()))
|
||||
.collect();
|
||||
|
||||
for (id, origin) in origins {
|
||||
self.check_one(&id, &origin).await;
|
||||
}
|
||||
let checks: Vec<_> = origins
|
||||
.iter()
|
||||
.map(|(id, origin)| self.check_one(id, origin))
|
||||
.collect();
|
||||
|
||||
join_all(checks).await;
|
||||
}
|
||||
|
||||
async fn check_one(&self, id: &OriginId, origin: &Arc<dyn Origin>) {
|
||||
let start = Instant::now();
|
||||
let status = origin.health().await;
|
||||
let health_timeout = Duration::from_millis(1500);
|
||||
|
||||
let status = match tokio::time::timeout(health_timeout, origin.health()).await {
|
||||
Ok(status) => status,
|
||||
Err(_) => {
|
||||
warn!(
|
||||
origin_id = %id,
|
||||
timeout_ms = health_timeout.as_millis() as u64,
|
||||
"Health check timed out"
|
||||
);
|
||||
HealthStatus::Unhealthy
|
||||
}
|
||||
};
|
||||
|
||||
let latency_ms = start.elapsed().as_millis() as u64;
|
||||
|
||||
let threshold = self.threshold_for(origin.origin_type());
|
||||
@@ -332,10 +349,13 @@ mod tests {
|
||||
let mut thresholds = HashMap::new();
|
||||
thresholds.insert(OriginType::Local, 3);
|
||||
|
||||
let monitor = HealthMonitor::new(Duration::from_secs(30))
|
||||
.with_per_type_thresholds(thresholds);
|
||||
let monitor =
|
||||
HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds);
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new("missing", std::path::Path::new("/nonexistent")));
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
"missing",
|
||||
std::path::Path::new("/nonexistent"),
|
||||
));
|
||||
monitor.add_origin(origin);
|
||||
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
@@ -355,7 +375,10 @@ mod tests {
|
||||
async fn test_local_origin_threshold_is_one() {
|
||||
let monitor = HealthMonitor::new(Duration::from_secs(30));
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new("missing", std::path::Path::new("/nonexistent")));
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
"missing",
|
||||
std::path::Path::new("/nonexistent"),
|
||||
));
|
||||
monitor.add_origin(origin);
|
||||
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
+10
-10
@@ -2,8 +2,9 @@ use crate::health::{HealthMonitor, HealthSnapshot};
|
||||
use crate::router::Router;
|
||||
use crate::traits::{Origin, WatchHandle};
|
||||
use musicfs_core::{OriginId, RealPath};
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub struct OriginRegistry {
|
||||
@@ -29,17 +30,17 @@ impl OriginRegistry {
|
||||
|
||||
self.router.set_priority(id.clone(), priority);
|
||||
self.health_monitor.add_origin(origin.clone());
|
||||
self.origins.write().unwrap().insert(id, origin);
|
||||
self.origins.write().insert(id, origin);
|
||||
}
|
||||
|
||||
pub fn unregister(&self, id: &OriginId) {
|
||||
info!("Unregistering origin {}", id);
|
||||
|
||||
if let Some(handles) = self.watch_handles.write().unwrap().remove(id) {
|
||||
if let Some(handles) = self.watch_handles.write().remove(id) {
|
||||
info!("Dropping {} watch handles for origin {}", handles.len(), id);
|
||||
}
|
||||
|
||||
self.origins.write().unwrap().remove(id);
|
||||
self.origins.write().remove(id);
|
||||
self.router.remove_priority(id);
|
||||
self.health_monitor.remove_origin(id);
|
||||
}
|
||||
@@ -47,22 +48,21 @@ impl OriginRegistry {
|
||||
pub fn register_watch(&self, origin_id: &OriginId, handle: WatchHandle) {
|
||||
self.watch_handles
|
||||
.write()
|
||||
.unwrap()
|
||||
.entry(origin_id.clone())
|
||||
.or_default()
|
||||
.push(handle);
|
||||
}
|
||||
|
||||
pub fn get(&self, id: &OriginId) -> Option<Arc<dyn Origin>> {
|
||||
self.origins.read().unwrap().get(id).cloned()
|
||||
self.origins.read().get(id).cloned()
|
||||
}
|
||||
|
||||
pub fn list(&self) -> Vec<Arc<dyn Origin>> {
|
||||
self.origins.read().unwrap().values().cloned().collect()
|
||||
self.origins.read().values().cloned().collect()
|
||||
}
|
||||
|
||||
pub fn route(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read().unwrap();
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let candidates: Vec<_> = origins
|
||||
@@ -86,7 +86,7 @@ impl OriginRegistry {
|
||||
}
|
||||
|
||||
pub fn route_with_fallback(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read().unwrap();
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let candidates: Vec<_> = origins
|
||||
@@ -109,7 +109,7 @@ impl OriginRegistry {
|
||||
}
|
||||
|
||||
pub fn route_all(&self, path: &RealPath) -> Vec<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read().unwrap();
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let mut result: Vec<_> = origins
|
||||
@@ -86,7 +86,7 @@ impl Router {
|
||||
(priority, latency)
|
||||
})
|
||||
.cloned();
|
||||
|
||||
|
||||
if let Some(ref id) = selected {
|
||||
let priority = self.get_priority(id);
|
||||
let latency = self.latency_stats.get(id).map(|s| s.p50_ms).unwrap_or(0);
|
||||
@@ -97,7 +97,7 @@ impl Router {
|
||||
"Selected healthy origin"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
selected
|
||||
}
|
||||
|
||||
@@ -141,7 +141,7 @@ impl Router {
|
||||
(failures, priority)
|
||||
})
|
||||
.cloned();
|
||||
|
||||
|
||||
if let Some(ref id) = selected {
|
||||
let failures = health.failure_count(id).unwrap_or(u32::MAX);
|
||||
trace!(
|
||||
@@ -151,7 +151,7 @@ impl Router {
|
||||
"Selected least-bad unhealthy origin"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
selected
|
||||
}
|
||||
}
|
||||
@@ -47,5 +47,3 @@
|
||||
mod implementation {
|
||||
// Full S3 implementation would go here when aws-sdk-s3 is enabled
|
||||
}
|
||||
|
||||
|
||||
@@ -91,11 +91,13 @@ impl Origin for SmbOrigin {
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
self.retry_on_disconnect(|| self.inner.read(path, offset, size)).await
|
||||
self.retry_on_disconnect(|| self.inner.read(path, offset, size))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
self.retry_on_disconnect(|| self.inner.read_full(path)).await
|
||||
self.retry_on_disconnect(|| self.inner.read_full(path))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
+10
-10
@@ -55,9 +55,8 @@ impl NativePluginHost {
|
||||
info!("Loading native plugin from {:?}", canonical);
|
||||
|
||||
let library = unsafe {
|
||||
Library::new(&canonical).map_err(|e| {
|
||||
PluginError::LoadFailed(format!("Failed to load library: {}", e))
|
||||
})?
|
||||
Library::new(&canonical)
|
||||
.map_err(|e| PluginError::LoadFailed(format!("Failed to load library: {}", e)))?
|
||||
};
|
||||
|
||||
self.verify_api_version(&library)?;
|
||||
@@ -190,9 +189,9 @@ impl NativePluginHost {
|
||||
|
||||
fn verify_api_version(&self, library: &Library) -> Result<()> {
|
||||
let version_fn: Symbol<unsafe extern "C" fn() -> *const std::ffi::c_char> = unsafe {
|
||||
library
|
||||
.get(b"musicfs_plugin_api_version")
|
||||
.map_err(|_| PluginError::SymbolNotFound("musicfs_plugin_api_version".to_string()))?
|
||||
library.get(b"musicfs_plugin_api_version").map_err(|_| {
|
||||
PluginError::SymbolNotFound("musicfs_plugin_api_version".to_string())
|
||||
})?
|
||||
};
|
||||
|
||||
let version_ptr = unsafe { version_fn() };
|
||||
@@ -203,10 +202,11 @@ impl NativePluginHost {
|
||||
actual: "<invalid UTF-8>".to_string(),
|
||||
})?;
|
||||
|
||||
let plugin_version = Version::parse(version_str).map_err(|_| PluginError::VersionMismatch {
|
||||
expected: PLUGIN_API_VERSION.to_string(),
|
||||
actual: version_str.to_string(),
|
||||
})?;
|
||||
let plugin_version =
|
||||
Version::parse(version_str).map_err(|_| PluginError::VersionMismatch {
|
||||
expected: PLUGIN_API_VERSION.to_string(),
|
||||
actual: version_str.to_string(),
|
||||
})?;
|
||||
|
||||
let expected_version = Version::parse(PLUGIN_API_VERSION).unwrap();
|
||||
|
||||
@@ -95,11 +95,7 @@ pub trait OriginPlugin: Plugin {
|
||||
///
|
||||
/// The config contains origin-specific settings (credentials, paths, etc).
|
||||
/// Returns a boxed Origin that can be used by the OriginRouter.
|
||||
async fn create_origin(
|
||||
&self,
|
||||
id: &str,
|
||||
config: Value,
|
||||
) -> Result<Box<dyn OriginInstance>>;
|
||||
async fn create_origin(&self, id: &str, config: Value) -> Result<Box<dyn OriginInstance>>;
|
||||
}
|
||||
|
||||
/// Instance created by OriginPlugin
|
||||
+13
-3
@@ -261,7 +261,12 @@ mod tests {
|
||||
let store = CollectionStore::new(&db_path).unwrap();
|
||||
|
||||
let collection = store
|
||||
.create("Jazz", CollectionQuery::Genre { genre: "Jazz".to_string() })
|
||||
.create(
|
||||
"Jazz",
|
||||
CollectionQuery::Genre {
|
||||
genre: "Jazz".to_string(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(collection.name, "Jazz");
|
||||
@@ -279,7 +284,9 @@ mod tests {
|
||||
let query = CollectionQuery::Compound {
|
||||
op: BoolOp::And,
|
||||
children: vec![
|
||||
CollectionQuery::Genre { genre: "Metal".to_string() },
|
||||
CollectionQuery::Genre {
|
||||
genre: "Metal".to_string(),
|
||||
},
|
||||
CollectionQuery::DateRange {
|
||||
field: "year".to_string(),
|
||||
start: 1980,
|
||||
@@ -306,6 +313,9 @@ mod tests {
|
||||
assert!(CollectionQuery::RecentlyAdded { days: 30 }.is_dynamic());
|
||||
assert!(CollectionQuery::RecentlyPlayed { days: 7 }.is_dynamic());
|
||||
assert!(CollectionQuery::MostPlayed { limit: 100 }.is_dynamic());
|
||||
assert!(!CollectionQuery::Genre { genre: "Rock".to_string() }.is_dynamic());
|
||||
assert!(!CollectionQuery::Genre {
|
||||
genre: "Rock".to_string()
|
||||
}
|
||||
.is_dynamic());
|
||||
}
|
||||
}
|
||||
@@ -4,9 +4,9 @@ use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tantivy::collector::TopDocs;
|
||||
use tantivy::query::{BooleanQuery, FuzzyTermQuery, Occur, Query, QueryParser};
|
||||
use tantivy::schema::{Field, Schema, Value, STORED, TEXT, INDEXED};
|
||||
use tantivy::schema::{Field, Schema, Value, INDEXED, STORED, TEXT};
|
||||
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const SCHEMA_VERSION: u32 = 1;
|
||||
|
||||
@@ -95,6 +95,27 @@ impl SearchIndex {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_with_recovery(index_path: &Path) -> Result<Self, SearchError> {
|
||||
match Self::open(index_path) {
|
||||
Ok(index) => {
|
||||
let docs = index.reader.searcher().num_docs();
|
||||
info!(docs, "Search index opened successfully");
|
||||
Ok(index)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
error = %e,
|
||||
path = ?index_path,
|
||||
"Search index corrupted, rebuilding from scratch"
|
||||
);
|
||||
if index_path.exists() {
|
||||
std::fs::remove_dir_all(index_path).map_err(SearchError::Io)?;
|
||||
}
|
||||
Self::open(index_path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn index_file(&self, file: &FileMeta) -> Result<(), SearchError> {
|
||||
let mut doc = TantivyDocument::new();
|
||||
|
||||
@@ -183,20 +204,21 @@ impl SearchIndex {
|
||||
self.schema.composer,
|
||||
];
|
||||
|
||||
let query: Box<dyn Query> = if let Some((term, distance)) = Self::parse_fuzzy_query(query_str) {
|
||||
let subqueries: Vec<(Occur, Box<dyn Query>)> = default_fields
|
||||
.iter()
|
||||
.map(|&field| {
|
||||
let term = Term::from_field_text(field, &term);
|
||||
let fuzzy = FuzzyTermQuery::new(term, distance, true);
|
||||
(Occur::Should, Box::new(fuzzy) as Box<dyn Query>)
|
||||
})
|
||||
.collect();
|
||||
Box::new(BooleanQuery::new(subqueries))
|
||||
} else {
|
||||
let query_parser = QueryParser::for_index(&self.index, default_fields);
|
||||
query_parser.parse_query(query_str)?
|
||||
};
|
||||
let query: Box<dyn Query> =
|
||||
if let Some((term, distance)) = Self::parse_fuzzy_query(query_str) {
|
||||
let subqueries: Vec<(Occur, Box<dyn Query>)> = default_fields
|
||||
.iter()
|
||||
.map(|&field| {
|
||||
let term = Term::from_field_text(field, &term);
|
||||
let fuzzy = FuzzyTermQuery::new(term, distance, true);
|
||||
(Occur::Should, Box::new(fuzzy) as Box<dyn Query>)
|
||||
})
|
||||
.collect();
|
||||
Box::new(BooleanQuery::new(subqueries))
|
||||
} else {
|
||||
let query_parser = QueryParser::for_index(&self.index, default_fields);
|
||||
query_parser.parse_query(query_str)?
|
||||
};
|
||||
|
||||
let top_docs = searcher.search(&*query, &TopDocs::with_limit(limit))?;
|
||||
|
||||
@@ -219,9 +241,18 @@ impl SearchIndex {
|
||||
results.push(SearchHit {
|
||||
file_id,
|
||||
virtual_path,
|
||||
artist: doc.get_first(self.schema.artist).and_then(|v| v.as_str()).map(String::from),
|
||||
album: doc.get_first(self.schema.album).and_then(|v| v.as_str()).map(String::from),
|
||||
title: doc.get_first(self.schema.title).and_then(|v| v.as_str()).map(String::from),
|
||||
artist: doc
|
||||
.get_first(self.schema.artist)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from),
|
||||
album: doc
|
||||
.get_first(self.schema.album)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from),
|
||||
title: doc
|
||||
.get_first(self.schema.title)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from),
|
||||
score,
|
||||
});
|
||||
}
|
||||
@@ -300,9 +331,15 @@ mod tests {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
|
||||
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap();
|
||||
index.index_file(&make_file(2, "Metallica", "Master of Puppets", "Battery")).unwrap();
|
||||
index.index_file(&make_file(3, "Iron Maiden", "Powerslave", "Aces High")).unwrap();
|
||||
index
|
||||
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
|
||||
.unwrap();
|
||||
index
|
||||
.index_file(&make_file(2, "Metallica", "Master of Puppets", "Battery"))
|
||||
.unwrap();
|
||||
index
|
||||
.index_file(&make_file(3, "Iron Maiden", "Powerslave", "Aces High"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
let results = index.search("metallica", 10).unwrap();
|
||||
@@ -318,7 +355,9 @@ mod tests {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
|
||||
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap();
|
||||
index
|
||||
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
let results = index.search("metalica~1", 10).unwrap();
|
||||
@@ -330,7 +369,9 @@ mod tests {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
|
||||
index.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman")).unwrap();
|
||||
index
|
||||
.index_file(&make_file(1, "Metallica", "Black Album", "Enter Sandman"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
let results = index.search("genre:Metal", 10).unwrap();
|
||||
@@ -342,7 +383,9 @@ mod tests {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
|
||||
index.index_file(&make_file(1, "Test", "Album", "Song")).unwrap();
|
||||
index
|
||||
.index_file(&make_file(1, "Test", "Album", "Song"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
|
||||
assert_eq!(index.search("test", 10).unwrap().len(), 1);
|
||||
@@ -359,7 +402,9 @@ mod tests {
|
||||
|
||||
{
|
||||
let index = SearchIndex::open(dir.path()).unwrap();
|
||||
index.index_file(&make_file(1, "Artist", "Album", "Track")).unwrap();
|
||||
index
|
||||
.index_file(&make_file(1, "Artist", "Album", "Track"))
|
||||
.unwrap();
|
||||
index.commit().unwrap();
|
||||
}
|
||||
|
||||
@@ -15,11 +15,7 @@ pub struct Indexer<M: MetadataLookup> {
|
||||
}
|
||||
|
||||
impl<M: MetadataLookup + 'static> Indexer<M> {
|
||||
pub fn new(
|
||||
index: Arc<SearchIndex>,
|
||||
event_bus: Arc<EventBus>,
|
||||
metadata_lookup: Arc<M>,
|
||||
) -> Self {
|
||||
pub fn new(index: Arc<SearchIndex>, event_bus: Arc<EventBus>, metadata_lookup: Arc<M>) -> Self {
|
||||
Self {
|
||||
index,
|
||||
event_bus,
|
||||
@@ -4,8 +4,7 @@ mod indexer;
|
||||
mod query;
|
||||
|
||||
pub use collections::{
|
||||
builtin_collections, BoolOp, CollectionError, CollectionQuery, CollectionStore,
|
||||
SmartCollection,
|
||||
builtin_collections, BoolOp, CollectionError, CollectionQuery, CollectionStore, SmartCollection,
|
||||
};
|
||||
pub use index::{SearchError, SearchHit, SearchIndex};
|
||||
pub use indexer::{Indexer, IndexerHandle, MetadataLookup};
|
||||
@@ -138,14 +138,21 @@ mod tests {
|
||||
|
||||
let shared = hashes1.intersection(&hashes2).count();
|
||||
|
||||
assert!(shared > 0, "CDC should produce stable boundaries, got {} chunks in original, {} after prepend", chunks1.len(), chunks2.len());
|
||||
assert!(
|
||||
shared > 0,
|
||||
"CDC should produce stable boundaries, got {} chunks in original, {} after prepend",
|
||||
chunks1.len(),
|
||||
chunks2.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cdc_chunk_sizes() {
|
||||
let chunker = CdcChunker::default();
|
||||
|
||||
let data: Vec<u8> = (0..1024 * 1024).map(|i| ((i * 17 + 31) % 256) as u8).collect();
|
||||
let data: Vec<u8> = (0..1024 * 1024)
|
||||
.map(|i| ((i * 17 + 31) % 256) as u8)
|
||||
.collect();
|
||||
|
||||
let chunks = chunker.chunk(&data);
|
||||
|
||||
@@ -68,7 +68,7 @@ impl DeltaDetector {
|
||||
) -> Result<ChangeSet, DeltaError> {
|
||||
let origin_id = origin.id().clone();
|
||||
info!(origin_id = %origin_id, "Starting delta detection");
|
||||
|
||||
|
||||
let mut changes = ChangeSet::default();
|
||||
|
||||
let origin_files = self.scan_origin(origin).await?;
|
||||
@@ -187,7 +187,11 @@ impl DeltaDetector {
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn compute_diff(&self, old_chunks: &[ManifestChunk], new_chunks: &[ManifestChunk]) -> ManifestDiff {
|
||||
fn compute_diff(
|
||||
&self,
|
||||
old_chunks: &[ManifestChunk],
|
||||
new_chunks: &[ManifestChunk],
|
||||
) -> ManifestDiff {
|
||||
let old_hashes: HashSet<_> = old_chunks.iter().map(|c| c.hash).collect();
|
||||
let new_hashes: HashSet<_> = new_chunks.iter().map(|c| c.hash).collect();
|
||||
|
||||
@@ -34,7 +34,8 @@ impl OriginWatcher {
|
||||
let origin_id_str = origin_id.to_string();
|
||||
tokio::spawn(
|
||||
async move {
|
||||
if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await {
|
||||
if let Err(e) = Self::watch_loop(&origin_id, &root, &event_bus, &mut stop_rx).await
|
||||
{
|
||||
error!("Watcher error: {}", e);
|
||||
}
|
||||
}
|
||||
@@ -126,7 +127,10 @@ impl OriginWatcher {
|
||||
}
|
||||
EventKind::Remove(_) => {
|
||||
trace!(origin_id = %origin_id, path = ?relative, "File removed");
|
||||
event_bus.publish(Event::FileRemoved { path: vpath, file_id: None });
|
||||
event_bus.publish(Event::FileRemoved {
|
||||
path: vpath,
|
||||
file_id: None,
|
||||
});
|
||||
}
|
||||
EventKind::Modify(_) => {
|
||||
trace!(origin_id = %origin_id, path = ?relative, "File modified");
|
||||
@@ -186,7 +190,8 @@ mod tests {
|
||||
let event_bus = Arc::new(EventBus::default());
|
||||
let mut rx = event_bus.subscribe();
|
||||
|
||||
let watcher = OriginWatcher::new(OriginId::from("test"), dir.path().to_path_buf(), event_bus);
|
||||
let watcher =
|
||||
OriginWatcher::new(OriginId::from("test"), dir.path().to_path_buf(), event_bus);
|
||||
let handle = watcher.start();
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
@@ -206,6 +211,8 @@ mod tests {
|
||||
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.flac")));
|
||||
assert!(OriginWatcher::is_audio_file(Path::new("/music/song.MP3")));
|
||||
assert!(!OriginWatcher::is_audio_file(Path::new("/music/cover.jpg")));
|
||||
assert!(!OriginWatcher::is_audio_file(Path::new("/music/readme.txt")));
|
||||
assert!(!OriginWatcher::is_audio_file(Path::new(
|
||||
"/music/readme.txt"
|
||||
)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
[package]
|
||||
name = "musicfs-test-utils"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
description = "Test utilities and fixtures for MusicFS resilience testing"
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-origins = { path = "../musicfs-origins" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-cache = { path = "../musicfs-cache" }
|
||||
musicfs-search = { path = "../musicfs-search" }
|
||||
|
||||
async-trait.workspace = true
|
||||
tokio = { workspace = true, features = ["full", "sync", "time"] }
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
parking_lot.workspace = true
|
||||
tempfile.workspace = true
|
||||
bytes.workspace = true
|
||||
|
||||
# Fault injection
|
||||
fail = { version = "0.5", optional = true }
|
||||
rlimit = { version = "0.10", optional = true }
|
||||
nix = { version = "0.29", optional = true, features = ["signal", "process"] }
|
||||
|
||||
# Docker/network tests
|
||||
noxious-client = { version = "1.0", optional = true }
|
||||
reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls"] }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
failpoints = ["fail/failpoints"]
|
||||
process-tests = ["nix"]
|
||||
resource-limits = ["rlimit"]
|
||||
docker-tests = ["noxious-client", "reqwest"]
|
||||
full = ["failpoints", "process-tests", "resource-limits", "docker-tests"]
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = "0.4"
|
||||
tokio-util.workspace = true
|
||||
sd-notify.workspace = true
|
||||
libc.workspace = true
|
||||
@@ -0,0 +1,204 @@
|
||||
use musicfs_cas::CasError;
|
||||
use musicfs_core::Error;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub fn assert_error_contains<T, E: std::fmt::Debug>(result: Result<T, E>, expected_text: &str) {
|
||||
match result {
|
||||
Ok(_) => panic!("Expected error containing '{}', but got Ok", expected_text),
|
||||
Err(e) => {
|
||||
let error_msg = format!("{:?}", e);
|
||||
assert!(
|
||||
error_msg.contains(expected_text),
|
||||
"Expected error containing '{}', but got: {}",
|
||||
expected_text,
|
||||
error_msg
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_io_error<T>(result: Result<T, Error>) {
|
||||
match result {
|
||||
Err(Error::Io(_)) => (),
|
||||
Err(e) => panic!("Expected Io error, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected Io error, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_cas_io_error<T>(result: Result<T, CasError>) {
|
||||
match result {
|
||||
Err(CasError::Io(_)) => (),
|
||||
Err(e) => panic!("Expected CasError::Io, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected CasError::Io, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_cas_not_found<T>(result: Result<T, CasError>) {
|
||||
match result {
|
||||
Err(CasError::NotFound(_)) => (),
|
||||
Err(e) => panic!("Expected CasError::NotFound, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected CasError::NotFound, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_cas_integrity_error<T>(result: Result<T, CasError>) {
|
||||
match result {
|
||||
Err(CasError::IntegrityError { .. }) => (),
|
||||
Err(e) => panic!("Expected CasError::IntegrityError, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected CasError::IntegrityError, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_file_not_found<T>(result: Result<T, Error>) {
|
||||
match result {
|
||||
Err(Error::FileNotFound(_)) => (),
|
||||
Err(e) => panic!("Expected FileNotFound error, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected FileNotFound error, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_origin_error<T>(result: Result<T, Error>) {
|
||||
match result {
|
||||
Err(Error::Origin(_)) => (),
|
||||
Err(e) => panic!("Expected Origin error, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected Origin error, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_timeout_error<T>(result: Result<T, Error>) {
|
||||
match result {
|
||||
Err(Error::Timeout(_)) => (),
|
||||
Err(e) => panic!("Expected Timeout error, got: {:?}", e),
|
||||
Ok(_) => panic!("Expected Timeout error, got Ok"),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TimedAssertion {
|
||||
start: Instant,
|
||||
min_duration: Option<Duration>,
|
||||
max_duration: Option<Duration>,
|
||||
}
|
||||
|
||||
impl TimedAssertion {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
start: Instant::now(),
|
||||
min_duration: None,
|
||||
max_duration: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn expect_at_least(mut self, duration: Duration) -> Self {
|
||||
self.min_duration = Some(duration);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn expect_at_most(mut self, duration: Duration) -> Self {
|
||||
self.max_duration = Some(duration);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn assert_elapsed(self) {
|
||||
let elapsed = self.start.elapsed();
|
||||
|
||||
if let Some(min) = self.min_duration {
|
||||
assert!(
|
||||
elapsed >= min,
|
||||
"Expected at least {:?}, but only {:?} elapsed",
|
||||
min,
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(max) = self.max_duration {
|
||||
assert!(
|
||||
elapsed <= max,
|
||||
"Expected at most {:?}, but {:?} elapsed",
|
||||
max,
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TimedAssertion {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn assert_completes_within<F, T>(future: F, timeout: Duration) -> T
|
||||
where
|
||||
F: std::future::Future<Output = T>,
|
||||
{
|
||||
tokio::time::timeout(timeout, future)
|
||||
.await
|
||||
.expect(&format!("Operation did not complete within {:?}", timeout))
|
||||
}
|
||||
|
||||
pub async fn assert_times_out<F, T>(future: F, timeout: Duration)
|
||||
where
|
||||
F: std::future::Future<Output = T>,
|
||||
{
|
||||
match tokio::time::timeout(timeout, future).await {
|
||||
Ok(_) => panic!("Expected operation to time out, but it completed"),
|
||||
Err(_) => (),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_assert_error_contains() {
|
||||
let result: Result<(), Error> = Err(Error::Origin("connection refused".into()));
|
||||
assert_error_contains(result, "connection");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "Expected error containing")]
|
||||
fn test_assert_error_contains_failure() {
|
||||
let result: Result<(), Error> = Err(Error::Origin("something else".into()));
|
||||
assert_error_contains(result, "connection");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_assert_io_error() {
|
||||
let result: Result<(), Error> = Err(Error::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"test",
|
||||
)));
|
||||
assert_io_error(result);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_timed_assertion_at_least() {
|
||||
let timer = TimedAssertion::new().expect_at_least(Duration::from_millis(10));
|
||||
std::thread::sleep(Duration::from_millis(15));
|
||||
timer.assert_elapsed();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_timed_assertion_at_most() {
|
||||
let timer = TimedAssertion::new().expect_at_most(Duration::from_millis(100));
|
||||
timer.assert_elapsed();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_assert_completes_within() {
|
||||
let result = assert_completes_within(async { 42 }, Duration::from_millis(100)).await;
|
||||
assert_eq!(result, 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_assert_times_out() {
|
||||
assert_times_out(
|
||||
async {
|
||||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||
},
|
||||
Duration::from_millis(10),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
use bytes::Bytes;
|
||||
use musicfs_cas::{CasConfig, CasError, CasStore, DedupStats};
|
||||
use musicfs_core::ChunkHash;
|
||||
use std::io::{self, ErrorKind};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct FaultyCasStore {
|
||||
inner: Arc<CasStore>,
|
||||
inject_enospc: AtomicBool,
|
||||
inject_eio_on_read: AtomicBool,
|
||||
inject_eio_on_write: AtomicBool,
|
||||
inject_corruption: AtomicBool,
|
||||
fail_after_n_puts: AtomicUsize,
|
||||
put_count: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FaultyCasStore {
|
||||
pub fn new(inner: Arc<CasStore>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
inject_enospc: AtomicBool::new(false),
|
||||
inject_eio_on_read: AtomicBool::new(false),
|
||||
inject_eio_on_write: AtomicBool::new(false),
|
||||
inject_corruption: AtomicBool::new(false),
|
||||
fail_after_n_puts: AtomicUsize::new(usize::MAX),
|
||||
put_count: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn open(config: CasConfig) -> Result<Self, CasError> {
|
||||
let store = CasStore::open(config).await?;
|
||||
Ok(Self::new(Arc::new(store)))
|
||||
}
|
||||
|
||||
pub fn set_inject_enospc(&self, enabled: bool) {
|
||||
self.inject_enospc.store(enabled, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_inject_eio_on_read(&self, enabled: bool) {
|
||||
self.inject_eio_on_read.store(enabled, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_inject_eio_on_write(&self, enabled: bool) {
|
||||
self.inject_eio_on_write.store(enabled, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_inject_corruption(&self, enabled: bool) {
|
||||
self.inject_corruption.store(enabled, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_fail_after_n_puts(&self, n: usize) {
|
||||
self.fail_after_n_puts.store(n, Ordering::SeqCst);
|
||||
self.put_count.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn reset_faults(&self) {
|
||||
self.inject_enospc.store(false, Ordering::SeqCst);
|
||||
self.inject_eio_on_read.store(false, Ordering::SeqCst);
|
||||
self.inject_eio_on_write.store(false, Ordering::SeqCst);
|
||||
self.inject_corruption.store(false, Ordering::SeqCst);
|
||||
self.fail_after_n_puts.store(usize::MAX, Ordering::SeqCst);
|
||||
self.put_count.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn put_count(&self) -> usize {
|
||||
self.put_count.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
|
||||
let count = self.put_count.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
if self.inject_enospc.load(Ordering::SeqCst) {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"No space left on device (ENOSPC injected)",
|
||||
)));
|
||||
}
|
||||
|
||||
if self.inject_eio_on_write.load(Ordering::SeqCst) {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"Input/output error (EIO injected)",
|
||||
)));
|
||||
}
|
||||
|
||||
let threshold = self.fail_after_n_puts.load(Ordering::SeqCst);
|
||||
if count >= threshold {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"Injected failure after N puts",
|
||||
)));
|
||||
}
|
||||
|
||||
self.inner.put(data).await
|
||||
}
|
||||
|
||||
pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
|
||||
if self.inject_eio_on_read.load(Ordering::SeqCst) {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"Input/output error (EIO injected)",
|
||||
)));
|
||||
}
|
||||
|
||||
let data = self.inner.get(hash).await?;
|
||||
|
||||
if self.inject_corruption.load(Ordering::SeqCst) {
|
||||
let mut corrupted = data.to_vec();
|
||||
if !corrupted.is_empty() {
|
||||
corrupted[0] = corrupted[0].wrapping_add(1);
|
||||
}
|
||||
return Err(CasError::IntegrityError {
|
||||
expected: hash.as_hex(),
|
||||
actual: ChunkHash::from_bytes(&corrupted).as_hex(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub fn exists(&self, hash: &ChunkHash) -> bool {
|
||||
self.inner.exists(hash)
|
||||
}
|
||||
|
||||
pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
|
||||
if self.inject_eio_on_write.load(Ordering::SeqCst) {
|
||||
return Err(CasError::Io(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"Input/output error (EIO injected)",
|
||||
)));
|
||||
}
|
||||
self.inner.delete(hash).await
|
||||
}
|
||||
|
||||
pub fn current_size(&self) -> u64 {
|
||||
self.inner.current_size()
|
||||
}
|
||||
|
||||
pub fn max_size(&self) -> u64 {
|
||||
self.inner.max_size()
|
||||
}
|
||||
|
||||
pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
|
||||
self.inner.list_chunks()
|
||||
}
|
||||
|
||||
pub fn dedup_stats(&self) -> DedupStats {
|
||||
self.inner.dedup_stats()
|
||||
}
|
||||
|
||||
pub fn inner(&self) -> &Arc<CasStore> {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
async fn test_store() -> (FaultyCasStore, TempDir) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 1024 * 1024,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = FaultyCasStore::open(config).await.unwrap();
|
||||
(store, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_healthy_passthrough() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"test data";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
let retrieved = store.get(&hash).await.unwrap();
|
||||
assert_eq!(&retrieved[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_inject_enospc() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
store.set_inject_enospc(true);
|
||||
let result = store.put(b"test").await;
|
||||
assert!(result.is_err());
|
||||
|
||||
let err = result.unwrap_err();
|
||||
assert!(matches!(err, CasError::Io(_)));
|
||||
|
||||
store.set_inject_enospc(false);
|
||||
assert!(store.put(b"test").await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_inject_eio_on_read() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let hash = store.put(b"test").await.unwrap();
|
||||
|
||||
store.set_inject_eio_on_read(true);
|
||||
let result = store.get(&hash).await;
|
||||
assert!(result.is_err());
|
||||
|
||||
store.set_inject_eio_on_read(false);
|
||||
assert!(store.get(&hash).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_inject_corruption() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let hash = store.put(b"test data").await.unwrap();
|
||||
|
||||
store.set_inject_corruption(true);
|
||||
let result = store.get(&hash).await;
|
||||
assert!(matches!(result, Err(CasError::IntegrityError { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fail_after_n_puts() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
store.set_fail_after_n_puts(2);
|
||||
|
||||
assert!(store.put(b"data1").await.is_ok());
|
||||
assert!(store.put(b"data2").await.is_ok());
|
||||
assert!(store.put(b"data3").await.is_err());
|
||||
assert!(store.put(b"data4").await.is_err());
|
||||
assert_eq!(store.put_count(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_reset_faults() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
store.set_inject_enospc(true);
|
||||
store.set_inject_eio_on_read(true);
|
||||
store.set_fail_after_n_puts(1);
|
||||
|
||||
store.reset_faults();
|
||||
|
||||
assert!(store.put(b"test").await.is_ok());
|
||||
let hash = store.put(b"test2").await.unwrap();
|
||||
assert!(store.get(&hash).await.is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, Error, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use musicfs_origins::{Origin, WatchCallback, WatchHandle};
|
||||
use parking_lot::RwLock;
|
||||
use std::io::{self, ErrorKind};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum FailMode {
|
||||
Healthy,
|
||||
FailEveryNth(usize),
|
||||
FailAfterN(usize),
|
||||
TimeoutMs(u64),
|
||||
PartialRead { max_bytes: usize },
|
||||
ReturnError(ErrorKind),
|
||||
}
|
||||
|
||||
impl Default for FailMode {
|
||||
fn default() -> Self {
|
||||
FailMode::Healthy
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FaultyOrigin {
|
||||
inner: Arc<dyn Origin>,
|
||||
fail_mode: Arc<RwLock<FailMode>>,
|
||||
call_count: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FaultyOrigin {
|
||||
pub fn new(inner: Arc<dyn Origin>, mode: FailMode) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
fail_mode: Arc::new(RwLock::new(mode)),
|
||||
call_count: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn wrap(inner: impl Origin + 'static) -> Self {
|
||||
Self::new(Arc::new(inner), FailMode::Healthy)
|
||||
}
|
||||
|
||||
pub fn set_mode(&self, mode: FailMode) {
|
||||
*self.fail_mode.write() = mode;
|
||||
}
|
||||
|
||||
pub fn call_count(&self) -> usize {
|
||||
self.call_count.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn reset_count(&self) {
|
||||
self.call_count.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn increment_and_check(&self) -> Option<Error> {
|
||||
let count = self.call_count.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
let mode = self.fail_mode.read();
|
||||
|
||||
match *mode {
|
||||
FailMode::Healthy => None,
|
||||
FailMode::FailEveryNth(n) if n > 0 && count % n == 0 => {
|
||||
Some(Error::Origin("Injected failure (every Nth)".into()))
|
||||
}
|
||||
FailMode::FailEveryNth(_) => None,
|
||||
FailMode::FailAfterN(n) if count > n => {
|
||||
Some(Error::Origin("Injected failure (after N)".into()))
|
||||
}
|
||||
FailMode::FailAfterN(_) => None,
|
||||
FailMode::TimeoutMs(_) => None,
|
||||
FailMode::PartialRead { .. } => None,
|
||||
FailMode::ReturnError(kind) => {
|
||||
Some(Error::Io(io::Error::new(kind, "Injected I/O error")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn maybe_timeout(&self) -> Option<Error> {
|
||||
let mode = self.fail_mode.read().clone();
|
||||
if let FailMode::TimeoutMs(ms) = mode {
|
||||
tokio::time::sleep(Duration::from_millis(ms)).await;
|
||||
Some(Error::Timeout("Injected timeout".into()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_if_partial(&self, mut data: Vec<u8>) -> Vec<u8> {
|
||||
let mode = self.fail_mode.read();
|
||||
if let FailMode::PartialRead { max_bytes } = *mode {
|
||||
data.truncate(max_bytes);
|
||||
}
|
||||
data
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for FaultyOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
self.inner.id()
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
self.inner.origin_type()
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
self.inner.display_name()
|
||||
}
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
self.inner.readdir(path).await
|
||||
}
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
self.inner.stat(path).await
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
let data = self.inner.read(path, offset, size).await?;
|
||||
Ok(self.truncate_if_partial(data))
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
let data = self.inner.read_full(path).await?;
|
||||
Ok(self.truncate_if_partial(data))
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
self.inner.exists(path).await
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
let mode = self.fail_mode.read().clone();
|
||||
match mode {
|
||||
FailMode::Healthy => self.inner.health().await,
|
||||
FailMode::ReturnError(_) => HealthStatus::Unhealthy,
|
||||
FailMode::TimeoutMs(ms) => {
|
||||
tokio::time::sleep(Duration::from_millis(ms)).await;
|
||||
HealthStatus::Unhealthy
|
||||
}
|
||||
FailMode::FailAfterN(n) if self.call_count.load(Ordering::SeqCst) >= n => {
|
||||
HealthStatus::Unhealthy
|
||||
}
|
||||
_ => self.inner.health().await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
|
||||
if let Some(err) = self.increment_and_check() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.maybe_timeout().await {
|
||||
return Err(err);
|
||||
}
|
||||
self.inner.open_read(path).await
|
||||
}
|
||||
|
||||
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
|
||||
self.inner.watch(path, callback).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::SystemTime;
|
||||
|
||||
struct MockOrigin {
|
||||
id: OriginId,
|
||||
}
|
||||
|
||||
impl MockOrigin {
|
||||
fn new(id: &str) -> Self {
|
||||
Self {
|
||||
id: OriginId::from(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for MockOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
&self.id
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
OriginType::Local
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
"mock"
|
||||
}
|
||||
|
||||
async fn readdir(&self, _path: &Path) -> Result<Vec<DirEntry>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
async fn stat(&self, _path: &Path) -> Result<FileStat> {
|
||||
Ok(FileStat {
|
||||
size: 1000,
|
||||
mtime: SystemTime::now(),
|
||||
is_dir: false,
|
||||
})
|
||||
}
|
||||
|
||||
async fn read(&self, _path: &Path, _offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
Ok(vec![0u8; size as usize])
|
||||
}
|
||||
|
||||
async fn read_full(&self, _path: &Path) -> Result<Vec<u8>> {
|
||||
Ok(vec![0u8; 100])
|
||||
}
|
||||
|
||||
async fn exists(&self, _path: &Path) -> Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
HealthStatus::Healthy
|
||||
}
|
||||
|
||||
async fn open_read(&self, _path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
|
||||
Err(Error::Origin("Not implemented".into()))
|
||||
}
|
||||
|
||||
async fn watch(&self, _path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
|
||||
Err(Error::Origin("Not implemented".into()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_healthy_passthrough() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
|
||||
|
||||
let result = faulty.stat(Path::new("/test")).await;
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(faulty.call_count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fail_every_nth() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::FailEveryNth(2));
|
||||
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
assert_eq!(faulty.call_count(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fail_after_n() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::FailAfterN(2));
|
||||
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_partial_read() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::PartialRead { max_bytes: 10 });
|
||||
|
||||
let data = faulty.read(Path::new("/test"), 0, 100).await.unwrap();
|
||||
assert_eq!(data.len(), 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mode_change_mid_test() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::ReturnError(ErrorKind::ConnectionRefused));
|
||||
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_err());
|
||||
|
||||
faulty.set_mode(FailMode::Healthy);
|
||||
assert!(faulty.stat(Path::new("/test")).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_health_reflects_mode() {
|
||||
let inner = Arc::new(MockOrigin::new("test"));
|
||||
let faulty = FaultyOrigin::new(inner, FailMode::Healthy);
|
||||
|
||||
assert_eq!(faulty.health().await, HealthStatus::Healthy);
|
||||
|
||||
faulty.set_mode(FailMode::ReturnError(ErrorKind::ConnectionRefused));
|
||||
assert_eq!(faulty.health().await, HealthStatus::Unhealthy);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_cas::{CasConfig, CasStore};
|
||||
use musicfs_core::{AudioFormat, AudioMeta, FileId, FileMeta, OriginId, RealPath, VirtualPath};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::SystemTime;
|
||||
use tempfile::TempDir;
|
||||
|
||||
pub fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from(vpath),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_file_meta_with_origin(id: i64, vpath: &str, size: u64, origin_id: &str) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from(origin_id),
|
||||
path: PathBuf::from(vpath),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_audio_meta(artist: &str, album: &str, title: &str) -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: Some(title.to_string()),
|
||||
artist: Some(artist.to_string()),
|
||||
album: Some(album.to_string()),
|
||||
album_artist: None,
|
||||
genre: None,
|
||||
year: None,
|
||||
track: None,
|
||||
disc: None,
|
||||
duration_ms: Some(180_000),
|
||||
bitrate: Some(320),
|
||||
sample_rate: Some(44100),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_audio_file(
|
||||
id: i64,
|
||||
vpath: &str,
|
||||
size: u64,
|
||||
artist: &str,
|
||||
album: &str,
|
||||
title: &str,
|
||||
) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from(vpath),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: Some(make_audio_meta(artist, album, title)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_audio_file_full(
|
||||
id: i64,
|
||||
vpath: &str,
|
||||
size: u64,
|
||||
artist: &str,
|
||||
album: &str,
|
||||
title: &str,
|
||||
track: u32,
|
||||
year: u32,
|
||||
) -> FileMeta {
|
||||
let mut audio = make_audio_meta(artist, album, title);
|
||||
audio.track = Some(track);
|
||||
audio.year = Some(year);
|
||||
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from(vpath),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: Some(audio),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TestCasStore {
|
||||
pub store: Arc<CasStore>,
|
||||
pub dir: TempDir,
|
||||
}
|
||||
|
||||
pub async fn setup_test_cas() -> TestCasStore {
|
||||
let dir = TempDir::new().expect("Failed to create temp dir for CAS");
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 100 * 1024 * 1024,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = CasStore::open(config)
|
||||
.await
|
||||
.expect("Failed to open CAS store");
|
||||
TestCasStore {
|
||||
store: Arc::new(store),
|
||||
dir,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn setup_test_cas_with_size(max_size: u64) -> TestCasStore {
|
||||
let dir = TempDir::new().expect("Failed to create temp dir for CAS");
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = CasStore::open(config)
|
||||
.await
|
||||
.expect("Failed to open CAS store");
|
||||
TestCasStore {
|
||||
store: Arc::new(store),
|
||||
dir,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn setup_test_tree(files: &[FileMeta]) -> Arc<RwLock<musicfs_cache::VirtualTree>> {
|
||||
let mut builder = TreeBuilder::new();
|
||||
for file in files {
|
||||
builder.add_file(file);
|
||||
}
|
||||
Arc::new(RwLock::new(builder.build()))
|
||||
}
|
||||
|
||||
pub fn create_test_file(dir: &Path, relative_path: &str, content: &[u8]) -> PathBuf {
|
||||
let full_path = dir.join(relative_path);
|
||||
if let Some(parent) = full_path.parent() {
|
||||
std::fs::create_dir_all(parent).expect("Failed to create parent directories");
|
||||
}
|
||||
std::fs::write(&full_path, content).expect("Failed to write test file");
|
||||
full_path
|
||||
}
|
||||
|
||||
pub fn create_test_dir_structure(base: &Path, structure: &[&str]) {
|
||||
for path in structure {
|
||||
let full_path = base.join(path);
|
||||
if path.ends_with('/') {
|
||||
std::fs::create_dir_all(&full_path).expect("Failed to create directory");
|
||||
} else {
|
||||
if let Some(parent) = full_path.parent() {
|
||||
std::fs::create_dir_all(parent).expect("Failed to create parent");
|
||||
}
|
||||
std::fs::write(&full_path, format!("content of {}", path))
|
||||
.expect("Failed to write file");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TestOriginDir {
|
||||
pub dir: TempDir,
|
||||
}
|
||||
|
||||
impl TestOriginDir {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
dir: TempDir::new().expect("Failed to create origin temp dir"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_file(&self, path: &str, content: &[u8]) -> PathBuf {
|
||||
create_test_file(self.dir.path(), path, content)
|
||||
}
|
||||
|
||||
pub fn add_audio_file(&self, path: &str) -> PathBuf {
|
||||
let fake_audio = b"FAKE_FLAC_HEADER_FOR_TESTING_ONLY";
|
||||
self.add_file(path, fake_audio)
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
self.dir.path()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TestOriginDir {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_make_file_meta() {
|
||||
let meta = make_file_meta(1, "/Artist/Album/Track.flac", 1000);
|
||||
assert_eq!(meta.id.0, 1);
|
||||
assert_eq!(meta.virtual_path.as_str(), "/Artist/Album/Track.flac");
|
||||
assert_eq!(meta.size, 1000);
|
||||
assert!(meta.audio.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_make_audio_file() {
|
||||
let meta = make_audio_file(1, "/path.flac", 5000, "Artist", "Album", "Title");
|
||||
assert!(meta.audio.is_some());
|
||||
let audio = meta.audio.unwrap();
|
||||
assert_eq!(audio.artist, Some("Artist".to_string()));
|
||||
assert_eq!(audio.album, Some("Album".to_string()));
|
||||
assert_eq!(audio.title, Some("Title".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_setup_test_cas() {
|
||||
let test_cas = setup_test_cas().await;
|
||||
let hash = test_cas.store.put(b"test data").await.unwrap();
|
||||
assert!(test_cas.store.exists(&hash));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_setup_test_tree() {
|
||||
let files = vec![
|
||||
make_file_meta(1, "/A/B/1.flac", 100),
|
||||
make_file_meta(2, "/A/B/2.flac", 200),
|
||||
];
|
||||
let tree = setup_test_tree(&files);
|
||||
let guard = tree.read().unwrap();
|
||||
assert!(guard.file_count() > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_origin_dir() {
|
||||
let origin = TestOriginDir::new();
|
||||
let path = origin.add_file("artist/album/track.flac", b"content");
|
||||
assert!(path.exists());
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user