Compare commits
39 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1dcfa7a803 | |||
| 78b4338f95 | |||
| 82be15b960 | |||
| 5f79b3c2b0 | |||
| f28091257e | |||
| 8f76dd5b39 | |||
| 8dba0c097d | |||
| d35aa2aecb | |||
| 0e50d8a2a6 | |||
| 9699da385b | |||
| d001e81128 | |||
| 601c466ce4 | |||
| 9eb537dfa4 | |||
| d1e1ac97c4 | |||
| ebf1c5b5e1 | |||
| 80dfe50aaa | |||
| 13f007ebb5 | |||
| 1a98d62357 | |||
| 4e946963e1 | |||
| 0779c0d076 | |||
| 24fd81ff9e | |||
| 53e6dbaef0 | |||
| a8262ac8f8 | |||
| 056d9b8c82 | |||
| d14710c073 | |||
| d49018bfc7 | |||
| 9737e26bc9 | |||
| 4f52549e46 | |||
| 68aba52362 | |||
| a5023a6441 | |||
| 3d1f9a206f | |||
| 536e117576 | |||
| b51c680af4 | |||
| f84061a30d | |||
| 9349d29200 | |||
| 872580602d | |||
| 50cb21b9e5 | |||
| b1f32372f7 | |||
| 83ff042b91 |
@@ -1 +1,8 @@
|
||||
use flake
|
||||
#!/usr/bin/env bash
|
||||
export GIT_CONFIG_GLOBAL=/dev/null
|
||||
|
||||
eval "$(devenv direnvrc)"
|
||||
|
||||
# You can pass flags to the devenv command
|
||||
# For example: use devenv --impure --option services.postgres.enable:bool true
|
||||
use devenv
|
||||
|
||||
+27
@@ -50,3 +50,30 @@ rustc-ice-*.txt
|
||||
dev/
|
||||
|
||||
.sisyphus/
|
||||
|
||||
# Devenv
|
||||
.devenv*
|
||||
devenv.local.nix
|
||||
devenv.local.yaml
|
||||
|
||||
# direnv
|
||||
.direnv
|
||||
|
||||
# pre-commit
|
||||
.pre-commit-config.yaml
|
||||
|
||||
# Devenv
|
||||
.devenv*
|
||||
devenv.local.nix
|
||||
devenv.local.yaml
|
||||
|
||||
# direnv
|
||||
.direnv
|
||||
|
||||
# pre-commit
|
||||
.pre-commit-config.yaml
|
||||
|
||||
|
||||
# Added by cargo
|
||||
|
||||
/target
|
||||
|
||||
@@ -1,295 +0,0 @@
|
||||
# AGENTS.md — beetfs/musicfs
|
||||
|
||||
> Everything an AI agent needs to get work done on this project.
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
cd beetfs/musicfs
|
||||
nix develop # Enter dev shell (ALL tooling provided)
|
||||
cargo check # Verify compilation
|
||||
cargo test # Run all tests (162 tests, ~10s)
|
||||
cargo build --release # Build release binary
|
||||
```
|
||||
|
||||
**No `rustup`, no `apt install`, no manual dependency management.** The Nix flake provides everything: Rust stable + rust-analyzer + clippy + rustfmt, FUSE3, SQLite, OpenSSL, protobuf, grpcurl, cargo-nextest, cargo-criterion, lld linker.
|
||||
|
||||
---
|
||||
|
||||
## Project Overview
|
||||
|
||||
MusicFS is a **read-only FUSE filesystem** that presents music libraries organized by metadata (artist/album/track) instead of physical file paths. It supports multiple storage backends (local, NFS, S3, SFTP), content-addressable caching with delta sync, and full-text search.
|
||||
|
||||
**Key constraint**: Read-only. Never modifies origin files. Never pushes changes to the origin server.
|
||||
|
||||
---
|
||||
|
||||
## Repository Layout
|
||||
|
||||
```
|
||||
beetfs/
|
||||
├── musicfs/ # Rust implementation (active)
|
||||
│ ├── Cargo.toml # Workspace root
|
||||
│ ├── flake.nix # Nix dev shell
|
||||
│ ├── .cargo/config.toml # LLD linker, aliases (t/c/b)
|
||||
│ ├── crates/ # 11 workspace crates
|
||||
│ │ ├── musicfs-cli/ # Binary entry point (clap)
|
||||
│ │ ├── musicfs-core/ # Types, errors, config, events
|
||||
│ │ ├── musicfs-fuse/ # FUSE ops (fuser)
|
||||
│ │ ├── musicfs-metadata/ # Audio parsing (symphonia)
|
||||
│ │ ├── musicfs-cache/ # Cache: tree, metadata, patterns, eviction
|
||||
│ │ ├── musicfs-cas/ # Content-addressable store (sled + chunks)
|
||||
│ │ ├── musicfs-origins/ # Origin backends: local, NFS, SMB, S3, SFTP
|
||||
│ │ ├── musicfs-sync/ # Delta sync, CDC chunking (fastcdc), watcher
|
||||
│ │ ├── musicfs-search/ # Full-text search (tantivy)
|
||||
│ │ ├── musicfs-grpc/ # gRPC control API (tonic + prost)
|
||||
│ │ └── musicfs-plugins/ # Plugin system (native .so + WASM)
|
||||
│ ├── tests/
|
||||
│ │ ├── e2e/e2e_players.rs # E2E: mpv/VLC playback (manual, #[ignore])
|
||||
│ │ └── integration/ # (placeholder)
|
||||
│ └── dist/ # Deployment
|
||||
│ ├── musicfs.service # systemd unit
|
||||
│ ├── config.example.toml # Example config
|
||||
│ ├── logrotate.d/musicfs # Log rotation
|
||||
│ ├── PKGBUILD # Arch package
|
||||
│ └── musicfs.spec # RPM spec
|
||||
├── docs/
|
||||
│ ├── templates/
|
||||
│ │ ├── bluedoc.md # Full design doc (5-20+ pages)
|
||||
│ │ └── greendoc.md # One-pager (1-2 pages)
|
||||
│ ├── v2/
|
||||
│ │ ├── architecture.md # System design (GOLDEN — source of truth)
|
||||
│ │ ├── requirements.md # Functional + non-functional requirements
|
||||
│ │ ├── development-plan.md # Implementation roadmap (weeks 1-14)
|
||||
│ │ └── plans/ # Weekly plans, feature plans, research
|
||||
│ └── v1/ # Original Python beetfs docs (reference only)
|
||||
└── beetsplug/beetFs.py # Original Python implementation (archived)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Build & Test Commands
|
||||
|
||||
```bash
|
||||
# Cargo aliases (.cargo/config.toml)
|
||||
cargo t # cargo test
|
||||
cargo c # cargo check
|
||||
cargo b # cargo build
|
||||
|
||||
# Common workflows
|
||||
cargo check # Fast compile check
|
||||
cargo test # All tests
|
||||
cargo test -p musicfs-core # Single crate
|
||||
cargo clippy # Lint
|
||||
cargo fmt # Format
|
||||
cargo nextest run # Parallel test runner (faster)
|
||||
|
||||
# gRPC
|
||||
cargo build -p musicfs-grpc # Triggers proto codegen via build.rs
|
||||
grpcurl -unix /run/musicfs.sock musicfs.v1.MusicFS/GetStatus
|
||||
|
||||
# Watch mode
|
||||
cargo watch -x 'check' -x 'test'
|
||||
|
||||
# Release
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
**Proto file location**: `crates/musicfs-grpc/proto/musicfs.proto` (codegen via `tonic-build` in `build.rs`)
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
**Read the full design**: [`docs/v2/architecture.md`](docs/v2/architecture.md) — this is the source of truth for all architectural decisions, component design, data schemas, API definitions, and diagrams.
|
||||
|
||||
**Quick orientation** — MusicFS is a workspace of 11 crates. The dependency flow is:
|
||||
|
||||
`musicfs-cli` → `musicfs-fuse` / `musicfs-grpc` / `musicfs-search` → `musicfs-core` → `musicfs-cache` / `musicfs-origins` / `musicfs-metadata` → `musicfs-cas` → `musicfs-sync`
|
||||
|
||||
Key concepts: Virtual Tree (in-memory directory structure from metadata), CAS (content-addressable chunk storage with sled index), Origin Federation (multi-backend with failover and health monitoring), CDC (content-defined chunking for delta sync), Event Bus (tokio broadcast for cross-component notifications).
|
||||
|
||||
For performance targets, scalability requirements, and quantitative constraints, see [`docs/v2/requirements.md`](docs/v2/requirements.md).
|
||||
|
||||
---
|
||||
|
||||
## Code Conventions
|
||||
|
||||
### Rust
|
||||
|
||||
- **Edition**: 2021, **MSRV**: 1.75+
|
||||
- **Linker**: LLD via clang (configured in `.cargo/config.toml`)
|
||||
- **Error handling**: `thiserror` for library errors, `anyhow` for CLI
|
||||
- **Async**: `tokio` runtime, `async-trait` for trait objects
|
||||
- **Concurrency**: `parking_lot` for hot-path locks, `dashmap` for concurrent maps, `std::sync::RwLock` elsewhere
|
||||
- **Logging**: `tracing` with structured fields (`#[instrument]`, `info!`, `debug!`, etc.)
|
||||
- **Serialization**: `serde` + `toml` for config, `rmp-serde` (msgpack) for binary data, `prost` for protobuf
|
||||
|
||||
### Never Do
|
||||
|
||||
- `as any`, `@ts-ignore` equivalents — no `unsafe` without justification
|
||||
- Empty `catch` / `let _ = result` on operations that can fail meaningfully
|
||||
- Suppress type errors
|
||||
- Commit secrets or credentials
|
||||
|
||||
### Testing Patterns
|
||||
|
||||
- **Fixtures**: `TempDir::new().unwrap()` for isolated storage (used in 29 files)
|
||||
- **In-memory DB**: `Database::open_memory()` for fast SQLite tests
|
||||
- **No mocking framework** — tests use real implementations with temp directories
|
||||
- **Async tests**: `#[tokio::test]`
|
||||
- **Helper functions**: `make_file_meta()`, `mock_health()` — currently duplicated per module
|
||||
|
||||
---
|
||||
|
||||
## Golden Documents
|
||||
|
||||
These are the authoritative references. All implementations must match them.
|
||||
|
||||
| Document | Path | Role |
|
||||
|----------|------|------|
|
||||
| **Architecture** | `docs/v2/architecture.md` | System design — THE source of truth |
|
||||
| **Requirements** | `docs/v2/requirements.md` | What to build (FR-*, NFR-*) |
|
||||
| **Development Plan** | `docs/v2/development-plan.md` | How to build it (week-by-week) |
|
||||
| **Proto Definition** | `crates/musicfs-grpc/proto/musicfs.proto` | API contract |
|
||||
|
||||
If code contradicts architecture.md, the architecture doc wins (unless explicitly superseded by a newer plan document).
|
||||
|
||||
---
|
||||
|
||||
## Documentation Rules
|
||||
|
||||
### Templates
|
||||
|
||||
Two templates exist in `docs/templates/`:
|
||||
|
||||
| Template | When to Use | Length | Review Level |
|
||||
|----------|-------------|--------|-------------|
|
||||
| **BlueDoc** | New systems, major architecture changes, new services | 5-20+ pages | Cross-functional |
|
||||
| **GreenDoc** | Bug fixes, small features, optimizations, config changes | 1-2 pages | Peer review |
|
||||
|
||||
**Decision rule**: If any GreenDoc section needs more than 3 paragraphs, upgrade to a BlueDoc.
|
||||
|
||||
### When Neither Template Fits
|
||||
|
||||
If the work doesn't fit BlueDoc or GreenDoc (e.g., research summaries, audit reports, testing strategies, runbooks):
|
||||
1. **Stop** — do not force-fit content into wrong template
|
||||
2. **Propose** a new template format to the user with: name, intended use case, suggested structure
|
||||
3. **Get approval** before writing the document
|
||||
4. Save approved template to `docs/templates/{name}.md` for future use
|
||||
|
||||
### Document Metadata
|
||||
|
||||
Every document MUST have at the top:
|
||||
|
||||
```markdown
|
||||
**Date**: YYYY-MM-DD
|
||||
**Status**: [Draft / In-Review / Approved / Shipped / Obsolete]
|
||||
**Prerequisites**: [links to dependent docs]
|
||||
```
|
||||
|
||||
BlueDoc additionally requires: Authors, Reviewers, Approvers.
|
||||
|
||||
### Writing Conventions
|
||||
|
||||
- **Tables**: Use for requirements mapping, deliverables tracking, comparisons
|
||||
- **Code blocks**: Include for implementation examples, config samples, commands
|
||||
- **Checklists**: `[ ]` for exit criteria and success metrics
|
||||
- **Section numbering**: Hierarchical (1., 1.1, 1.2)
|
||||
- **Cross-references**: Relative markdown links (`[architecture](../architecture.md)`)
|
||||
- **Requirement tracing**: Reference FR-X.Y / NFR-X.Y from requirements.md
|
||||
- **Diagrams**: PlantUML or Mermaid (architecture.md uses PlantUML)
|
||||
|
||||
### Post-Task Documentation Check (MANDATORY)
|
||||
|
||||
After completing any non-trivial task, check whether documentation needs updating:
|
||||
|
||||
1. **Did the architecture change?** (new component, changed data flow, new dependency) → Update `docs/v2/architecture.md`
|
||||
2. **Did requirements change?** (new constraint, relaxed target, dropped feature) → Update `docs/v2/requirements.md`
|
||||
3. **Did the API change?** (new RPC, changed message, removed endpoint) → Update `proto/musicfs.proto` + `docs/api/`
|
||||
4. **Did build/tooling change?** (new dependency, changed Nix flake, new cargo feature) → Update this `AGENTS.md`
|
||||
5. **Did a plan get completed or invalidated?** → Update the plan's **Status** field (Draft → Shipped / Obsolete)
|
||||
6. **Did the config format change?** → Update `dist/config.example.toml`
|
||||
|
||||
If documentation is out of date, **fix it in the same commit** as the code change. Do not leave it for later.
|
||||
|
||||
### File Naming
|
||||
|
||||
```
|
||||
docs/v2/plans/week-NN-{feature}.md # Weekly implementation plans
|
||||
docs/v2/plans/{feature}-{type}.md # Feature plans, research, proposals
|
||||
docs/v2/{topic}.md # Top-level docs (architecture, requirements)
|
||||
docs/templates/{name}.md # Document templates
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Current State & Known Issues
|
||||
|
||||
### What's Implemented (Weeks 1-11)
|
||||
|
||||
- FUSE filesystem with local origin
|
||||
- Metadata extraction (symphonia: FLAC, MP3, AAC, OGG, Opus)
|
||||
- Virtual tree with configurable path templates
|
||||
- CAS with CDC chunking and deduplication
|
||||
- Multi-origin federation with failover and health monitoring
|
||||
- NFS/SMB origin wrappers with retry logic
|
||||
- Full-text search (tantivy) with `.search/` virtual directory
|
||||
- Smart collections, artwork caching, predictive prefetch
|
||||
- Plugin system (native + WASM)
|
||||
- gRPC control API with event streaming
|
||||
- Comprehensive tracing/logging with journald integration
|
||||
|
||||
### Critical Open Issues
|
||||
|
||||
Detailed in `docs/v2/plans/resilience-fault-tolerance.md` and `docs/v2/plans/persistent-state.md`:
|
||||
|
||||
1. **No persistent state on mount** — every restart does full origin scan (O(N) instead of O(1)). SQLite, tantivy, and manifests persist on disk but are never loaded.
|
||||
2. **No signal handling** — SIGTERM kills the daemon instantly, no graceful shutdown
|
||||
3. **No crash recovery** — corrupted cache = crash on startup, no repair
|
||||
4. **FUSE↔tokio deadlock risk** — `block_on()` in sync FUSE callback can hang under load
|
||||
5. **Fire-and-forget tasks** — background tasks (health monitor, watcher, indexer) not supervised
|
||||
6. **RwLock poison** — single panic in a writer kills all FUSE operations
|
||||
|
||||
### S3/SFTP Origins
|
||||
|
||||
`s3.rs` and `sftp.rs` are **feature-gated stubs** (not implemented). The `Origin` trait and failover infrastructure work, but only `local`, `nfs`, and `smb` origins have real implementations.
|
||||
|
||||
---
|
||||
|
||||
## Running the Filesystem
|
||||
|
||||
```bash
|
||||
# Development
|
||||
nix develop
|
||||
cargo build
|
||||
./target/debug/musicfs mount /mnt/music --origin /path/to/music
|
||||
|
||||
# Production (systemd)
|
||||
sudo cp dist/musicfs.service /etc/systemd/system/
|
||||
sudo systemctl enable --now musicfs
|
||||
|
||||
# E2E tests (requires mounted filesystem)
|
||||
MUSICFS_TEST_MOUNT=/mnt/music cargo test --test e2e_players -- --ignored
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Key Dependencies
|
||||
|
||||
| Crate | Version | Purpose |
|
||||
|-------|---------|---------|
|
||||
| `fuser` | 0.14 | FUSE interface |
|
||||
| `tokio` | 1.x | Async runtime (full features) |
|
||||
| `rusqlite` | 0.31 | SQLite (bundled) |
|
||||
| `sled` | 0.34 | Embedded KV (CAS chunk index) |
|
||||
| `tantivy` | 0.22 | Full-text search |
|
||||
| `symphonia` | 0.5 | Audio metadata extraction |
|
||||
| `fastcdc` | 3.x | Content-defined chunking |
|
||||
| `tonic` | 0.11 | gRPC server |
|
||||
| `tracing` | 0.1 | Structured logging |
|
||||
| `clap` | 4.x | CLI argument parsing |
|
||||
| `parking_lot` | 0.12 | Fast locks |
|
||||
| `dashmap` | 5.x | Concurrent HashMap |
|
||||
@@ -1,674 +0,0 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU General Public License is a free, copyleft license for
|
||||
software and other kinds of works.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
the GNU General Public License is intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users. We, the Free Software Foundation, use the
|
||||
GNU General Public License for most of our software; it applies also to
|
||||
any other work released this way by its authors. You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to prevent others from denying you
|
||||
these rights or asking you to surrender the rights. Therefore, you have
|
||||
certain responsibilities if you distribute copies of the software, or if
|
||||
you modify it: responsibilities to respect the freedom of others.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must pass on to the recipients the same
|
||||
freedoms that you received. You must make sure that they, too, receive
|
||||
or can get the source code. And you must show them these terms so they
|
||||
know their rights.
|
||||
|
||||
Developers that use the GNU GPL protect your rights with two steps:
|
||||
(1) assert copyright on the software, and (2) offer you this License
|
||||
giving you legal permission to copy, distribute and/or modify it.
|
||||
|
||||
For the developers' and authors' protection, the GPL clearly explains
|
||||
that there is no warranty for this free software. For both users' and
|
||||
authors' sake, the GPL requires that modified versions be marked as
|
||||
changed, so that their problems will not be attributed erroneously to
|
||||
authors of previous versions.
|
||||
|
||||
Some devices are designed to deny users access to install or run
|
||||
modified versions of the software inside them, although the manufacturer
|
||||
can do so. This is fundamentally incompatible with the aim of
|
||||
protecting users' freedom to change the software. The systematic
|
||||
pattern of such abuse occurs in the area of products for individuals to
|
||||
use, which is precisely where it is most unacceptable. Therefore, we
|
||||
have designed this version of the GPL to prohibit the practice for those
|
||||
products. If such problems arise substantially in other domains, we
|
||||
stand ready to extend this provision to those domains in future versions
|
||||
of the GPL, as needed to protect the freedom of users.
|
||||
|
||||
Finally, every program is threatened constantly by software patents.
|
||||
States should not allow patents to restrict development and use of
|
||||
software on general-purpose computers, but in those that do, we wish to
|
||||
avoid the special danger that patents applied to a free program could
|
||||
make it effectively proprietary. To prevent this, the GPL assures that
|
||||
patents cannot be used to render the program non-free.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Use with the GNU Affero General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU Affero General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the special requirements of the GNU Affero General Public License,
|
||||
section 13, concerning interaction through a network will apply to the
|
||||
combination as such.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program does terminal interaction, make it output a short
|
||||
notice like this when it starts in an interactive mode:
|
||||
|
||||
<program> Copyright (C) <year> <name of author>
|
||||
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, your program's commands
|
||||
might be different; for a GUI interface, you would use an "about box".
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU GPL, see
|
||||
<http://www.gnu.org/licenses/>.
|
||||
|
||||
The GNU General Public License does not permit incorporating your program
|
||||
into proprietary programs. If your program is a subroutine library, you
|
||||
may consider it more useful to permit linking proprietary applications with
|
||||
the library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License. But first, please read
|
||||
<http://www.gnu.org/philosophy/why-not-lgpl.html>.
|
||||
Generated
+1525
-2727
File diff suppressed because it is too large
Load Diff
+36
-80
@@ -1,95 +1,51 @@
|
||||
[workspace]
|
||||
members = [
|
||||
"crates/musicfs-proto",
|
||||
"crates/musicfs-core",
|
||||
"crates/musicfs-server",
|
||||
"crates/musicfs-client",
|
||||
]
|
||||
resolver = "2"
|
||||
members = ["crates/*"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "MIT OR Apache-2.0"
|
||||
rust-version = "1.75"
|
||||
authors = ["MusicFS Contributors"]
|
||||
repository = "https://github.com/user/musicfs"
|
||||
edition = "2024"
|
||||
|
||||
[workspace.dependencies]
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-util = { version = "0.7", features = ["rt"] }
|
||||
async-trait = "0.1"
|
||||
futures = "0.3"
|
||||
musicfs-proto = { path = "crates/musicfs-proto" }
|
||||
musicfs-core = { path = "crates/musicfs-core" }
|
||||
|
||||
# Error handling
|
||||
thiserror = "1"
|
||||
anyhow = "1"
|
||||
prost = "0.14"
|
||||
tonic = "0.14"
|
||||
tonic-prost = "0.14"
|
||||
tonic-health = "0.14"
|
||||
tonic-reflection = "0.14"
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
rmp-serde = "1"
|
||||
toml = "0.8"
|
||||
|
||||
# Concurrent collections
|
||||
dashmap = "5"
|
||||
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
||||
tracing-appender = "0.2"
|
||||
tracing-journald = "0.3"
|
||||
|
||||
# FUSE
|
||||
fuser = "0.14"
|
||||
|
||||
# Database
|
||||
rusqlite = { version = "0.31", features = ["bundled"] }
|
||||
sled = "0.34"
|
||||
|
||||
# Hashing (per architecture 8.3)
|
||||
xxhash-rust = { version = "0.8", features = ["xxh64"] }
|
||||
hex = "0.4"
|
||||
|
||||
# Audio metadata
|
||||
symphonia = { version = "0.5", default-features = false, features = [
|
||||
"aac", "alac", "flac", "mp3", "ogg", "vorbis", "wav"
|
||||
"aac", "alac", "flac", "mp3", "ogg", "vorbis", "wav",
|
||||
] }
|
||||
twox-hash = "2.1.2"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
tracing-appender = "0.2.5"
|
||||
|
||||
# Bytes handling
|
||||
bytes = "1"
|
||||
|
||||
# Platform directories
|
||||
dirs = "5"
|
||||
|
||||
# CLI
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
||||
# Testing
|
||||
tempfile = "3"
|
||||
fail = "0.5"
|
||||
rlimit = "0.10"
|
||||
nix = { version = "0.29", features = ["signal", "process"] }
|
||||
wiremock = "0.6"
|
||||
assert_cmd = "2.0"
|
||||
noxious-client = "1.0"
|
||||
|
||||
# Platform-specific
|
||||
libc = "0.2"
|
||||
|
||||
# Search (Week 8)
|
||||
tantivy = "0.22"
|
||||
moka = { version = "0.12", features = ["sync"] }
|
||||
|
||||
# Concurrency
|
||||
parking_lot = "0.12"
|
||||
|
||||
# gRPC (Week 8)
|
||||
tonic = "0.11"
|
||||
prost = "0.12"
|
||||
tokio = { version = "1", features = [
|
||||
"macros", "rt-multi-thread", "signal", "fs", "io-util", "sync",
|
||||
] }
|
||||
anyhow = "1"
|
||||
async-trait = "0.1"
|
||||
clap = { version = "4.6.1", features = ["derive"] }
|
||||
notify = "8.2.0"
|
||||
tokio-stream = "0.1"
|
||||
|
||||
# Smart Features (Week 9)
|
||||
image = { version = "0.24", default-features = false, features = ["jpeg", "png"] }
|
||||
chrono = "0.4"
|
||||
fuser = "0.17.0"
|
||||
libc = "0.2.186"
|
||||
sea-orm = { version = "2.0.0-rc", features = [
|
||||
"sqlx-postgres", "runtime-tokio", "macros",
|
||||
] }
|
||||
log = "0.4"
|
||||
bytes = "1"
|
||||
http = "1"
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
|
||||
sd-notify = "0.4"
|
||||
|
||||
[workspace.dependencies.tonic-build]
|
||||
version = "0.11"
|
||||
tempfile = "3"
|
||||
|
||||
@@ -1,879 +0,0 @@
|
||||
# MusicFS
|
||||
|
||||
> A read-only FUSE filesystem that presents your music library organized by metadata — artist, album, track — regardless of how files are stored on disk.
|
||||
|
||||
Browse `/Artist/Album/Track.flac` in any media player or file manager. Original files are never touched.
|
||||
|
||||
---
|
||||
|
||||
## What It Does
|
||||
|
||||
MusicFS mounts as a virtual filesystem. Point it at your music storage (local drive, NFS share, S3 bucket, SFTP server) and it exposes a clean metadata-based directory tree:
|
||||
|
||||
```
|
||||
/mnt/music/
|
||||
├── Metallica/
|
||||
│ └── 72 Seasons (2023) [FLAC]/
|
||||
│ ├── 01 - 72 Seasons.flac
|
||||
│ ├── 02 - Shadows Follow.flac
|
||||
│ └── cover.jpg
|
||||
├── Pink Floyd/
|
||||
│ └── The Wall (1979) [FLAC]/
|
||||
│ ├── 01 - In the Flesh?.flac
|
||||
│ └── ...
|
||||
└── .search/
|
||||
└── (full-text search — see Search section)
|
||||
```
|
||||
|
||||
Files are read directly from origin storage with local chunk caching. Once cached, playback works entirely offline. Write operations return `EROFS` — origin files are always safe.
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
| Feature | Details |
|
||||
|---------|---------|
|
||||
| **Instant mount** | O(1) regardless of library size (<500ms) |
|
||||
| **Metadata-organized paths** | Configurable path templates via `$artist`, `$album`, `$year`, etc. |
|
||||
| **Multi-origin federation** | Local, NFS, SMB, S3, SFTP — automatic failover by priority |
|
||||
| **Content-addressable cache** | Chunk-level deduplication, LRU eviction, delta sync (>90% bandwidth savings) |
|
||||
| **Full-text search** | `/.search/metallica/` returns instant results across 1M+ tracks |
|
||||
| **Metadata overlay** | Set/override tags in the virtual layer without modifying originals |
|
||||
| **Album art** | Virtual `cover.jpg` per album, extracted from embedded tags |
|
||||
| **Plugin system** | Native `.so` and WASM plugins for custom origins, formats, metadata sources |
|
||||
| **gRPC control API** | Cache stats, origin health, live event streaming, metadata management |
|
||||
| **systemd integration** | `sd_notify` ready, journald logging, clean SIGTERM handling |
|
||||
|
||||
**Supported formats:** FLAC, MP3, OGG, WAV, M4A, AAC, Opus
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Enter dev environment (provides Rust, FUSE3, SQLite, everything)
|
||||
nix develop
|
||||
|
||||
# 2. Build
|
||||
cargo build
|
||||
|
||||
# 3. Mount your music library
|
||||
./target/debug/musicfs mount /mnt/music --origin /path/to/your/music
|
||||
|
||||
# 4. Browse
|
||||
ls /mnt/music
|
||||
mpv /mnt/music/Artist/Album/01\ -\ Track.flac
|
||||
|
||||
# 5. Unmount
|
||||
fusermount -u /mnt/music
|
||||
```
|
||||
|
||||
No `rustup`, no `apt install`. The Nix flake provides the full toolchain.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
### From Nix (recommended)
|
||||
|
||||
```bash
|
||||
# Development shell — everything you need
|
||||
nix develop
|
||||
|
||||
# Or install the binary into your profile
|
||||
nix profile install .#musicfs
|
||||
```
|
||||
|
||||
### From Source
|
||||
|
||||
**Prerequisites (non-Nix):**
|
||||
- Rust 1.75+
|
||||
- `libfuse3-dev` / `fuse3` (package name varies by distro)
|
||||
- `libsqlite3-dev`
|
||||
- `libssl-dev`
|
||||
- `protobuf-compiler` (for gRPC)
|
||||
- `clang` + `lld`
|
||||
|
||||
```bash
|
||||
git clone https://github.com/user/musicfs
|
||||
cd musicfs/musicfs
|
||||
cargo build --release
|
||||
sudo cp target/release/musicfs /usr/local/bin/
|
||||
```
|
||||
|
||||
### System Requirements
|
||||
|
||||
| Resource | Minimum | Recommended |
|
||||
|----------|---------|-------------|
|
||||
| CPU | 1 core | 4 cores |
|
||||
| RAM | 256 MB | 2 GB |
|
||||
| Disk (cache) | 1 GB | 50 GB |
|
||||
| Linux kernel | 4.x+ | 5.x+ |
|
||||
| FUSE module | required | — |
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
MusicFS can be configured via file (`--config`), CLI flags, or environment variables (`RUST_LOG` for log level).
|
||||
|
||||
### Minimal Config
|
||||
|
||||
```toml
|
||||
mount_point = "/mnt/music"
|
||||
cache_dir = "/home/user/.cache/musicfs"
|
||||
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/mnt/nas/music"
|
||||
```
|
||||
|
||||
```bash
|
||||
musicfs mount --config /etc/musicfs/config.toml
|
||||
```
|
||||
|
||||
### Full Config Reference
|
||||
|
||||
<!-- embedme config.example.toml -->
|
||||
```toml
|
||||
# MusicFS Configuration
|
||||
# Copy to /etc/musicfs/config.toml or ~/.config/musicfs/config.toml
|
||||
|
||||
# Required: where to mount the virtual filesystem
|
||||
mount_point = "/mnt/music"
|
||||
|
||||
# Required: directory for cache data (CAS chunks, metadata, search index)
|
||||
cache_dir = "/var/cache/musicfs"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Origins - music sources (at least one required)
|
||||
# Supported types: local, nfs, smb, s3, sftp
|
||||
# Lower priority number = preferred source for failover
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[[origins]]
|
||||
id = "local-music"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
enabled = true
|
||||
path = "/home/user/Music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas-nfs"
|
||||
origin_type = "nfs"
|
||||
priority = 2
|
||||
enabled = true
|
||||
path = "/mnt/nas/music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas-smb"
|
||||
origin_type = "smb"
|
||||
priority = 3
|
||||
enabled = false
|
||||
path = "/mnt/smb/music"
|
||||
|
||||
[[origins]]
|
||||
id = "cloud-backup"
|
||||
origin_type = "s3"
|
||||
priority = 10
|
||||
enabled = false
|
||||
bucket = "my-music-backup"
|
||||
region = "us-east-1"
|
||||
|
||||
[[origins]]
|
||||
id = "remote-server"
|
||||
origin_type = "sftp"
|
||||
priority = 10
|
||||
enabled = false
|
||||
host = "music.example.com"
|
||||
port = 22
|
||||
user = "musicfs"
|
||||
path = "/srv/music"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Cache settings
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[cache]
|
||||
# In-memory metadata cache size (artist/album/track info)
|
||||
metadata_cache_mb = 100
|
||||
|
||||
# On-disk content cache size (audio chunks)
|
||||
content_cache_gb = 10
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Health monitoring for origin failover
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[health]
|
||||
# How often to check origin health
|
||||
check_interval_secs = 30
|
||||
|
||||
# Timeout for health check probes
|
||||
timeout_ms = 5000
|
||||
|
||||
# Consecutive failures before marking origin unhealthy
|
||||
unhealthy_threshold = 3
|
||||
|
||||
# Per-origin type thresholds (overrides unhealthy_threshold)
|
||||
[health.per_origin_thresholds]
|
||||
local = 1
|
||||
nfs = 3
|
||||
smb = 3
|
||||
s3 = 3
|
||||
sftp = 3
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[logging]
|
||||
# Directory for log files
|
||||
log_dir = "/var/log/musicfs"
|
||||
|
||||
# Output logs as JSON (for log aggregators)
|
||||
json_output = false
|
||||
|
||||
# Send logs to systemd journal
|
||||
journald = true
|
||||
|
||||
# Log level filter (tracing format)
|
||||
# Examples: "info", "debug", "musicfs=debug,warn", "musicfs_fuse=trace"
|
||||
level = "musicfs=info,warn"
|
||||
|
||||
# Trace sampling rate for performance tracing (0.0 to 1.0)
|
||||
trace_sample_rate = 1.0
|
||||
```
|
||||
|
||||
### Cache Layout on Disk
|
||||
|
||||
```
|
||||
~/.cache/musicfs/
|
||||
├── musicfs.db # SQLite: file metadata, virtual tree, overlay data
|
||||
├── musicfs.lock # Single-instance lock
|
||||
├── musicfs.pid # Daemon PID
|
||||
├── chunks/ # Content-addressable chunk files
|
||||
│ ├── aa/ # 256 subdirs (first 2 hex chars of hash)
|
||||
│ │ └── aa1b2c… # 64 KB average chunk
|
||||
│ └── ...
|
||||
├── search.idx/ # Tantivy full-text search index
|
||||
└── chunks.sled/ # Sled KV: content hash → chunk location
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
musicfs [OPTIONS] <COMMAND>
|
||||
|
||||
OPTIONS:
|
||||
-l, --log-level <LEVEL> Log verbosity [default: info]
|
||||
```
|
||||
|
||||
### `mount` — Start the filesystem
|
||||
|
||||
```bash
|
||||
# From CLI flags (quick start)
|
||||
musicfs mount /mnt/music --origin /path/to/music
|
||||
|
||||
# From config file
|
||||
musicfs mount --config /etc/musicfs/config.toml
|
||||
|
||||
# All flags
|
||||
musicfs mount [MOUNTPOINT] \
|
||||
--config <path> # Config file (overrides flags)
|
||||
--origin <path> # Source music directory
|
||||
--cache-dir <path> # Cache location [default: ~/.cache/musicfs]
|
||||
--grpc-port <port> # gRPC server port [default: 50052]
|
||||
```
|
||||
|
||||
### `status` — Daemon status
|
||||
|
||||
```bash
|
||||
musicfs status
|
||||
```
|
||||
|
||||
### `cache` — Cache management
|
||||
|
||||
```bash
|
||||
musicfs cache stats # Hit rate, size, dedup ratio
|
||||
musicfs cache clear # Clear all caches
|
||||
musicfs cache clear <origin-id> # Clear cache for one origin
|
||||
musicfs cache prefetch <path> [path…] # Pre-warm cache for paths
|
||||
```
|
||||
|
||||
### `search` — Full-text search
|
||||
|
||||
```bash
|
||||
musicfs search "metallica" # Search across all metadata
|
||||
musicfs search "dark side" --limit 20 # Limit results [default: 100]
|
||||
```
|
||||
|
||||
Search results are also browsable as a virtual directory (see [Search](#search)).
|
||||
|
||||
### `origin` — Origin management
|
||||
|
||||
```bash
|
||||
musicfs origin list # List all configured origins
|
||||
musicfs origin health <id> # Check health of one origin
|
||||
musicfs origin rescan <id> # Force re-scan and re-index
|
||||
```
|
||||
|
||||
### `metadata` — Metadata overlay
|
||||
|
||||
```bash
|
||||
# Requires running daemon
|
||||
musicfs metadata get "/Artist/Album/01 - Track.flac"
|
||||
musicfs metadata get "/Artist/Album/01 - Track.flac" --field artist
|
||||
|
||||
musicfs metadata set "/Artist/Album/01 - Track.flac" \
|
||||
--title "New Title" \
|
||||
--artist "New Artist" \
|
||||
--album "New Album" \
|
||||
--track 1 \
|
||||
--genre "Rock" \
|
||||
--date "2023"
|
||||
|
||||
# Set from JSON
|
||||
musicfs metadata set "/path/to/file.flac" --json '{"title":"foo","year":2023}'
|
||||
|
||||
# Show current (overlaid) metadata
|
||||
musicfs metadata diff "/path/to/file.flac"
|
||||
|
||||
# Revert overlay — restore original metadata
|
||||
musicfs metadata clear "/path/to/file.flac"
|
||||
|
||||
# Bulk import/export
|
||||
musicfs metadata import library.csv
|
||||
musicfs metadata import library.json
|
||||
musicfs metadata export --output library.json
|
||||
musicfs metadata export --output library.csv --query "artist:Metallica"
|
||||
```
|
||||
|
||||
> **Note:** `--endpoint` flag (default `http://[::1]:50051`) selects the gRPC server.
|
||||
|
||||
### `trash` — Deleted file recovery
|
||||
|
||||
When files disappear from the origin, MusicFS moves them to a virtual trash rather than removing them immediately.
|
||||
|
||||
```bash
|
||||
musicfs trash list --config /etc/musicfs/config.toml
|
||||
musicfs trash list --since 7d # Deleted in last 7 days
|
||||
musicfs trash list --origin local # Filter by origin
|
||||
musicfs trash list --path "/Metallica" # Filter by path prefix
|
||||
|
||||
musicfs trash restore "/Metallica/72 Seasons" # Restore folder
|
||||
musicfs trash restore --all # Restore everything
|
||||
|
||||
musicfs trash empty --older-than 30d # Permanently delete old entries
|
||||
musicfs trash empty --pattern "/Unknown*" # Delete by pattern
|
||||
```
|
||||
|
||||
### `events` — Live event stream
|
||||
|
||||
```bash
|
||||
musicfs events # All events
|
||||
musicfs events --type file_added # Filter by type
|
||||
# Event types: file_added, file_removed, file_modified,
|
||||
# origin_connected, origin_disconnected,
|
||||
# sync_started, sync_completed, cache_eviction
|
||||
```
|
||||
|
||||
### `shutdown` — Stop the daemon
|
||||
|
||||
```bash
|
||||
musicfs shutdown # Graceful (drain in-flight ops)
|
||||
musicfs shutdown --graceful false # Immediate
|
||||
musicfs shutdown --timeout 60 # Max drain timeout seconds
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Storage Origins
|
||||
|
||||
### Local Filesystem
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/mnt/nas/music"
|
||||
```
|
||||
|
||||
Changes detected via `inotify`. Zero-latency access.
|
||||
|
||||
### NFS
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "nfs"
|
||||
origin_type = "nfs"
|
||||
priority = 2
|
||||
host = "nas.local"
|
||||
export = "/exports/music"
|
||||
```
|
||||
|
||||
### SMB / CIFS
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "smb"
|
||||
origin_type = "smb"
|
||||
priority = 3
|
||||
host = "nas.local"
|
||||
share = "music"
|
||||
```
|
||||
|
||||
### S3 (stub — not yet functional)
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "s3"
|
||||
origin_type = "s3"
|
||||
priority = 4
|
||||
bucket = "my-music"
|
||||
region = "us-east-1"
|
||||
# Credentials via AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars
|
||||
```
|
||||
|
||||
### SFTP (stub — not yet functional)
|
||||
|
||||
```toml
|
||||
[[origins]]
|
||||
id = "sftp"
|
||||
origin_type = "sftp"
|
||||
priority = 4
|
||||
host = "server.example.com"
|
||||
port = 22
|
||||
username = "alice"
|
||||
# Auth via SSH agent or key file — never store passwords in config
|
||||
```
|
||||
|
||||
### Multi-Origin Failover
|
||||
|
||||
Multiple origins are federates into a single virtual tree. MusicFS selects origins by priority, falling back automatically when one becomes unhealthy. Health is polled every `check_interval_secs` (default: 30s). When all origins for a file are unavailable, cached data is served seamlessly.
|
||||
|
||||
---
|
||||
|
||||
## Virtual Filesystem Layout
|
||||
|
||||
### Path Templates
|
||||
|
||||
The virtual path for each file is built from its audio metadata using a configurable template. Variables are sanitized (no `/`, `\`, `:`).
|
||||
|
||||
**Default template:**
|
||||
```
|
||||
$artist/$album ($year) [$format_upper]/$track - $title.$format
|
||||
```
|
||||
|
||||
**Template variables:**
|
||||
|
||||
| Variable | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `$artist` | Track artist | `Metallica` |
|
||||
| `$album` | Album name | `72 Seasons` |
|
||||
| `$title` | Track title | `Lux Æterna` |
|
||||
| `$track` | Track number (zero-padded) | `03` |
|
||||
| `$disc` | Disc number | `1` |
|
||||
| `$year` | Release year | `2023` |
|
||||
| `$genre` | Genre | `Metal` |
|
||||
| `$format` | File extension (lowercase) | `flac` |
|
||||
| `$format_upper` | File extension (uppercase) | `FLAC` |
|
||||
|
||||
Files with missing metadata fall back to `Unknown Artist/Unknown Album/filename`.
|
||||
|
||||
### Album Art
|
||||
|
||||
Each album directory includes a virtual `cover.jpg` extracted from the embedded tags of the first track. No files are written to disk by MusicFS — the image is synthesized on read.
|
||||
|
||||
### Search
|
||||
|
||||
The `/.search/` virtual directory exposes full-text search as filesystem paths:
|
||||
|
||||
```bash
|
||||
# Search via filesystem — use the query as a directory name
|
||||
ls "/mnt/music/.search/dark side of the moon/"
|
||||
# → Returns matching tracks as symlinks to their virtual paths
|
||||
|
||||
# Or use the CLI
|
||||
musicfs search "dark side of the moon"
|
||||
musicfs search "artist:Metallica" --limit 50
|
||||
```
|
||||
|
||||
**Query syntax** (powered by [tantivy](https://github.com/quickwit-oss/tantivy)):
|
||||
|
||||
| Syntax | Example | Matches |
|
||||
|--------|---------|---------|
|
||||
| Simple terms | `metallica sandman` | All fields contain both words |
|
||||
| Field-specific | `artist:Metallica` | Artist field only |
|
||||
| Phrase | `album:"Master of Puppets"` | Exact phrase in album |
|
||||
| Fuzzy | `metalica~1` | Within Levenshtein distance 1 |
|
||||
| Range | `year:[1980 TO 1989]` | Numeric range |
|
||||
| Boolean | `genre:Metal AND year:[1980 TO 1989]` | Combined conditions |
|
||||
|
||||
Indexed fields: `title`, `artist`, `album`, `album_artist`, `genre`, `composer`, `year`.
|
||||
Results cached for 5 minutes. Max 1000 results per query. Queries capped at 256 characters.
|
||||
|
||||
### Smart Collections
|
||||
|
||||
Built-in and custom query-based virtual folders appear alongside regular directories:
|
||||
|
||||
- **Recently Added** — tracks added in the last 30 days
|
||||
- **80s Music** — year 1980–1989
|
||||
- **90s Music** — year 1990–1999
|
||||
|
||||
Custom collections can be defined via the gRPC API with compound boolean queries over any indexed field.
|
||||
|
||||
---
|
||||
|
||||
## Metadata Overlay
|
||||
|
||||
MusicFS lets you override metadata in the virtual layer **without touching origin files**. Overlaid metadata is synthesized into the audio file header on read — players see your corrected tags, the origin file is unchanged.
|
||||
|
||||
```bash
|
||||
# Fix a misnamed artist
|
||||
musicfs metadata set "/Unknown/Best Of/01 - Track.flac" \
|
||||
--artist "The Beatles" \
|
||||
--album "Past Masters"
|
||||
|
||||
# Verify
|
||||
musicfs metadata get "/The Beatles/Past Masters/01 - Track.flac"
|
||||
|
||||
# See what's been overlaid vs. original
|
||||
musicfs metadata diff "/The Beatles/Past Masters/01 - Track.flac"
|
||||
|
||||
# Revert
|
||||
musicfs metadata clear "/The Beatles/Past Masters/01 - Track.flac"
|
||||
```
|
||||
|
||||
Supported fields: `title`, `artist`, `album`, `album-artist`, `track`, `disc`, `genre`, `date`, `composer`, `comment`, `lyrics`, `copyright`, `compilation`, sort fields (`artist-sort`, etc.), MusicBrainz IDs, ReplayGain values, and arbitrary custom tags.
|
||||
|
||||
---
|
||||
|
||||
## Plugin Development
|
||||
|
||||
Plugins extend MusicFS without modifying core code. Three plugin types:
|
||||
|
||||
| Type | Purpose | Examples |
|
||||
|------|---------|---------|
|
||||
| **Origin** | Custom storage backends | Google Drive, Dropbox, custom NAS protocol |
|
||||
| **Metadata** | External tag enrichment | MusicBrainz, Discogs, Last.fm |
|
||||
| **Format** | Custom audio formats | Game audio, proprietary codecs |
|
||||
|
||||
### Native Plugin (`.so`)
|
||||
|
||||
```rust
|
||||
// Cargo.toml
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
musicfs-plugins = { path = "..." }
|
||||
semver = "1"
|
||||
serde_json = "1"
|
||||
```
|
||||
|
||||
```rust
|
||||
use musicfs_plugins::{declare_plugin, Plugin, PluginType, FormatPlugin};
|
||||
use musicfs_core::AudioMeta;
|
||||
use semver::Version;
|
||||
use serde_json::Value;
|
||||
|
||||
struct MyFormatPlugin;
|
||||
|
||||
impl Plugin for MyFormatPlugin {
|
||||
fn name(&self) -> &str { "my-format" }
|
||||
fn version(&self) -> Version { Version::new(1, 0, 0) }
|
||||
fn plugin_type(&self) -> PluginType { PluginType::Format }
|
||||
fn init(&mut self, _config: Value) -> musicfs_plugins::Result<()> { Ok(()) }
|
||||
fn shutdown(&mut self) -> musicfs_plugins::Result<()> { Ok(()) }
|
||||
}
|
||||
|
||||
impl FormatPlugin for MyFormatPlugin {
|
||||
fn extensions(&self) -> &[&str] { &["xyz"] }
|
||||
|
||||
fn parse(&self, reader: &mut dyn std::io::Read) -> musicfs_plugins::Result<AudioMeta> {
|
||||
// Parse your format and return metadata
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn synthesize_header(&self, metadata: &AudioMeta) -> musicfs_plugins::Result<Vec<u8>> {
|
||||
// Build a new file header with updated metadata
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
// Required export — MusicFS calls this to instantiate the plugin
|
||||
declare_plugin!(MyFormatPlugin, MyFormatPlugin);
|
||||
```
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
# produces target/release/libmy_format_plugin.so
|
||||
```
|
||||
|
||||
### Loading Plugins
|
||||
|
||||
```toml
|
||||
[plugins]
|
||||
enabled = true
|
||||
search_paths = ["/usr/lib/musicfs/plugins"] # Auto-discover .so files here
|
||||
|
||||
[plugins.plugins.my-format]
|
||||
path = "/path/to/libmy_format_plugin.so"
|
||||
enabled = true
|
||||
config = { key = "value" } # Passed to Plugin::init()
|
||||
```
|
||||
|
||||
### WASM Plugins (experimental)
|
||||
|
||||
```toml
|
||||
[plugins.wasm]
|
||||
enabled = true
|
||||
max_memory_mb = 64
|
||||
max_cpu_time_ms = 5000
|
||||
```
|
||||
|
||||
Load a `.wasm` binary at runtime via the gRPC API or by placing it in a search path. WASM plugins run sandboxed inside [wasmtime](https://wasmtime.dev/).
|
||||
|
||||
### Plugin API Version
|
||||
|
||||
Current: `0.1.0`. Breaking changes will increment the major version. MusicFS checks `musicfs_plugin_api_version()` before loading any native plugin.
|
||||
|
||||
---
|
||||
|
||||
## Control API (gRPC)
|
||||
|
||||
MusicFS exposes a gRPC API for programmatic control. The server starts automatically with the daemon.
|
||||
|
||||
**Default port:** `50052` (override with `--grpc-port`)
|
||||
**Proto definition:** `crates/musicfs-grpc/proto/musicfs.proto`
|
||||
|
||||
### Available RPCs
|
||||
|
||||
```
|
||||
MusicFS service:
|
||||
GetStatus → daemon version, uptime, mount state, open handles
|
||||
Shutdown → graceful or forced stop
|
||||
GetCacheStats → hit rate, chunk count, dedup ratio, per-tier breakdown
|
||||
ClearCache → clear all or per-origin, per-tier, dry-run supported
|
||||
Prefetch → pre-warm cache for paths or search queries
|
||||
ListOrigins → all configured origins with file count and health
|
||||
GetOriginHealth → health status and latency for one origin
|
||||
RescanOrigin → force re-scan with streaming progress
|
||||
Search → full-text search (paginated or streaming)
|
||||
SubscribeEvents → server-streaming live event feed
|
||||
|
||||
MetadataService:
|
||||
GetMetadata → all tags for a virtual path
|
||||
UpdateMetadata → set overlay tags for a file
|
||||
ClearOverlay → revert to original metadata
|
||||
ImportMetadata → bulk import from CSV/JSON (streaming progress)
|
||||
```
|
||||
|
||||
### Query with `grpcurl`
|
||||
|
||||
```bash
|
||||
# Daemon status
|
||||
grpcurl -plaintext localhost:50052 musicfs.v1.MusicFS/GetStatus
|
||||
|
||||
# Search
|
||||
grpcurl -plaintext -d '{"query": "metallica", "limit": 10}' \
|
||||
localhost:50052 musicfs.v1.MusicFS/Search
|
||||
|
||||
# Cache stats
|
||||
grpcurl -plaintext localhost:50052 musicfs.v1.MusicFS/GetCacheStats
|
||||
|
||||
# List origins
|
||||
grpcurl -plaintext localhost:50052 musicfs.v1.MusicFS/ListOrigins
|
||||
|
||||
# Trigger rescan with live progress
|
||||
grpcurl -plaintext -d '{"origin_id": "local"}' \
|
||||
localhost:50052 musicfs.v1.MusicFS/RescanOrigin
|
||||
|
||||
# Live event stream
|
||||
grpcurl -plaintext localhost:50052 musicfs.v1.MusicFS/SubscribeEvents
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Production Deployment
|
||||
|
||||
### systemd
|
||||
|
||||
```bash
|
||||
sudo cp dist/musicfs.service /etc/systemd/system/
|
||||
|
||||
# Edit the service to match your paths:
|
||||
# ExecStart=/usr/bin/musicfs mount --config /etc/musicfs/config.toml
|
||||
|
||||
sudo systemctl enable --now musicfs
|
||||
sudo systemctl status musicfs
|
||||
```
|
||||
|
||||
<!-- embedme dist/musicfs.service -->
|
||||
```ini
|
||||
[Unit]
|
||||
Description=MusicFS - Virtual FUSE Filesystem for Music
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/bin/musicfs mount /mnt/music --origin /path/to/music
|
||||
ExecStopPost=/usr/bin/fusermount -u /mnt/music
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
MusicFS sends `sd_notify(READY)` when the mount is live and `sd_notify(STOPPING)` during shutdown. Use `Type=notify` for precise readiness tracking.
|
||||
|
||||
### Signals
|
||||
|
||||
| Signal | Behavior |
|
||||
|--------|---------|
|
||||
| `SIGTERM` | Graceful shutdown — drains in-flight ops, unmounts |
|
||||
| `SIGINT` | Graceful shutdown (same) |
|
||||
| `SIGHUP` | Process pending file restores from trash |
|
||||
|
||||
### Security Notes
|
||||
|
||||
- Run as an **unprivileged user** — no root required.
|
||||
- Store remote credentials in the **system keyring** or environment variables. Never put them in the config file.
|
||||
- Credentials are redacted from logs and `RUST_LOG` output.
|
||||
- WASM plugins run sandboxed. Native `.so` plugins have full process access — only load plugins you trust.
|
||||
|
||||
---
|
||||
|
||||
## Observability
|
||||
|
||||
### Logs
|
||||
|
||||
```bash
|
||||
# Set level at startup
|
||||
musicfs mount ... --log-level debug
|
||||
# or via env
|
||||
RUST_LOG=musicfs=debug,warn musicfs mount ...
|
||||
```
|
||||
|
||||
| Level | Content |
|
||||
|-------|---------|
|
||||
| `error` | Unrecoverable failures, data corruption |
|
||||
| `warn` | Recoverable failures, origin timeouts, skipped files |
|
||||
| `info` | Mount/unmount, sync completion, config reload |
|
||||
| `debug` | Cache hits/misses, origin selection, file scans |
|
||||
| `trace` | Individual FUSE operations, chunk I/O |
|
||||
|
||||
Log files rotate daily in `log_dir` (default: `/var/log/musicfs/`). Structured JSON available with `json_output = true`. On Linux, logs forward to journald by default (`journald = true`).
|
||||
|
||||
### Prometheus Metrics
|
||||
|
||||
Metrics are exposed in Prometheus format via the gRPC API:
|
||||
|
||||
```
|
||||
musicfs_fuse_ops_total{op="read"} 152341
|
||||
musicfs_fuse_ops_total{op="readdir"} 8234
|
||||
musicfs_fuse_latency_seconds{op="read",quantile="0.99"} 0.004
|
||||
musicfs_cache_hits_total 142107
|
||||
musicfs_cache_misses_total 10234
|
||||
musicfs_cache_size_bytes 5368709120
|
||||
musicfs_origin_health{origin="local"} 1
|
||||
musicfs_origin_health{origin="s3"} 0
|
||||
musicfs_sync_files_changed{origin="local"} 15
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Performance
|
||||
|
||||
| Operation | Target | Maximum |
|
||||
|-----------|--------|---------|
|
||||
| Mount (any library size) | <100ms | 500ms |
|
||||
| `stat()` cached | <1ms | 5ms |
|
||||
| `readdir()` cached | <10ms | 50ms |
|
||||
| `open()` cached | <5ms | 20ms |
|
||||
| `read()` cached | <1ms | 5ms |
|
||||
| `read()` cache miss, local | <50ms | 200ms |
|
||||
| `read()` cache miss, remote | <200ms | 1000ms |
|
||||
| Search (1M tracks) | <500ms | 1000ms |
|
||||
| Sequential read (cached) | >500 MB/s | — |
|
||||
| Metadata ops | >1000 ops/s | — |
|
||||
|
||||
Memory: <50 MB idle, <200 MB with 1K files active, <500 MB peak.
|
||||
Scales to 10M+ files with O(1) mount and O(log n) lookups.
|
||||
|
||||
---
|
||||
|
||||
## Known Limitations
|
||||
|
||||
These are tracked issues — see `docs/v2/plans/` for details.
|
||||
|
||||
| Issue | Impact | Workaround |
|
||||
|-------|--------|-----------|
|
||||
| **No persistent state on mount** | Every restart does a full origin scan (O(N)). SQLite/search index persist but are not loaded on startup. | — |
|
||||
| **S3 and SFTP origins are stubs** | Only `local`, `nfs`, and `smb` have real implementations. | Use NFS/SMB mount as proxy for remote storage. |
|
||||
| **No write-through for metadata** | Overlaid metadata exists only in MusicFS's database, not in the actual audio files. | Use a tagger (beets, mp3tag) to write back if needed. |
|
||||
| **FUSE↔tokio deadlock risk** | `block_on()` in sync FUSE callbacks can stall under heavy concurrent load. | Keep concurrent open handles below ~500. |
|
||||
| **No background task supervision** | Health monitor, watcher, and indexer are fire-and-forget. A crash silently stops background work. | Restart the daemon periodically in critical deployments. |
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
MusicFS is a workspace of 11 Rust crates:
|
||||
|
||||
```
|
||||
musicfs-cli → binary, CLI parsing, startup wiring
|
||||
musicfs-fuse → FUSE operations (fuser), virtual tree serving
|
||||
musicfs-core → shared types, config, events, errors
|
||||
musicfs-cache → SQLite metadata DB, virtual tree, format handlers
|
||||
musicfs-cas → content-addressable chunk store (sled + xxHash64)
|
||||
musicfs-origins → origin backends (local, NFS, SMB, S3 stub, SFTP stub)
|
||||
musicfs-metadata → audio tag extraction (symphonia)
|
||||
musicfs-sync → delta sync, CDC chunking (FastCDC), inotify watcher
|
||||
musicfs-search → full-text index (tantivy), .search/ virtual dir
|
||||
musicfs-grpc → gRPC server (tonic + prost), proto codegen
|
||||
musicfs-plugins → plugin host, native .so loader, WASM sandbox
|
||||
```
|
||||
|
||||
Data flow on a cache miss: `FUSE read()` → `VirtualPathResolver` → `CAS` (chunk lookup) → `OriginFederation` (fetch missing range) → CDC chunk → store → return.
|
||||
|
||||
Full design: [`docs/v2/architecture.md`](docs/v2/architecture.md)
|
||||
Requirements: [`docs/v2/requirements.md`](docs/v2/requirements.md)
|
||||
Roadmap: [`docs/v2/development-plan.md`](docs/v2/development-plan.md)
|
||||
|
||||
---
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
nix develop # Enter dev shell
|
||||
|
||||
cargo check # Fast compile check
|
||||
cargo test # All 162 tests
|
||||
cargo test -p musicfs-core # Single crate
|
||||
cargo clippy # Lint
|
||||
cargo fmt # Format
|
||||
cargo nextest run # Parallel test runner (faster)
|
||||
cargo watch -x check -x test # Watch mode
|
||||
|
||||
# Cargo aliases
|
||||
cargo t # test
|
||||
cargo c # check
|
||||
cargo b # build
|
||||
|
||||
# gRPC codegen (runs via build.rs automatically)
|
||||
cargo build -p musicfs-grpc
|
||||
```
|
||||
|
||||
Pre-commit hooks (rustfmt + clippy) are installed automatically in the Nix dev shell.
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
MIT OR Apache-2.0 — see [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE).
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
#+title: musicfs
|
||||
|
||||
A read-only FUSE filesystem that presents a music library reorganised by
|
||||
*metadata* rather than by however the files happen to sit on disk. Point it at a
|
||||
messy ~~/Music~ tree (or a remote ~musicfs-server~) and it mounts a clean
|
||||
=Artist/Album/Track= view, driven entirely by the tags parsed out of the audio
|
||||
files themselves.
|
||||
|
||||
#+begin_example
|
||||
mountpoint/
|
||||
├── ДДТ/
|
||||
│ └── Творчество в пустоте/
|
||||
│ ├── 01 Intro.flac
|
||||
│ └── 02 ...
|
||||
└── Some Artist/
|
||||
└── Some Album/
|
||||
└── 01 Track.flac
|
||||
#+end_example
|
||||
|
||||
The source files are never moved or modified. The directory hierarchy is
|
||||
*virtual*: synthesised from ~album_artist~/~album~ tags, with stable synthetic
|
||||
inodes so the layout survives remounts.
|
||||
|
||||
* How it works
|
||||
|
||||
Two *origins* feed the same FUSE frontend:
|
||||
|
||||
- *LocalOrigin* — walks a directory on the host, parses tags, builds the
|
||||
snapshot. A =notify= watcher keeps it live as files change.
|
||||
- *NetworkOrigin* — talks gRPC to a remote ~musicfs-server~. Metadata and file
|
||||
bytes arrive over the wire; a Postgres table caches both. Reconciliation is
|
||||
hash-based: the client sends the ~(inode, hash)~ pairs it has, the server
|
||||
replies with only what changed or was deleted, so almost nothing crosses the
|
||||
network on a steady-state refresh.
|
||||
|
||||
Which one is used is chosen automatically from the ~--source~ argument: an
|
||||
=http(s)://= URL means network, anything else is treated as a local path.
|
||||
|
||||
State lives in Postgres — parsed metadata, the virtual-path layout, and (for the
|
||||
network origin) a lazy byte cache populated only for files that were actually
|
||||
read.
|
||||
|
||||
* Layout
|
||||
|
||||
| Crate | Responsibility |
|
||||
|------------------+-----------------------------------------------------------------------|
|
||||
| =musicfs-proto= | Protobuf/gRPC definitions (=proto/musicfs.proto=), generated bindings |
|
||||
| =musicfs-core= | Shared logic: tag parsing (FLAC/MP3 via symphonia), hashing, logging |
|
||||
| =musicfs-client= | FUSE mount, origins, Postgres cache/sync — the =musicfs= binary |
|
||||
| =musicfs-server= | gRPC server sharing a library — the =musicfs-server= binary |
|
||||
|
||||
The gRPC surface (see =proto/musicfs.proto=): ~Reconcile~, ~GetMetadata~,
|
||||
~GetManifest~, ~GetFile~ (whole-file or byte-range streaming), and
|
||||
~SubscribeEvents~ (change wake-ups). The client also serves a ~ClientStatus~ RPC
|
||||
for introspection.
|
||||
|
||||
* Running the client
|
||||
|
||||
Everything runs inside the =devenv= shell, which provides the Rust toolchain,
|
||||
Postgres, FUSE tooling, and gRPC utilities.
|
||||
|
||||
#+begin_src bash
|
||||
devenv shell
|
||||
devenv up --profile local # starts Postgres + the FUSE mount
|
||||
#+end_src
|
||||
|
||||
Predefined profiles in =devenv.nix=:
|
||||
|
||||
| Profile | Source | Purpose |
|
||||
|-----------+---------------------------------+-------------------------------|
|
||||
| =local= | a local directory | mount a host music folder |
|
||||
| =remote= | =http://…:50051= | mount a remote musicfs-server |
|
||||
| =e2e= | =http://127.0.0.1:50061= | end-to-end test harness |
|
||||
|
||||
Or run the binary directly:
|
||||
|
||||
#+begin_src bash
|
||||
cargo run -p musicfs-client -- \
|
||||
--source /home/you/Music \
|
||||
--mountpoint /tmp/musicfs \
|
||||
--database "postgresql://you@localhost/musicfs?host=$PGHOST"
|
||||
#+end_src
|
||||
|
||||
Key flags: =--source= (local path or ~http(s)://~ server URL), =--mountpoint=,
|
||||
=--database= (Postgres URL), =--log-dir= (daily-rotated logs, default =./logs=),
|
||||
=--listen= (address for the client's status/health RPCs, default
|
||||
=127.0.0.1:50052=).
|
||||
|
||||
* Running the server in an Incus VM
|
||||
|
||||
~musicfs-server~ (gRPC) runs inside an Incus VM, live-sharing the host's
|
||||
~~/Music~ as a read-only virtiofs mount. =scripts/vm.sh= builds the binary on
|
||||
the host, bundles its nix ~glibc~ so it runs unmodified in the VM, ships it in,
|
||||
and runs it under systemd.
|
||||
|
||||
#+begin_src bash
|
||||
devenv shell # cargo, incus, patchelf, grpcurl
|
||||
scripts/vm.sh up # create VM, share ~/Music, build, ship, start server
|
||||
#+end_src
|
||||
|
||||
Options (env): =VM_NAME=, =IMAGE=, =MUSIC_SOURCE=, =LISTEN_PORT=, =RUST_LOG=.
|
||||
|
||||
| Command | Action |
|
||||
|------------------------+-------------------------------|
|
||||
| =scripts/vm.sh up= | create + build + ship + start |
|
||||
| =scripts/vm.sh redeploy= | rebuild after a code change |
|
||||
| =scripts/vm.sh logs= | tail server logs |
|
||||
| =scripts/vm.sh status= | VM + service status |
|
||||
| =scripts/vm.sh shell= | shell inside the VM |
|
||||
| =scripts/vm.sh down= | stop the VM |
|
||||
| =scripts/vm.sh destroy= | delete the VM |
|
||||
|
||||
Reach the server at the VM's bridge IP (=scripts/vm.sh status= prints it).
|
||||
|
||||
** Poke at it (Nushell)
|
||||
|
||||
#+begin_src nushell
|
||||
let VMIP = (incus list musicfs -f csv -c 4 | lines | first | split row " " | first)
|
||||
let ADDR = $"($VMIP):50051"
|
||||
|
||||
# liveness
|
||||
^nc -z -w 3 $VMIP 50051
|
||||
if $env.LAST_EXIT_CODE == 0 { print "alive" } else { print "dead" }
|
||||
|
||||
# gRPC: stream the manifest, count entries
|
||||
grpcurl -plaintext -import-path proto -proto musicfs.proto -d "{}" $ADDR musicfs.MusicFs/GetManifest | lines | find relPath | length
|
||||
|
||||
# files the server scans
|
||||
incus exec musicfs -- find /music -type f | lines | length
|
||||
|
||||
# play a track straight out of the VM's shared /music
|
||||
incus exec musicfs -- cat "/music/DDT/ДДТ - Творчество в пустоте – 2 - 01 Intro.flac" | ^mpv -
|
||||
#+end_src
|
||||
|
||||
Install mpv with =nix profile install nixpkgs#mpv=, or use =^ffplay -= instead.
|
||||
|
||||
In Nushell, always put =| lines= between an external command's stdout and a
|
||||
builtin (=find=, =first=, =length=, …); external-to-external pipes (=cat | mpv=)
|
||||
are byte-stable and don't need it.
|
||||
|
||||
* Development
|
||||
|
||||
#+begin_src bash
|
||||
just build # cargo build
|
||||
cargo test # unit tests
|
||||
just e2e # end-to-end suite (scripts/e2e/run.sh)
|
||||
just e2e-resilience # resilience suite
|
||||
#+end_src
|
||||
|
||||
Formatting and lint (clippy, treefmt with rustfmt + nixfmt) run as git hooks via
|
||||
=devenv=. Requires Rust edition 2024.
|
||||
@@ -1,75 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-05-12T14:42:37.343765",
|
||||
"results": [
|
||||
{
|
||||
"mean_ms": null,
|
||||
"runs": 0,
|
||||
"name": "mount_time",
|
||||
"memory_kb": null,
|
||||
"error": "Mount process died: Traceback (most recent call last):\n File \"/tmp/nix-shell.VlFHpy/nix-shell.rhvctI/beetfs_bench_JfIl36/mount.py\", line 40, in <module>\n beetFs.directory_structure.adddir(sub_elements, level_subbed[level])\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 414, in adddir\n node = self.getnode(elements, root=root)\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 403, in getnode\n return self.getnode(elements, root=root.dirs[topdir])\nKeyError: u'Bench Artist'\n",
|
||||
"min_ms": null,
|
||||
"metadata": {},
|
||||
"max_ms": null
|
||||
},
|
||||
{
|
||||
"mean_ms": null,
|
||||
"runs": 0,
|
||||
"name": "readdir",
|
||||
"memory_kb": null,
|
||||
"error": "Mount process died: Traceback (most recent call last):\n File \"/tmp/nix-shell.VlFHpy/nix-shell.rhvctI/beetfs_bench_JfIl36/mount.py\", line 40, in <module>\n beetFs.directory_structure.adddir(sub_elements, level_subbed[level])\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 414, in adddir\n node = self.getnode(elements, root=root)\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 403, in getnode\n return self.getnode(elements, root=root.dirs[topdir])\nKeyError: u'Bench Artist'\n",
|
||||
"min_ms": null,
|
||||
"metadata": {},
|
||||
"max_ms": null
|
||||
},
|
||||
{
|
||||
"mean_ms": null,
|
||||
"runs": 0,
|
||||
"name": "stat_latency",
|
||||
"memory_kb": null,
|
||||
"error": "Mount process died: Traceback (most recent call last):\n File \"/tmp/nix-shell.VlFHpy/nix-shell.rhvctI/beetfs_bench_JfIl36/mount.py\", line 40, in <module>\n beetFs.directory_structure.adddir(sub_elements, level_subbed[level])\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 414, in adddir\n node = self.getnode(elements, root=root)\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 403, in getnode\n return self.getnode(elements, root=root.dirs[topdir])\nKeyError: u'Bench Artist'\n",
|
||||
"min_ms": null,
|
||||
"metadata": {},
|
||||
"max_ms": null
|
||||
},
|
||||
{
|
||||
"mean_ms": null,
|
||||
"runs": 0,
|
||||
"name": "enoent_lookup",
|
||||
"memory_kb": null,
|
||||
"error": "Mount process died: Traceback (most recent call last):\n File \"/tmp/nix-shell.VlFHpy/nix-shell.rhvctI/beetfs_bench_JfIl36/mount.py\", line 40, in <module>\n beetFs.directory_structure.adddir(sub_elements, level_subbed[level])\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 414, in adddir\n node = self.getnode(elements, root=root)\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 403, in getnode\n return self.getnode(elements, root=root.dirs[topdir])\nKeyError: u'Bench Artist'\n",
|
||||
"min_ms": null,
|
||||
"metadata": {},
|
||||
"max_ms": null
|
||||
},
|
||||
{
|
||||
"mean_ms": null,
|
||||
"runs": 0,
|
||||
"name": "file_open",
|
||||
"memory_kb": null,
|
||||
"error": "Mount process died: Traceback (most recent call last):\n File \"/tmp/nix-shell.VlFHpy/nix-shell.rhvctI/beetfs_bench_JfIl36/mount.py\", line 40, in <module>\n beetFs.directory_structure.adddir(sub_elements, level_subbed[level])\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 414, in adddir\n node = self.getnode(elements, root=root)\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 403, in getnode\n return self.getnode(elements, root=root.dirs[topdir])\nKeyError: u'Bench Artist'\n",
|
||||
"min_ms": null,
|
||||
"metadata": {},
|
||||
"max_ms": null
|
||||
},
|
||||
{
|
||||
"mean_ms": null,
|
||||
"runs": 0,
|
||||
"name": "read_throughput",
|
||||
"memory_kb": null,
|
||||
"error": "Mount process died: Traceback (most recent call last):\n File \"/tmp/nix-shell.VlFHpy/nix-shell.rhvctI/beetfs_bench_JfIl36/mount.py\", line 40, in <module>\n beetFs.directory_structure.adddir(sub_elements, level_subbed[level])\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 414, in adddir\n node = self.getnode(elements, root=root)\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 403, in getnode\n return self.getnode(elements, root=root.dirs[topdir])\nKeyError: u'Bench Artist'\n",
|
||||
"min_ms": null,
|
||||
"metadata": {},
|
||||
"max_ms": null
|
||||
},
|
||||
{
|
||||
"mean_ms": null,
|
||||
"runs": 0,
|
||||
"name": "memory_usage",
|
||||
"memory_kb": null,
|
||||
"error": "Mount process died: Traceback (most recent call last):\n File \"/tmp/nix-shell.VlFHpy/nix-shell.rhvctI/beetfs_bench_JfIl36/mount.py\", line 40, in <module>\n beetFs.directory_structure.adddir(sub_elements, level_subbed[level])\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 414, in adddir\n node = self.getnode(elements, root=root)\n File \"/home/fujin/Code/agregators/music-agregator/beetfs/beetsplug/beetFs.py\", line 403, in getnode\n return self.getnode(elements, root=root.dirs[topdir])\nKeyError: u'Bench Artist'\n",
|
||||
"min_ms": null,
|
||||
"metadata": {},
|
||||
"max_ms": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,624 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
beetfs Benchmark Suite
|
||||
Measures mount time, metadata ops, file I/O, and memory usage.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import tempfile
|
||||
import shutil
|
||||
import subprocess
|
||||
import signal
|
||||
import resource
|
||||
import datetime
|
||||
|
||||
# Add project paths
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'beetsplug'))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'tests'))
|
||||
|
||||
from conftest import create_synthetic_flac
|
||||
|
||||
|
||||
class BenchmarkResult(object):
|
||||
"""Stores benchmark results."""
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.timings = []
|
||||
self.memory_kb = None
|
||||
self.error = None
|
||||
self.metadata = {}
|
||||
|
||||
def add_timing(self, seconds):
|
||||
self.timings.append(seconds)
|
||||
|
||||
@property
|
||||
def mean(self):
|
||||
if not self.timings:
|
||||
return None
|
||||
return sum(self.timings) / len(self.timings)
|
||||
|
||||
@property
|
||||
def min_time(self):
|
||||
return min(self.timings) if self.timings else None
|
||||
|
||||
@property
|
||||
def max_time(self):
|
||||
return max(self.timings) if self.timings else None
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'name': self.name,
|
||||
'mean_ms': self.mean * 1000 if self.mean else None,
|
||||
'min_ms': self.min_time * 1000 if self.min_time else None,
|
||||
'max_ms': self.max_time * 1000 if self.max_time else None,
|
||||
'runs': len(self.timings),
|
||||
'memory_kb': self.memory_kb,
|
||||
'error': self.error,
|
||||
'metadata': self.metadata
|
||||
}
|
||||
|
||||
|
||||
class BeetFSBenchmark(object):
|
||||
"""Benchmark harness for beetfs."""
|
||||
|
||||
def __init__(self, output_dir):
|
||||
self.output_dir = output_dir
|
||||
self.results = []
|
||||
self.temp_dir = None
|
||||
self.mount_dir = None
|
||||
self.music_dir = None
|
||||
self.db_path = None
|
||||
self.mount_process = None
|
||||
|
||||
def setup(self, num_tracks=10, track_size_mb=5):
|
||||
"""Create test environment with synthetic tracks."""
|
||||
self.temp_dir = tempfile.mkdtemp(prefix='beetfs_bench_')
|
||||
self.mount_dir = os.path.join(self.temp_dir, 'mount')
|
||||
self.music_dir = os.path.join(self.temp_dir, 'music')
|
||||
self.db_path = os.path.join(self.temp_dir, 'library.db')
|
||||
self.config_dir = os.path.join(self.temp_dir, 'config')
|
||||
|
||||
os.makedirs(self.mount_dir)
|
||||
os.makedirs(self.music_dir)
|
||||
os.makedirs(self.config_dir)
|
||||
|
||||
# Create beets config
|
||||
config_path = os.path.join(self.config_dir, 'config.yaml')
|
||||
with open(config_path, 'w') as f:
|
||||
f.write('directory: {}\n'.format(self.music_dir))
|
||||
f.write('library: {}\n'.format(self.db_path))
|
||||
f.write('plugins: []\n')
|
||||
|
||||
os.environ['BEETSDIR'] = self.config_dir
|
||||
|
||||
# Create synthetic FLAC files
|
||||
print("Creating {} synthetic tracks ({} MB each)...".format(num_tracks, track_size_mb))
|
||||
track_paths = []
|
||||
for i in range(num_tracks):
|
||||
artist = 'Bench Artist'
|
||||
album = 'Bench Album'
|
||||
title = 'Track {:03d}'.format(i + 1)
|
||||
filename = '{:02d} - {} - {}.flac'.format(i + 1, artist, title)
|
||||
track_path = os.path.join(self.music_dir, artist, album, filename)
|
||||
self._makedirs(os.path.dirname(track_path))
|
||||
create_synthetic_flac(track_path, duration_sec=track_size_mb * 10,
|
||||
artist=artist, title=title, album=album, track=str(i + 1))
|
||||
track_paths.append(track_path)
|
||||
|
||||
# Import into beets library
|
||||
print("Importing tracks into beets library...")
|
||||
from beets import config
|
||||
from beets.library import Library
|
||||
config.read(user=False)
|
||||
config['directory'].set(self.music_dir)
|
||||
config['library'].set(self.db_path)
|
||||
|
||||
lib = Library(self.db_path)
|
||||
from beets.library import Item
|
||||
for i, path in enumerate(track_paths):
|
||||
item = Item(
|
||||
path=path,
|
||||
artist=u'Bench Artist',
|
||||
album=u'Bench Album',
|
||||
title=u'Track {:03d}'.format(i + 1),
|
||||
track=i + 1,
|
||||
year=2024,
|
||||
genre=u'Benchmark',
|
||||
format='flac'
|
||||
)
|
||||
lib.add(item)
|
||||
lib._close()
|
||||
|
||||
return len(track_paths)
|
||||
|
||||
def _makedirs(self, path):
|
||||
"""Python 2 compatible makedirs."""
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
def teardown(self):
|
||||
"""Clean up test environment."""
|
||||
self.unmount()
|
||||
if self.temp_dir and os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
|
||||
def mount(self):
|
||||
"""Mount beetfs and return time taken."""
|
||||
# Create mount script
|
||||
beetfs_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
beetsplug = os.path.join(beetfs_root, 'beetsplug')
|
||||
|
||||
mount_script = '''
|
||||
import sys
|
||||
sys.path.insert(0, "{beetfs_root}")
|
||||
sys.path.insert(0, "{beetsplug}")
|
||||
|
||||
import os
|
||||
import re
|
||||
os.environ["BEETSDIR"] = "{config_dir}"
|
||||
|
||||
from beets import config
|
||||
from beets.library import Library
|
||||
|
||||
config.read(user=False)
|
||||
config["directory"] = "{music_dir}"
|
||||
config["library"] = "{db_path}"
|
||||
|
||||
lib = Library("{db_path}")
|
||||
|
||||
import beetFs
|
||||
import fuse
|
||||
|
||||
fuse.fuse_python_api = (0, 2)
|
||||
|
||||
beetFs.library = lib
|
||||
beetFs.structure_depth = 4
|
||||
beetFs.structure_split = [0, 1, 2, 3]
|
||||
beetFs.directory_structure = beetFs.FSNode({{}}, {{}})
|
||||
|
||||
for item in lib.items():
|
||||
mapping = beetFs.template_mapping(lib, item)
|
||||
path_str = beetFs.PATH_FORMAT
|
||||
for key, val in mapping.items():
|
||||
if val is not None:
|
||||
clean_val = re.sub(r"[\\\\/:]|^\\.", "_", unicode(val))
|
||||
path_str = path_str.replace("$" + key, clean_val)
|
||||
elements = path_str.split("/")
|
||||
sub_elements = elements[0:beetFs.structure_depth-1]
|
||||
for level in range(len(sub_elements)):
|
||||
level_subbed = sub_elements[0:level+1]
|
||||
beetFs.directory_structure.adddir(sub_elements, level_subbed[level])
|
||||
beetFs.directory_structure.addfile(
|
||||
sub_elements,
|
||||
elements[beetFs.structure_depth-1],
|
||||
item.id
|
||||
)
|
||||
|
||||
fs = beetFs.beetFileSystem(
|
||||
version="%prog " + fuse.__version__,
|
||||
usage="beetfs benchmark",
|
||||
dash_s_do="setsingle"
|
||||
)
|
||||
|
||||
fs.parser.add_option(mountopt="root", metavar="PATH", default="{music_dir}",
|
||||
help="music library root path")
|
||||
fs.parse(args=["{mount_dir}"], errex=1)
|
||||
fs.flags = 0
|
||||
fs.multithreaded = False
|
||||
fs.fuse_args.setmod("foreground")
|
||||
fs.fuse_args.add("fsname=beetfs")
|
||||
fs.fuse_args.add("nonempty")
|
||||
fs.lib = lib
|
||||
|
||||
fs.main()
|
||||
'''.format(
|
||||
beetfs_root=beetfs_root,
|
||||
beetsplug=beetsplug,
|
||||
config_dir=self.config_dir,
|
||||
music_dir=self.music_dir,
|
||||
db_path=self.db_path,
|
||||
mount_dir=self.mount_dir
|
||||
)
|
||||
|
||||
script_path = os.path.join(self.temp_dir, 'mount.py')
|
||||
with open(script_path, 'w') as f:
|
||||
f.write(mount_script)
|
||||
|
||||
start_time = time.time()
|
||||
self.mount_process = subprocess.Popen(
|
||||
[sys.executable, script_path],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
|
||||
# Wait for mount
|
||||
timeout = 30
|
||||
poll_interval = 0.05
|
||||
elapsed = 0
|
||||
while elapsed < timeout:
|
||||
if os.path.ismount(self.mount_dir):
|
||||
mount_time = time.time() - start_time
|
||||
return mount_time
|
||||
time.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
|
||||
# Check if process died
|
||||
if self.mount_process.poll() is not None:
|
||||
stdout, stderr = self.mount_process.communicate()
|
||||
raise RuntimeError("Mount process died: {}".format(stderr.decode('utf-8', errors='replace')))
|
||||
|
||||
raise RuntimeError("Mount timeout after {} seconds".format(timeout))
|
||||
|
||||
def unmount(self):
|
||||
"""Unmount beetfs."""
|
||||
if os.path.ismount(self.mount_dir):
|
||||
subprocess.call(['fusermount', '-u', self.mount_dir])
|
||||
time.sleep(0.5)
|
||||
|
||||
if self.mount_process and self.mount_process.poll() is None:
|
||||
self.mount_process.terminate()
|
||||
try:
|
||||
self.mount_process.wait(timeout=5)
|
||||
except:
|
||||
self.mount_process.kill()
|
||||
|
||||
def get_memory_usage(self):
|
||||
"""Get current process memory usage in KB."""
|
||||
if self.mount_process and self.mount_process.poll() is None:
|
||||
try:
|
||||
with open('/proc/{}/status'.format(self.mount_process.pid)) as f:
|
||||
for line in f:
|
||||
if line.startswith('VmRSS:'):
|
||||
return int(line.split()[1])
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
# ========================
|
||||
# BENCHMARK METHODS
|
||||
# ========================
|
||||
|
||||
def bench_mount_time(self, runs=5):
|
||||
"""Benchmark mount time."""
|
||||
result = BenchmarkResult('mount_time')
|
||||
print("\n=== Mount Time Benchmark ({} runs) ===".format(runs))
|
||||
|
||||
for i in range(runs):
|
||||
try:
|
||||
mount_time = self.mount()
|
||||
result.add_timing(mount_time)
|
||||
print(" Run {}: {:.3f}s".format(i + 1, mount_time))
|
||||
result.memory_kb = self.get_memory_usage()
|
||||
self.unmount()
|
||||
time.sleep(0.5)
|
||||
except Exception as e:
|
||||
result.error = str(e)
|
||||
print(" Run {}: ERROR - {}".format(i + 1, e))
|
||||
break
|
||||
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
def bench_stat_latency(self, runs=50):
|
||||
"""Benchmark single stat() call latency."""
|
||||
result = BenchmarkResult('stat_latency')
|
||||
print("\n=== Stat Latency Benchmark ({} runs) ===".format(runs))
|
||||
|
||||
try:
|
||||
self.mount()
|
||||
time.sleep(1) # Let mount settle
|
||||
|
||||
# Find a file to stat
|
||||
test_path = None
|
||||
for root, dirs, files in os.walk(self.mount_dir):
|
||||
if files:
|
||||
test_path = os.path.join(root, files[0])
|
||||
break
|
||||
|
||||
if not test_path:
|
||||
result.error = "No files found in mount"
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
result.metadata['test_path'] = test_path
|
||||
|
||||
for i in range(runs):
|
||||
start = time.time()
|
||||
try:
|
||||
os.stat(test_path)
|
||||
elapsed = time.time() - start
|
||||
result.add_timing(elapsed)
|
||||
except OSError as e:
|
||||
result.error = "stat failed: {} (errno {})".format(e.strerror, e.errno)
|
||||
print(" ERROR: {}".format(result.error))
|
||||
break
|
||||
|
||||
if result.timings:
|
||||
print(" Mean: {:.3f}ms, Min: {:.3f}ms, Max: {:.3f}ms".format(
|
||||
result.mean * 1000, result.min_time * 1000, result.max_time * 1000))
|
||||
|
||||
self.unmount()
|
||||
except Exception as e:
|
||||
result.error = str(e)
|
||||
print(" ERROR: {}".format(e))
|
||||
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
def bench_readdir(self, runs=20):
|
||||
"""Benchmark directory listing."""
|
||||
result = BenchmarkResult('readdir')
|
||||
print("\n=== Readdir Benchmark ({} runs) ===".format(runs))
|
||||
|
||||
try:
|
||||
self.mount()
|
||||
time.sleep(1)
|
||||
|
||||
for i in range(runs):
|
||||
start = time.time()
|
||||
try:
|
||||
entries = os.listdir(self.mount_dir)
|
||||
elapsed = time.time() - start
|
||||
result.add_timing(elapsed)
|
||||
if i == 0:
|
||||
result.metadata['entry_count'] = len(entries)
|
||||
except OSError as e:
|
||||
result.error = "listdir failed: {} (errno {})".format(e.strerror, e.errno)
|
||||
print(" ERROR: {}".format(result.error))
|
||||
break
|
||||
|
||||
if result.timings:
|
||||
print(" Mean: {:.3f}ms, Entries: {}".format(
|
||||
result.mean * 1000, result.metadata.get('entry_count', 'N/A')))
|
||||
|
||||
self.unmount()
|
||||
except Exception as e:
|
||||
result.error = str(e)
|
||||
print(" ERROR: {}".format(e))
|
||||
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
def bench_file_open(self, runs=10):
|
||||
"""Benchmark file open latency."""
|
||||
result = BenchmarkResult('file_open')
|
||||
print("\n=== File Open Benchmark ({} runs) ===".format(runs))
|
||||
|
||||
try:
|
||||
self.mount()
|
||||
time.sleep(1)
|
||||
|
||||
# Find a file to open
|
||||
test_path = None
|
||||
for root, dirs, files in os.walk(self.mount_dir):
|
||||
if files:
|
||||
test_path = os.path.join(root, files[0])
|
||||
break
|
||||
|
||||
if not test_path:
|
||||
result.error = "No files found in mount"
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
result.metadata['test_path'] = test_path
|
||||
|
||||
for i in range(runs):
|
||||
# Clear page cache between runs (requires sudo, skip if not available)
|
||||
try:
|
||||
subprocess.call(['sync'])
|
||||
except:
|
||||
pass
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
f = open(test_path, 'rb')
|
||||
f.read(1) # Trigger actual open
|
||||
f.close()
|
||||
elapsed = time.time() - start
|
||||
result.add_timing(elapsed)
|
||||
except (IOError, OSError) as e:
|
||||
result.error = "open failed: {}".format(e)
|
||||
print(" ERROR: {}".format(result.error))
|
||||
break
|
||||
|
||||
if result.timings:
|
||||
print(" Mean: {:.3f}ms, Min: {:.3f}ms, Max: {:.3f}ms".format(
|
||||
result.mean * 1000, result.min_time * 1000, result.max_time * 1000))
|
||||
|
||||
self.unmount()
|
||||
except Exception as e:
|
||||
result.error = str(e)
|
||||
print(" ERROR: {}".format(e))
|
||||
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
def bench_read_throughput(self):
|
||||
"""Benchmark read throughput."""
|
||||
result = BenchmarkResult('read_throughput')
|
||||
print("\n=== Read Throughput Benchmark ===")
|
||||
|
||||
try:
|
||||
self.mount()
|
||||
time.sleep(1)
|
||||
|
||||
# Find a file to read
|
||||
test_path = None
|
||||
for root, dirs, files in os.walk(self.mount_dir):
|
||||
if files:
|
||||
test_path = os.path.join(root, files[0])
|
||||
break
|
||||
|
||||
if not test_path:
|
||||
result.error = "No files found in mount"
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
result.metadata['test_path'] = test_path
|
||||
|
||||
# Read entire file and measure throughput
|
||||
start = time.time()
|
||||
try:
|
||||
with open(test_path, 'rb') as f:
|
||||
data = f.read()
|
||||
elapsed = time.time() - start
|
||||
|
||||
file_size = len(data)
|
||||
throughput_mbps = (file_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0
|
||||
|
||||
result.add_timing(elapsed)
|
||||
result.metadata['file_size_bytes'] = file_size
|
||||
result.metadata['throughput_mbps'] = throughput_mbps
|
||||
|
||||
print(" File size: {:.2f} MB, Time: {:.3f}s, Throughput: {:.2f} MB/s".format(
|
||||
file_size / (1024 * 1024), elapsed, throughput_mbps))
|
||||
|
||||
except (IOError, OSError) as e:
|
||||
result.error = "read failed: {}".format(e)
|
||||
print(" ERROR: {}".format(result.error))
|
||||
|
||||
self.unmount()
|
||||
except Exception as e:
|
||||
result.error = str(e)
|
||||
print(" ERROR: {}".format(e))
|
||||
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
def bench_memory_usage(self):
|
||||
"""Benchmark memory usage."""
|
||||
result = BenchmarkResult('memory_usage')
|
||||
print("\n=== Memory Usage Benchmark ===")
|
||||
|
||||
try:
|
||||
self.mount()
|
||||
time.sleep(2)
|
||||
|
||||
# Measure idle memory
|
||||
idle_mem = self.get_memory_usage()
|
||||
result.metadata['idle_memory_kb'] = idle_mem
|
||||
print(" Idle memory: {} KB".format(idle_mem))
|
||||
|
||||
# Open a file and measure
|
||||
test_path = None
|
||||
for root, dirs, files in os.walk(self.mount_dir):
|
||||
if files:
|
||||
test_path = os.path.join(root, files[0])
|
||||
break
|
||||
|
||||
if test_path:
|
||||
try:
|
||||
with open(test_path, 'rb') as f:
|
||||
f.read()
|
||||
after_read_mem = self.get_memory_usage()
|
||||
result.metadata['after_read_memory_kb'] = after_read_mem
|
||||
print(" After file read: {} KB".format(after_read_mem))
|
||||
if idle_mem and after_read_mem:
|
||||
print(" Memory increase: {} KB".format(after_read_mem - idle_mem))
|
||||
except (IOError, OSError) as e:
|
||||
result.error = "read failed: {}".format(e)
|
||||
|
||||
result.memory_kb = self.get_memory_usage()
|
||||
self.unmount()
|
||||
except Exception as e:
|
||||
result.error = str(e)
|
||||
print(" ERROR: {}".format(e))
|
||||
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
def bench_enoent_lookup(self, runs=50):
|
||||
"""Benchmark ENOENT lookup (missing file) latency."""
|
||||
result = BenchmarkResult('enoent_lookup')
|
||||
print("\n=== ENOENT Lookup Benchmark ({} runs) ===".format(runs))
|
||||
|
||||
try:
|
||||
self.mount()
|
||||
time.sleep(1)
|
||||
|
||||
# Non-existent file path
|
||||
missing_path = os.path.join(self.mount_dir, 'nonexistent', 'cover.jpg')
|
||||
|
||||
for i in range(runs):
|
||||
start = time.time()
|
||||
try:
|
||||
os.stat(missing_path)
|
||||
except OSError:
|
||||
pass # Expected
|
||||
elapsed = time.time() - start
|
||||
result.add_timing(elapsed)
|
||||
|
||||
if result.timings:
|
||||
print(" Mean: {:.3f}ms, Min: {:.3f}ms, Max: {:.3f}ms".format(
|
||||
result.mean * 1000, result.min_time * 1000, result.max_time * 1000))
|
||||
|
||||
self.unmount()
|
||||
except Exception as e:
|
||||
result.error = str(e)
|
||||
print(" ERROR: {}".format(e))
|
||||
|
||||
self.results.append(result)
|
||||
return result
|
||||
|
||||
def save_results(self, filename='benchmark_results.json'):
|
||||
"""Save results to JSON file."""
|
||||
output_path = os.path.join(self.output_dir, filename)
|
||||
data = {
|
||||
'timestamp': datetime.datetime.now().isoformat(),
|
||||
'results': [r.to_dict() for r in self.results]
|
||||
}
|
||||
with open(output_path, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
print("\nResults saved to: {}".format(output_path))
|
||||
return output_path
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("beetfs Benchmark Suite")
|
||||
print("=" * 60)
|
||||
|
||||
output_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'results')
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
bench = BeetFSBenchmark(output_dir)
|
||||
|
||||
try:
|
||||
# Setup with 10 tracks, 5MB each
|
||||
num_tracks = bench.setup(num_tracks=10, track_size_mb=5)
|
||||
print("Setup complete: {} tracks".format(num_tracks))
|
||||
|
||||
# Run benchmarks
|
||||
bench.bench_mount_time(runs=3)
|
||||
bench.bench_readdir(runs=10)
|
||||
bench.bench_stat_latency(runs=20)
|
||||
bench.bench_enoent_lookup(runs=20)
|
||||
bench.bench_file_open(runs=5)
|
||||
bench.bench_read_throughput()
|
||||
bench.bench_memory_usage()
|
||||
|
||||
# Save results
|
||||
bench.save_results()
|
||||
|
||||
finally:
|
||||
bench.teardown()
|
||||
|
||||
# Print summary
|
||||
print("\n" + "=" * 60)
|
||||
print("SUMMARY")
|
||||
print("=" * 60)
|
||||
for r in bench.results:
|
||||
status = "OK" if not r.error else "FAIL"
|
||||
mean_str = "{:.3f}ms".format(r.mean * 1000) if r.mean else "N/A"
|
||||
print("{:20} {:6} Mean: {:>12} Error: {}".format(
|
||||
r.name, status, mean_str, r.error or "None"))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,107 +0,0 @@
|
||||
# MusicFS Configuration
|
||||
# Copy to /etc/musicfs/config.toml or ~/.config/musicfs/config.toml
|
||||
|
||||
# Required: where to mount the virtual filesystem
|
||||
mount_point = "/mnt/music"
|
||||
|
||||
# Required: directory for cache data (CAS chunks, metadata, search index)
|
||||
cache_dir = "/var/cache/musicfs"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Origins - music sources (at least one required)
|
||||
# Supported types: local, nfs, smb, s3, sftp
|
||||
# Lower priority number = preferred source for failover
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[[origins]]
|
||||
id = "local-music"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
enabled = true
|
||||
path = "/home/user/Music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas-nfs"
|
||||
origin_type = "nfs"
|
||||
priority = 2
|
||||
enabled = true
|
||||
path = "/mnt/nas/music"
|
||||
|
||||
[[origins]]
|
||||
id = "nas-smb"
|
||||
origin_type = "smb"
|
||||
priority = 3
|
||||
enabled = false
|
||||
path = "/mnt/smb/music"
|
||||
|
||||
[[origins]]
|
||||
id = "cloud-backup"
|
||||
origin_type = "s3"
|
||||
priority = 10
|
||||
enabled = false
|
||||
bucket = "my-music-backup"
|
||||
region = "us-east-1"
|
||||
|
||||
[[origins]]
|
||||
id = "remote-server"
|
||||
origin_type = "sftp"
|
||||
priority = 10
|
||||
enabled = false
|
||||
host = "music.example.com"
|
||||
port = 22
|
||||
user = "musicfs"
|
||||
path = "/srv/music"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Cache settings
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[cache]
|
||||
# In-memory metadata cache size (artist/album/track info)
|
||||
metadata_cache_mb = 100
|
||||
|
||||
# On-disk content cache size (audio chunks)
|
||||
content_cache_gb = 10
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Health monitoring for origin failover
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[health]
|
||||
# How often to check origin health
|
||||
check_interval_secs = 30
|
||||
|
||||
# Timeout for health check probes
|
||||
timeout_ms = 5000
|
||||
|
||||
# Consecutive failures before marking origin unhealthy
|
||||
unhealthy_threshold = 3
|
||||
|
||||
# Per-origin type thresholds (overrides unhealthy_threshold)
|
||||
[health.per_origin_thresholds]
|
||||
local = 1
|
||||
nfs = 3
|
||||
smb = 3
|
||||
s3 = 3
|
||||
sftp = 3
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[logging]
|
||||
# Directory for log files
|
||||
log_dir = "/var/log/musicfs"
|
||||
|
||||
# Output logs as JSON (for log aggregators)
|
||||
json_output = false
|
||||
|
||||
# Send logs to systemd journal
|
||||
journald = true
|
||||
|
||||
# Log level filter (tracing format)
|
||||
# Examples: "info", "debug", "musicfs=debug,warn", "musicfs_fuse=trace"
|
||||
level = "musicfs=info,warn"
|
||||
|
||||
# Trace sampling rate for performance tracing (0.0 to 1.0)
|
||||
trace_sample_rate = 1.0
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
mount_point = "./dev/music"
|
||||
cache_dir = "./dev/cache/musicfs"
|
||||
|
||||
[[origins]]
|
||||
id = "local-storage"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
enabled = true
|
||||
path = "/home/fujin/.local/share/docker/volumes/containers_downloads/_data"
|
||||
|
||||
[cache]
|
||||
metadata_cache_mb = 100
|
||||
content_cache_gb = 10
|
||||
|
||||
[health]
|
||||
check_interval_secs = 30
|
||||
timeout_ms = 5000
|
||||
unhealthy_threshold = 3
|
||||
|
||||
[logging]
|
||||
log_dir = "./dev/log"
|
||||
json_output = false
|
||||
journald = true
|
||||
level = "musicfs=info,warn"
|
||||
trace_sample_rate = 1.0
|
||||
@@ -1,25 +0,0 @@
|
||||
[package]
|
||||
name = "musicfs-cache"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-metadata = { path = "../musicfs-metadata" }
|
||||
bytes.workspace = true
|
||||
rusqlite = { workspace = true, features = ["bundled"] }
|
||||
sled.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
image.workspace = true
|
||||
lofty = "0.24"
|
||||
parking_lot.workspace = true
|
||||
chrono.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -1,213 +0,0 @@
|
||||
use image::ImageFormat;
|
||||
use musicfs_cas::CasStore;
|
||||
use musicfs_core::ChunkHash;
|
||||
use musicfs_metadata::artwork::{ArtSize, ArtType, Artwork};
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
const MAX_ARTWORK_INPUT_SIZE: usize = 10 * 1024 * 1024;
|
||||
|
||||
pub struct ArtworkCache {
|
||||
store: Arc<CasStore>,
|
||||
db_path: std::path::PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct CachedArtwork {
|
||||
pub file_id: i64,
|
||||
pub art_type: String,
|
||||
pub chunk_hash: ChunkHash,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
impl ArtworkCache {
|
||||
pub fn new(store: Arc<CasStore>, db_path: &Path) -> Result<Self, ArtworkError> {
|
||||
let db = rusqlite::Connection::open(db_path)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS artwork (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL,
|
||||
art_type TEXT NOT NULL,
|
||||
chunk_hash TEXT NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
UNIQUE(file_id, art_type)
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
info!(path = ?db_path, "Artwork cache opened");
|
||||
Ok(Self {
|
||||
store,
|
||||
db_path: db_path.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn store(&self, file_id: i64, artwork: &Artwork) -> Result<ChunkHash, ArtworkError> {
|
||||
trace!(
|
||||
file_id = file_id,
|
||||
size_bytes = artwork.data.len(),
|
||||
"Storing artwork"
|
||||
);
|
||||
if artwork.data.len() > MAX_ARTWORK_INPUT_SIZE {
|
||||
warn!(
|
||||
file_id = file_id,
|
||||
size = artwork.data.len(),
|
||||
max = MAX_ARTWORK_INPUT_SIZE,
|
||||
"Artwork too large"
|
||||
);
|
||||
return Err(ArtworkError::ImageTooLarge(artwork.data.len()));
|
||||
}
|
||||
|
||||
let hash = self.store.put(&artwork.data).await?;
|
||||
|
||||
let art_type_str = match artwork.art_type {
|
||||
ArtType::Front => "front",
|
||||
ArtType::Back => "back",
|
||||
ArtType::Other => "other",
|
||||
};
|
||||
|
||||
let db_path = self.db_path.clone();
|
||||
let art_type_clone = art_type_str.to_string();
|
||||
let hash_hex = hash.to_hex();
|
||||
let width = artwork.width;
|
||||
let height = artwork.height;
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let db = rusqlite::Connection::open(&db_path)?;
|
||||
db.execute(
|
||||
"INSERT OR REPLACE INTO artwork
|
||||
(file_id, art_type, chunk_hash, width, height)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
rusqlite::params![file_id, art_type_clone, hash_hex, width, height],
|
||||
)?;
|
||||
Ok::<_, ArtworkError>(())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))??;
|
||||
|
||||
debug!("Cached artwork for file {}", file_id);
|
||||
Ok(hash)
|
||||
}
|
||||
|
||||
pub async fn get(
|
||||
&self,
|
||||
file_id: i64,
|
||||
art_type: &str,
|
||||
size: ArtSize,
|
||||
) -> Result<Option<Vec<u8>>, ArtworkError> {
|
||||
trace!(file_id = file_id, art_type = %art_type, "Getting artwork");
|
||||
let db_path = self.db_path.clone();
|
||||
let art_type_clone = art_type.to_string();
|
||||
|
||||
let hash_hex: Option<String> = tokio::task::spawn_blocking(move || {
|
||||
let db = rusqlite::Connection::open(&db_path)?;
|
||||
db.query_row(
|
||||
"SELECT chunk_hash FROM artwork WHERE file_id = ?1 AND art_type = ?2",
|
||||
rusqlite::params![file_id, art_type_clone],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.ok()
|
||||
.ok_or(ArtworkError::NotFound)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))?
|
||||
.ok();
|
||||
|
||||
match hash_hex {
|
||||
Some(hex) => {
|
||||
trace!(file_id = file_id, "Artwork cache hit");
|
||||
let hash = ChunkHash::from_hex(&hex).ok_or(ArtworkError::InvalidHash)?;
|
||||
let data = self.store.get(&hash).await?;
|
||||
|
||||
match size {
|
||||
ArtSize::Full => Ok(Some(data.to_vec())),
|
||||
ArtSize::Thumbnail | ArtSize::Medium => {
|
||||
let resized = self.resize_on_demand(&data, size)?;
|
||||
Ok(Some(resized))
|
||||
}
|
||||
}
|
||||
}
|
||||
None => {
|
||||
trace!(file_id = file_id, "Artwork cache miss");
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn has(&self, file_id: i64, art_type: &str) -> Result<bool, ArtworkError> {
|
||||
let db_path = self.db_path.clone();
|
||||
let art_type_clone = art_type.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let db = rusqlite::Connection::open(&db_path)?;
|
||||
let count: i64 = db.query_row(
|
||||
"SELECT COUNT(*) FROM artwork WHERE file_id = ?1 AND art_type = ?2",
|
||||
rusqlite::params![file_id, art_type_clone],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok(count > 0)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ArtworkError::SpawnBlocking(e.to_string()))?
|
||||
}
|
||||
|
||||
fn resize_on_demand(&self, data: &[u8], size: ArtSize) -> Result<Vec<u8>, ArtworkError> {
|
||||
let max_dim = size.max_dimension().unwrap_or(300);
|
||||
let img = image::load_from_memory(data).map_err(|_| ArtworkError::InvalidImage)?;
|
||||
|
||||
if img.width() <= max_dim && img.height() <= max_dim {
|
||||
return Ok(data.to_vec());
|
||||
}
|
||||
|
||||
let resized = img.thumbnail(max_dim, max_dim);
|
||||
let mut output = Vec::new();
|
||||
let mut cursor = Cursor::new(&mut output);
|
||||
resized
|
||||
.write_to(&mut cursor, ImageFormat::Jpeg)
|
||||
.map_err(|_| ArtworkError::ResizeFailed)?;
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ArtworkError {
|
||||
#[error("database error: {0}")]
|
||||
Database(#[from] rusqlite::Error),
|
||||
|
||||
#[error("CAS error: {0}")]
|
||||
Cas(#[from] musicfs_cas::CasError),
|
||||
|
||||
#[error("invalid hash")]
|
||||
InvalidHash,
|
||||
|
||||
#[error("artwork not found")]
|
||||
NotFound,
|
||||
|
||||
#[error("image too large: {0} bytes (max 10MB)")]
|
||||
ImageTooLarge(usize),
|
||||
|
||||
#[error("invalid image data")]
|
||||
InvalidImage,
|
||||
|
||||
#[error("resize failed")]
|
||||
ResizeFailed,
|
||||
|
||||
#[error("spawn_blocking error: {0}")]
|
||||
SpawnBlocking(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_max_artwork_size() {
|
||||
assert_eq!(MAX_ARTWORK_INPUT_SIZE, 10 * 1024 * 1024);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,155 +0,0 @@
|
||||
use musicfs_cas::CasStore;
|
||||
use musicfs_core::ChunkHash;
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Instant;
|
||||
use tracing::info;
|
||||
|
||||
pub trait EvictionPolicy: Send + Sync {
|
||||
fn record_access(&self, hash: ChunkHash);
|
||||
fn select_victims(&self, count: usize) -> Vec<ChunkHash>;
|
||||
fn remove(&self, hash: &ChunkHash);
|
||||
}
|
||||
|
||||
pub struct LruEviction {
|
||||
access_times: RwLock<BTreeMap<Instant, ChunkHash>>,
|
||||
hash_to_time: RwLock<std::collections::HashMap<ChunkHash, Instant>>,
|
||||
}
|
||||
|
||||
impl LruEviction {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
access_times: RwLock::new(BTreeMap::new()),
|
||||
hash_to_time: RwLock::new(std::collections::HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn evict_to_target(
|
||||
&self,
|
||||
store: &CasStore,
|
||||
target_size: u64,
|
||||
) -> Result<u64, EvictionError> {
|
||||
let mut bytes_freed = 0u64;
|
||||
|
||||
while store.current_size() > target_size {
|
||||
let victims = self.select_victims(10);
|
||||
|
||||
if victims.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
for hash in victims {
|
||||
if let Ok(data) = store.get(&hash).await {
|
||||
bytes_freed += data.len() as u64;
|
||||
store.delete(&hash).await?;
|
||||
self.remove(&hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if bytes_freed > 0 {
|
||||
info!("Evicted {} bytes from cache", bytes_freed);
|
||||
}
|
||||
|
||||
Ok(bytes_freed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LruEviction {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl EvictionPolicy for LruEviction {
|
||||
fn record_access(&self, hash: ChunkHash) {
|
||||
let now = Instant::now();
|
||||
let mut times = self.access_times.write();
|
||||
let mut h2t = self.hash_to_time.write();
|
||||
|
||||
if let Some(old_time) = h2t.remove(&hash) {
|
||||
times.remove(&old_time);
|
||||
}
|
||||
|
||||
times.insert(now, hash);
|
||||
h2t.insert(hash, now);
|
||||
}
|
||||
|
||||
fn select_victims(&self, count: usize) -> Vec<ChunkHash> {
|
||||
let times = self.access_times.read();
|
||||
times.values().take(count).copied().collect()
|
||||
}
|
||||
|
||||
fn remove(&self, hash: &ChunkHash) {
|
||||
let mut times = self.access_times.write();
|
||||
let mut h2t = self.hash_to_time.write();
|
||||
|
||||
if let Some(time) = h2t.remove(hash) {
|
||||
times.remove(&time);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum EvictionError {
|
||||
#[error("CAS error: {0}")]
|
||||
Cas(#[from] musicfs_cas::CasError),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_lru_access_order() {
|
||||
let lru = LruEviction::new();
|
||||
|
||||
let h1 = ChunkHash::from_bytes(b"chunk1");
|
||||
let h2 = ChunkHash::from_bytes(b"chunk2");
|
||||
let h3 = ChunkHash::from_bytes(b"chunk3");
|
||||
|
||||
lru.record_access(h1);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
lru.record_access(h2);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
lru.record_access(h3);
|
||||
|
||||
let victims = lru.select_victims(2);
|
||||
assert_eq!(victims.len(), 2);
|
||||
assert_eq!(victims[0], h1);
|
||||
assert_eq!(victims[1], h2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lru_reaccess_updates_order() {
|
||||
let lru = LruEviction::new();
|
||||
|
||||
let h1 = ChunkHash::from_bytes(b"chunk1");
|
||||
let h2 = ChunkHash::from_bytes(b"chunk2");
|
||||
|
||||
lru.record_access(h1);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
lru.record_access(h2);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
lru.record_access(h1);
|
||||
|
||||
let victims = lru.select_victims(1);
|
||||
assert_eq!(victims[0], h2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lru_remove() {
|
||||
let lru = LruEviction::new();
|
||||
|
||||
let h1 = ChunkHash::from_bytes(b"chunk1");
|
||||
let h2 = ChunkHash::from_bytes(b"chunk2");
|
||||
|
||||
lru.record_access(h1);
|
||||
lru.record_access(h2);
|
||||
lru.remove(&h1);
|
||||
|
||||
let victims = lru.select_victims(10);
|
||||
assert_eq!(victims.len(), 1);
|
||||
assert_eq!(victims[0], h2);
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
use crate::FormatLayout;
|
||||
use musicfs_core::AudioMeta;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Error types for format handling operations
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FormatError {
|
||||
#[error("Unsupported format")]
|
||||
UnsupportedFormat,
|
||||
|
||||
#[error("Invalid data: {0}")]
|
||||
InvalidData(String),
|
||||
|
||||
#[error("Synthesis failed: {0}")]
|
||||
SynthesisFailed(String),
|
||||
}
|
||||
|
||||
/// Trait for format-specific metadata handling.
|
||||
///
|
||||
/// Implementations handle:
|
||||
/// 1. Analyzing original files to find audio boundaries
|
||||
/// 2. Synthesizing new headers from database metadata
|
||||
pub trait FormatHandler: Send + Sync + 'static {
|
||||
/// Unique identifier for this handler
|
||||
fn id(&self) -> &'static str;
|
||||
|
||||
/// Human-readable name
|
||||
fn name(&self) -> &'static str;
|
||||
|
||||
/// File extensions this handler supports
|
||||
fn extensions(&self) -> &[&'static str];
|
||||
|
||||
/// MIME types this handler supports
|
||||
fn mime_types(&self) -> &[&'static str];
|
||||
|
||||
/// Analyze file bytes to determine audio layout
|
||||
fn analyze(
|
||||
&self,
|
||||
data: &[u8],
|
||||
file_size: u64,
|
||||
) -> std::result::Result<FormatLayout, FormatError>;
|
||||
|
||||
/// Synthesize header bytes from metadata. Called on every read().
|
||||
fn synthesize(
|
||||
&self,
|
||||
metadata: &AudioMeta,
|
||||
layout: &FormatLayout,
|
||||
) -> std::result::Result<Vec<u8>, FormatError>;
|
||||
|
||||
/// Extract metadata from header bytes (for initial ingest)
|
||||
fn extract(&self, data: &[u8]) -> std::result::Result<AudioMeta, FormatError>;
|
||||
|
||||
/// Estimate header size without full synthesis (for getattr)
|
||||
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
|
||||
10 * 1024 // 10KB default
|
||||
}
|
||||
}
|
||||
|
||||
/// Registry for format handlers
|
||||
pub struct FormatHandlerRegistry {
|
||||
handlers: HashMap<String, Arc<dyn FormatHandler>>,
|
||||
extension_map: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl FormatHandlerRegistry {
|
||||
/// Create empty registry
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
handlers: HashMap::new(),
|
||||
extension_map: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a format handler
|
||||
pub fn register(&mut self, handler: Arc<dyn FormatHandler>) {
|
||||
let id = handler.id().to_string();
|
||||
|
||||
// Map extensions to handler ID
|
||||
for ext in handler.extensions() {
|
||||
self.extension_map.insert(ext.to_string(), id.clone());
|
||||
}
|
||||
|
||||
self.handlers.insert(id, handler);
|
||||
}
|
||||
|
||||
/// Get handler by file extension
|
||||
pub fn get_by_extension(&self, ext: &str) -> Option<Arc<dyn FormatHandler>> {
|
||||
let id = self.extension_map.get(ext)?;
|
||||
self.handlers.get(id).cloned()
|
||||
}
|
||||
|
||||
/// Get handler by format ID
|
||||
pub fn get_by_format(&self, format: &str) -> Option<Arc<dyn FormatHandler>> {
|
||||
self.handlers.get(format).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FormatHandlerRegistry {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
use musicfs_core::AudioFormat;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Describes the byte layout of an audio file for overlay splicing.
|
||||
///
|
||||
/// This struct tracks where the audio data begins and ends in the origin file,
|
||||
/// allowing the OverlayReader to splice synthetic headers with original audio.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FormatLayout {
|
||||
/// Byte offset where audio data begins in the origin file
|
||||
pub audio_start: u64,
|
||||
|
||||
/// Byte offset where audio data ends in the origin file
|
||||
pub audio_end: u64,
|
||||
|
||||
/// Audio format (from musicfs-core)
|
||||
pub format: AudioFormat,
|
||||
|
||||
/// Format-specific data (e.g., FLAC STREAMINFO block, MP4 stco offsets)
|
||||
/// Stored as raw bytes, interpreted by format handlers
|
||||
pub format_data: Option<Vec<u8>>,
|
||||
}
|
||||
@@ -1,886 +0,0 @@
|
||||
//! FLAC format handler for metadata synthesis.
|
||||
//!
|
||||
//! FLAC files use Vorbis comments for metadata. The file structure is:
|
||||
//! - "fLaC" marker (4 bytes)
|
||||
//! - STREAMINFO block (mandatory, 38 bytes total: 4 header + 34 data)
|
||||
//! - Optional metadata blocks (VORBIS_COMMENT, PICTURE, PADDING, etc.)
|
||||
//! - Audio frames
|
||||
//!
|
||||
//! CRITICAL: STREAMINFO must be preserved from the original file as it contains
|
||||
//! MD5 checksum, sample count, and audio properties that must match the audio data.
|
||||
|
||||
use crate::{FormatError, FormatHandler, FormatLayout};
|
||||
use lofty::config::ParseOptions;
|
||||
use lofty::file::AudioFile;
|
||||
use lofty::flac::FlacFile;
|
||||
use lofty::ogg::VorbisComments;
|
||||
use lofty::tag::Accessor;
|
||||
use musicfs_core::{AudioFormat, AudioMeta};
|
||||
use std::borrow::Cow;
|
||||
use std::io::Cursor;
|
||||
|
||||
/// FLAC stream marker: "fLaC" in ASCII
|
||||
const FLAC_MARKER: &[u8; 4] = b"fLaC";
|
||||
|
||||
/// FLAC metadata block types
|
||||
const BLOCK_TYPE_STREAMINFO: u8 = 0;
|
||||
const BLOCK_TYPE_VORBIS_COMMENT: u8 = 4;
|
||||
|
||||
/// STREAMINFO block data size (always 34 bytes)
|
||||
const STREAMINFO_DATA_SIZE: usize = 34;
|
||||
|
||||
/// Metadata block header size (1 byte type/flags + 3 bytes length)
|
||||
const BLOCK_HEADER_SIZE: usize = 4;
|
||||
|
||||
/// Full STREAMINFO block size (header + data)
|
||||
const STREAMINFO_BLOCK_SIZE: usize = BLOCK_HEADER_SIZE + STREAMINFO_DATA_SIZE;
|
||||
|
||||
pub struct FlacHandler;
|
||||
|
||||
impl FlacHandler {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
/// Parse FLAC metadata block header.
|
||||
/// Returns (is_last, block_type, block_size).
|
||||
fn parse_block_header(data: &[u8]) -> Option<(bool, u8, usize)> {
|
||||
if data.len() < BLOCK_HEADER_SIZE {
|
||||
return None;
|
||||
}
|
||||
let is_last = (data[0] & 0x80) != 0;
|
||||
let block_type = data[0] & 0x7F;
|
||||
let block_size =
|
||||
((data[1] as usize) << 16) | ((data[2] as usize) << 8) | (data[3] as usize);
|
||||
Some((is_last, block_type, block_size))
|
||||
}
|
||||
|
||||
/// Write a metadata block header.
|
||||
fn write_block_header(is_last: bool, block_type: u8, size: usize) -> [u8; 4] {
|
||||
let type_byte = if is_last {
|
||||
block_type | 0x80
|
||||
} else {
|
||||
block_type
|
||||
};
|
||||
[
|
||||
type_byte,
|
||||
((size >> 16) & 0xFF) as u8,
|
||||
((size >> 8) & 0xFF) as u8,
|
||||
(size & 0xFF) as u8,
|
||||
]
|
||||
}
|
||||
|
||||
/// Build Vorbis comments from AudioMeta.
|
||||
fn build_vorbis_comments(metadata: &AudioMeta) -> VorbisComments {
|
||||
let mut tag = VorbisComments::default();
|
||||
|
||||
// Basic fields (using Accessor trait)
|
||||
if let Some(ref title) = metadata.title {
|
||||
tag.set_title(title.clone());
|
||||
}
|
||||
if let Some(ref artist) = metadata.artist {
|
||||
tag.set_artist(artist.clone());
|
||||
}
|
||||
if let Some(ref album) = metadata.album {
|
||||
tag.set_album(album.clone());
|
||||
}
|
||||
if let Some(ref genre) = metadata.genre {
|
||||
tag.set_genre(genre.clone());
|
||||
}
|
||||
|
||||
// Album artist
|
||||
if let Some(ref album_artist) = metadata.album_artist {
|
||||
tag.insert("ALBUMARTIST".to_string(), album_artist.clone());
|
||||
}
|
||||
|
||||
// Year/Date
|
||||
if let Some(ref date) = metadata.date {
|
||||
tag.insert("DATE".to_string(), date.clone());
|
||||
} else if let Some(year) = metadata.year {
|
||||
tag.insert("DATE".to_string(), year.to_string());
|
||||
}
|
||||
|
||||
// Track/Disc numbers
|
||||
if let Some(track) = metadata.track {
|
||||
tag.insert("TRACKNUMBER".to_string(), track.to_string());
|
||||
}
|
||||
if let Some(track_total) = metadata.track_total {
|
||||
tag.insert("TRACKTOTAL".to_string(), track_total.to_string());
|
||||
}
|
||||
if let Some(disc) = metadata.disc {
|
||||
tag.insert("DISCNUMBER".to_string(), disc.to_string());
|
||||
}
|
||||
if let Some(disc_total) = metadata.disc_total {
|
||||
tag.insert("DISCTOTAL".to_string(), disc_total.to_string());
|
||||
}
|
||||
|
||||
// Extended metadata
|
||||
if let Some(ref composer) = metadata.composer {
|
||||
tag.insert("COMPOSER".to_string(), composer.clone());
|
||||
}
|
||||
if let Some(ref comment) = metadata.comment {
|
||||
tag.insert("COMMENT".to_string(), comment.clone());
|
||||
}
|
||||
if let Some(ref lyrics) = metadata.lyrics {
|
||||
tag.insert("LYRICS".to_string(), lyrics.clone());
|
||||
}
|
||||
if let Some(ref copyright) = metadata.copyright {
|
||||
tag.insert("COPYRIGHT".to_string(), copyright.clone());
|
||||
}
|
||||
if let Some(compilation) = metadata.compilation {
|
||||
tag.insert(
|
||||
"COMPILATION".to_string(),
|
||||
if compilation { "1" } else { "0" }.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Sort fields
|
||||
if let Some(ref title_sort) = metadata.title_sort {
|
||||
tag.insert("TITLESORT".to_string(), title_sort.clone());
|
||||
}
|
||||
if let Some(ref artist_sort) = metadata.artist_sort {
|
||||
tag.insert("ARTISTSORT".to_string(), artist_sort.clone());
|
||||
}
|
||||
if let Some(ref album_sort) = metadata.album_sort {
|
||||
tag.insert("ALBUMSORT".to_string(), album_sort.clone());
|
||||
}
|
||||
if let Some(ref album_artist_sort) = metadata.album_artist_sort {
|
||||
tag.insert("ALBUMARTISTSORT".to_string(), album_artist_sort.clone());
|
||||
}
|
||||
|
||||
// MusicBrainz IDs
|
||||
if let Some(ref mb_recording_id) = metadata.mb_recording_id {
|
||||
tag.insert("MUSICBRAINZ_TRACKID".to_string(), mb_recording_id.clone());
|
||||
}
|
||||
if let Some(ref mb_album_id) = metadata.mb_album_id {
|
||||
tag.insert("MUSICBRAINZ_ALBUMID".to_string(), mb_album_id.clone());
|
||||
}
|
||||
if let Some(ref mb_artist_id) = metadata.mb_artist_id {
|
||||
tag.insert("MUSICBRAINZ_ARTISTID".to_string(), mb_artist_id.clone());
|
||||
}
|
||||
if let Some(ref mb_album_artist_id) = metadata.mb_album_artist_id {
|
||||
tag.insert(
|
||||
"MUSICBRAINZ_ALBUMARTISTID".to_string(),
|
||||
mb_album_artist_id.clone(),
|
||||
);
|
||||
}
|
||||
if let Some(ref mb_release_group_id) = metadata.mb_release_group_id {
|
||||
tag.insert(
|
||||
"MUSICBRAINZ_RELEASEGROUPID".to_string(),
|
||||
mb_release_group_id.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
// ReplayGain
|
||||
if let Some(gain) = metadata.replaygain_track_gain {
|
||||
tag.insert(
|
||||
"REPLAYGAIN_TRACK_GAIN".to_string(),
|
||||
format!("{:.2} dB", gain),
|
||||
);
|
||||
}
|
||||
if let Some(peak) = metadata.replaygain_track_peak {
|
||||
tag.insert("REPLAYGAIN_TRACK_PEAK".to_string(), format!("{:.6}", peak));
|
||||
}
|
||||
if let Some(gain) = metadata.replaygain_album_gain {
|
||||
tag.insert(
|
||||
"REPLAYGAIN_ALBUM_GAIN".to_string(),
|
||||
format!("{:.2} dB", gain),
|
||||
);
|
||||
}
|
||||
if let Some(peak) = metadata.replaygain_album_peak {
|
||||
tag.insert("REPLAYGAIN_ALBUM_PEAK".to_string(), format!("{:.6}", peak));
|
||||
}
|
||||
|
||||
// Encoder
|
||||
if let Some(ref encoder) = metadata.encoder {
|
||||
tag.insert("ENCODER".to_string(), encoder.clone());
|
||||
}
|
||||
|
||||
tag
|
||||
}
|
||||
|
||||
/// Serialize Vorbis comments to bytes (without block header).
|
||||
/// Format: vendor_length (4 LE) + vendor + comment_count (4 LE) + comments
|
||||
fn serialize_vorbis_comments(tag: &VorbisComments) -> Vec<u8> {
|
||||
let vendor = tag.vendor();
|
||||
let vendor = if vendor.is_empty() { "musicfs" } else { vendor };
|
||||
let mut data = Vec::new();
|
||||
|
||||
// Vendor string (little-endian length + UTF-8 string)
|
||||
let vendor_bytes = vendor.as_bytes();
|
||||
data.extend_from_slice(&(vendor_bytes.len() as u32).to_le_bytes());
|
||||
data.extend_from_slice(vendor_bytes);
|
||||
|
||||
// Collect all comments
|
||||
let comments: Vec<_> = tag.items().collect();
|
||||
data.extend_from_slice(&(comments.len() as u32).to_le_bytes());
|
||||
|
||||
for (key, value) in comments {
|
||||
let comment = format!("{}={}", key, value);
|
||||
let comment_bytes = comment.as_bytes();
|
||||
data.extend_from_slice(&(comment_bytes.len() as u32).to_le_bytes());
|
||||
data.extend_from_slice(comment_bytes);
|
||||
}
|
||||
|
||||
data
|
||||
}
|
||||
|
||||
/// Extract metadata from Vorbis comments tag.
|
||||
fn extract_from_vorbis_comments(tag: &VorbisComments) -> AudioMeta {
|
||||
let mut meta = AudioMeta::default();
|
||||
meta.format = AudioFormat::Flac;
|
||||
|
||||
// Basic fields (using Accessor trait)
|
||||
meta.title = tag.title().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.artist = tag.artist().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.album = tag.album().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.genre = tag.genre().map(|c: Cow<'_, str>| c.into_owned());
|
||||
|
||||
// Album artist
|
||||
meta.album_artist = tag.get("ALBUMARTIST").map(String::from);
|
||||
|
||||
// Date/Year
|
||||
meta.date = tag.get("DATE").map(String::from);
|
||||
if let Some(ref date) = meta.date {
|
||||
if let Some(year_str) = date.split('-').next() {
|
||||
meta.year = year_str.parse().ok();
|
||||
}
|
||||
}
|
||||
|
||||
// Track/Disc numbers
|
||||
meta.track = tag.get("TRACKNUMBER").and_then(|s| s.parse().ok());
|
||||
meta.track_total = tag.get("TRACKTOTAL").and_then(|s| s.parse().ok());
|
||||
meta.disc = tag.get("DISCNUMBER").and_then(|s| s.parse().ok());
|
||||
meta.disc_total = tag.get("DISCTOTAL").and_then(|s| s.parse().ok());
|
||||
|
||||
// Extended metadata
|
||||
meta.composer = tag.get("COMPOSER").map(String::from);
|
||||
meta.comment = tag.get("COMMENT").map(String::from);
|
||||
meta.lyrics = tag.get("LYRICS").map(String::from);
|
||||
meta.copyright = tag.get("COPYRIGHT").map(String::from);
|
||||
meta.compilation = tag
|
||||
.get("COMPILATION")
|
||||
.map(|s| s == "1" || s.eq_ignore_ascii_case("true"));
|
||||
|
||||
// Sort fields
|
||||
meta.title_sort = tag.get("TITLESORT").map(String::from);
|
||||
meta.artist_sort = tag.get("ARTISTSORT").map(String::from);
|
||||
meta.album_sort = tag.get("ALBUMSORT").map(String::from);
|
||||
meta.album_artist_sort = tag.get("ALBUMARTISTSORT").map(String::from);
|
||||
|
||||
// MusicBrainz IDs
|
||||
meta.mb_recording_id = tag.get("MUSICBRAINZ_TRACKID").map(String::from);
|
||||
meta.mb_album_id = tag.get("MUSICBRAINZ_ALBUMID").map(String::from);
|
||||
meta.mb_artist_id = tag.get("MUSICBRAINZ_ARTISTID").map(String::from);
|
||||
meta.mb_album_artist_id = tag.get("MUSICBRAINZ_ALBUMARTISTID").map(String::from);
|
||||
meta.mb_release_group_id = tag.get("MUSICBRAINZ_RELEASEGROUPID").map(String::from);
|
||||
|
||||
// ReplayGain
|
||||
meta.replaygain_track_gain = tag
|
||||
.get("REPLAYGAIN_TRACK_GAIN")
|
||||
.and_then(|s| Self::parse_replaygain_value(s));
|
||||
meta.replaygain_track_peak = tag
|
||||
.get("REPLAYGAIN_TRACK_PEAK")
|
||||
.and_then(|s| s.parse().ok());
|
||||
meta.replaygain_album_gain = tag
|
||||
.get("REPLAYGAIN_ALBUM_GAIN")
|
||||
.and_then(|s| Self::parse_replaygain_value(s));
|
||||
meta.replaygain_album_peak = tag
|
||||
.get("REPLAYGAIN_ALBUM_PEAK")
|
||||
.and_then(|s| s.parse().ok());
|
||||
|
||||
// Encoder
|
||||
meta.encoder = tag.get("ENCODER").map(String::from);
|
||||
|
||||
meta
|
||||
}
|
||||
|
||||
/// Parse ReplayGain value, stripping optional "dB" suffix.
|
||||
fn parse_replaygain_value(value: &str) -> Option<f32> {
|
||||
value
|
||||
.trim()
|
||||
.trim_end_matches(" dB")
|
||||
.trim_end_matches("dB")
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FlacHandler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl FormatHandler for FlacHandler {
|
||||
fn id(&self) -> &'static str {
|
||||
"flac"
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"FLAC"
|
||||
}
|
||||
|
||||
fn extensions(&self) -> &[&'static str] {
|
||||
&["flac"]
|
||||
}
|
||||
|
||||
fn mime_types(&self) -> &[&'static str] {
|
||||
&["audio/flac", "audio/x-flac"]
|
||||
}
|
||||
|
||||
fn analyze(&self, data: &[u8], file_size: u64) -> Result<FormatLayout, FormatError> {
|
||||
// Verify FLAC marker
|
||||
if data.len() < FLAC_MARKER.len() || &data[0..4] != FLAC_MARKER {
|
||||
return Err(FormatError::InvalidData("Not a FLAC file".to_string()));
|
||||
}
|
||||
|
||||
let mut offset = FLAC_MARKER.len();
|
||||
let mut streaminfo_data: Option<Vec<u8>> = None;
|
||||
|
||||
// Parse metadata blocks to find audio_start and extract STREAMINFO
|
||||
loop {
|
||||
if offset + BLOCK_HEADER_SIZE > data.len() {
|
||||
return Err(FormatError::InvalidData(
|
||||
"Truncated FLAC metadata".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let (is_last, block_type, block_size) = Self::parse_block_header(&data[offset..])
|
||||
.ok_or_else(|| FormatError::InvalidData("Invalid block header".to_string()))?;
|
||||
|
||||
// Extract STREAMINFO block data (without header)
|
||||
if block_type == BLOCK_TYPE_STREAMINFO {
|
||||
if block_size != STREAMINFO_DATA_SIZE {
|
||||
return Err(FormatError::InvalidData(format!(
|
||||
"Invalid STREAMINFO size: {} (expected {})",
|
||||
block_size, STREAMINFO_DATA_SIZE
|
||||
)));
|
||||
}
|
||||
let data_start = offset + BLOCK_HEADER_SIZE;
|
||||
let data_end = data_start + block_size;
|
||||
if data_end > data.len() {
|
||||
return Err(FormatError::InvalidData(
|
||||
"Truncated STREAMINFO block".to_string(),
|
||||
));
|
||||
}
|
||||
streaminfo_data = Some(data[data_start..data_end].to_vec());
|
||||
}
|
||||
|
||||
offset += BLOCK_HEADER_SIZE + block_size;
|
||||
|
||||
if is_last {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let streaminfo = streaminfo_data
|
||||
.ok_or_else(|| FormatError::InvalidData("Missing STREAMINFO block".to_string()))?;
|
||||
|
||||
Ok(FormatLayout {
|
||||
audio_start: offset as u64,
|
||||
audio_end: file_size,
|
||||
format: AudioFormat::Flac,
|
||||
format_data: Some(streaminfo),
|
||||
})
|
||||
}
|
||||
|
||||
fn synthesize(
|
||||
&self,
|
||||
metadata: &AudioMeta,
|
||||
layout: &FormatLayout,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
// STREAMINFO must be preserved from original
|
||||
let streaminfo_data = layout.format_data.as_ref().ok_or_else(|| {
|
||||
FormatError::SynthesisFailed("Missing STREAMINFO data in layout".to_string())
|
||||
})?;
|
||||
|
||||
if streaminfo_data.len() != STREAMINFO_DATA_SIZE {
|
||||
return Err(FormatError::SynthesisFailed(format!(
|
||||
"Invalid STREAMINFO size: {} (expected {})",
|
||||
streaminfo_data.len(),
|
||||
STREAMINFO_DATA_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
// Build Vorbis comments
|
||||
let vorbis_tag = Self::build_vorbis_comments(metadata);
|
||||
let vorbis_data = Self::serialize_vorbis_comments(&vorbis_tag);
|
||||
|
||||
// Calculate total header size
|
||||
let total_size =
|
||||
FLAC_MARKER.len() + STREAMINFO_BLOCK_SIZE + BLOCK_HEADER_SIZE + vorbis_data.len();
|
||||
let mut buffer = Vec::with_capacity(total_size);
|
||||
|
||||
// Write FLAC marker
|
||||
buffer.extend_from_slice(FLAC_MARKER);
|
||||
|
||||
// Write STREAMINFO block (not last)
|
||||
let streaminfo_header =
|
||||
Self::write_block_header(false, BLOCK_TYPE_STREAMINFO, STREAMINFO_DATA_SIZE);
|
||||
buffer.extend_from_slice(&streaminfo_header);
|
||||
buffer.extend_from_slice(streaminfo_data);
|
||||
|
||||
// Write VORBIS_COMMENT block (last)
|
||||
let vorbis_header =
|
||||
Self::write_block_header(true, BLOCK_TYPE_VORBIS_COMMENT, vorbis_data.len());
|
||||
buffer.extend_from_slice(&vorbis_header);
|
||||
buffer.extend_from_slice(&vorbis_data);
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
fn extract(&self, data: &[u8]) -> Result<AudioMeta, FormatError> {
|
||||
let mut cursor = Cursor::new(data);
|
||||
|
||||
let flac_file = FlacFile::read_from(&mut cursor, ParseOptions::new())
|
||||
.map_err(|e| FormatError::InvalidData(e.to_string()))?;
|
||||
|
||||
let tag = flac_file
|
||||
.vorbis_comments()
|
||||
.ok_or_else(|| FormatError::InvalidData("No Vorbis comments found".to_string()))?;
|
||||
|
||||
Ok(Self::extract_from_vorbis_comments(tag))
|
||||
}
|
||||
|
||||
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
|
||||
// fLaC (4) + STREAMINFO (38) + VORBIS_COMMENT header (4) + typical comments (~4KB)
|
||||
8192
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_test_meta() -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: Some("Test Title".to_string()),
|
||||
artist: Some("Test Artist".to_string()),
|
||||
album: Some("Test Album".to_string()),
|
||||
album_artist: Some("Test Album Artist".to_string()),
|
||||
genre: Some("Rock".to_string()),
|
||||
year: Some(2024),
|
||||
track: Some(5),
|
||||
track_total: Some(12),
|
||||
disc: Some(1),
|
||||
disc_total: Some(2),
|
||||
format: AudioFormat::Flac,
|
||||
date: Some("2024-03-15".to_string()),
|
||||
composer: Some("Test Composer".to_string()),
|
||||
comment: Some("Test Comment".to_string()),
|
||||
lyrics: Some("Test Lyrics\nLine 2".to_string()),
|
||||
copyright: Some("2024 Test Copyright".to_string()),
|
||||
compilation: Some(false),
|
||||
title_sort: Some("Title, Test".to_string()),
|
||||
artist_sort: Some("Artist, Test".to_string()),
|
||||
album_sort: Some("Album, Test".to_string()),
|
||||
album_artist_sort: Some("Album Artist, Test".to_string()),
|
||||
mb_recording_id: Some("rec-12345".to_string()),
|
||||
mb_album_id: Some("alb-12345".to_string()),
|
||||
mb_artist_id: Some("art-12345".to_string()),
|
||||
mb_album_artist_id: Some("albart-12345".to_string()),
|
||||
mb_release_group_id: Some("rg-12345".to_string()),
|
||||
replaygain_track_gain: Some(-6.5),
|
||||
replaygain_track_peak: Some(0.987654),
|
||||
replaygain_album_gain: Some(-5.2),
|
||||
replaygain_album_peak: Some(0.999999),
|
||||
encoder: Some("FLAC 1.4.0".to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a minimal valid FLAC file header for testing.
|
||||
fn make_minimal_flac_header() -> Vec<u8> {
|
||||
let mut data = Vec::new();
|
||||
|
||||
// FLAC marker
|
||||
data.extend_from_slice(b"fLaC");
|
||||
|
||||
// STREAMINFO block (last=true for minimal file)
|
||||
// Header: type=0 (STREAMINFO), last=1, size=34
|
||||
data.push(0x80); // 0x80 = last flag set, type 0
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x22); // 34 bytes
|
||||
|
||||
// STREAMINFO data (34 bytes) - minimal valid values
|
||||
// min_block_size (16 bits) = 4096
|
||||
data.push(0x10);
|
||||
data.push(0x00);
|
||||
// max_block_size (16 bits) = 4096
|
||||
data.push(0x10);
|
||||
data.push(0x00);
|
||||
// min_frame_size (24 bits) = 0 (unknown)
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
// max_frame_size (24 bits) = 0 (unknown)
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
// sample_rate (20 bits) = 44100, channels-1 (3 bits) = 1, bits-1 (5 bits) = 15
|
||||
// 44100 = 0xAC44, channels=2 (1), bits=16 (15)
|
||||
// Packed: SSSS SSSS SSSS SSSS SSSS CCCC CBBB BB
|
||||
// 0xAC44 << 12 | (1 << 9) | (15 << 4) = ...
|
||||
// Let's use simpler encoding:
|
||||
// Byte 0-1: sample_rate high 16 bits of 20
|
||||
// Byte 2: sample_rate low 4 bits | channels 3 bits | bits high 1 bit
|
||||
// Byte 3: bits low 4 bits | total_samples high 4 bits
|
||||
// Actually the format is:
|
||||
// 20 bits sample rate, 3 bits channels-1, 5 bits bits-1, 36 bits total samples
|
||||
// 44100 = 0x0AC44
|
||||
data.push(0x0A); // sample_rate bits 19-12
|
||||
data.push(0xC4); // sample_rate bits 11-4
|
||||
data.push(0x42); // sample_rate bits 3-0 (0x4), channels-1 (0x1=stereo), bits-1 high bit (0)
|
||||
data.push(0xF0); // bits-1 low 4 bits (0xF=15, so 16 bits), total_samples high 4 bits (0)
|
||||
// total_samples (remaining 32 bits) = 0
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
// MD5 signature (128 bits = 16 bytes)
|
||||
data.extend_from_slice(&[0u8; 16]);
|
||||
|
||||
data
|
||||
}
|
||||
|
||||
/// Create a FLAC header with Vorbis comments for testing extract().
|
||||
fn make_flac_with_vorbis_comments() -> Vec<u8> {
|
||||
let mut data = Vec::new();
|
||||
|
||||
// FLAC marker
|
||||
data.extend_from_slice(b"fLaC");
|
||||
|
||||
// STREAMINFO block (not last)
|
||||
data.push(0x00); // type=0, last=0
|
||||
data.push(0x00);
|
||||
data.push(0x00);
|
||||
data.push(0x22); // 34 bytes
|
||||
|
||||
// STREAMINFO data (34 bytes)
|
||||
data.push(0x10);
|
||||
data.push(0x00);
|
||||
data.push(0x10);
|
||||
data.push(0x00);
|
||||
data.extend_from_slice(&[0u8; 6]); // frame sizes
|
||||
data.push(0x0A);
|
||||
data.push(0xC4);
|
||||
data.push(0x42);
|
||||
data.push(0xF0);
|
||||
data.extend_from_slice(&[0u8; 4]); // total samples
|
||||
data.extend_from_slice(&[0u8; 16]); // MD5
|
||||
|
||||
// VORBIS_COMMENT block (last)
|
||||
// Vendor: "test"
|
||||
// Comments: TITLE=Test Song, ARTIST=Test Artist
|
||||
let vendor = b"test";
|
||||
let comments = [
|
||||
b"TITLE=Test Song".as_slice(),
|
||||
b"ARTIST=Test Artist".as_slice(),
|
||||
b"ALBUM=Test Album".as_slice(),
|
||||
b"TRACKNUMBER=3".as_slice(),
|
||||
b"REPLAYGAIN_TRACK_GAIN=-5.50 dB".as_slice(),
|
||||
];
|
||||
|
||||
let mut vorbis_data = Vec::new();
|
||||
// Vendor length (LE)
|
||||
vorbis_data.extend_from_slice(&(vendor.len() as u32).to_le_bytes());
|
||||
vorbis_data.extend_from_slice(vendor);
|
||||
// Comment count (LE)
|
||||
vorbis_data.extend_from_slice(&(comments.len() as u32).to_le_bytes());
|
||||
for comment in &comments {
|
||||
vorbis_data.extend_from_slice(&(comment.len() as u32).to_le_bytes());
|
||||
vorbis_data.extend_from_slice(*comment);
|
||||
}
|
||||
|
||||
// VORBIS_COMMENT header
|
||||
data.push(0x84); // type=4, last=1
|
||||
data.push(((vorbis_data.len() >> 16) & 0xFF) as u8);
|
||||
data.push(((vorbis_data.len() >> 8) & 0xFF) as u8);
|
||||
data.push((vorbis_data.len() & 0xFF) as u8);
|
||||
data.extend_from_slice(&vorbis_data);
|
||||
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_id_and_name() {
|
||||
let handler = FlacHandler::new();
|
||||
assert_eq!(handler.id(), "flac");
|
||||
assert_eq!(handler.name(), "FLAC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extensions_and_mime_types() {
|
||||
let handler = FlacHandler::new();
|
||||
assert_eq!(handler.extensions(), &["flac"]);
|
||||
assert_eq!(handler.mime_types(), &["audio/flac", "audio/x-flac"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_header_size() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = AudioMeta::default();
|
||||
assert_eq!(handler.estimate_header_size(&meta), 8192);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_valid_flac() {
|
||||
let handler = FlacHandler::new();
|
||||
let data = make_minimal_flac_header();
|
||||
let file_size = data.len() as u64 + 1000; // Pretend there's audio data
|
||||
|
||||
let result = handler.analyze(&data, file_size);
|
||||
assert!(result.is_ok(), "analyze failed: {:?}", result.err());
|
||||
|
||||
let layout = result.unwrap();
|
||||
assert_eq!(layout.audio_start, 42); // 4 (marker) + 38 (STREAMINFO)
|
||||
assert_eq!(layout.audio_end, file_size);
|
||||
assert_eq!(layout.format, AudioFormat::Flac);
|
||||
assert!(layout.format_data.is_some());
|
||||
assert_eq!(
|
||||
layout.format_data.as_ref().unwrap().len(),
|
||||
STREAMINFO_DATA_SIZE
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_invalid_marker() {
|
||||
let handler = FlacHandler::new();
|
||||
let data = b"ID3\x04\x00\x00"; // MP3 header, not FLAC
|
||||
|
||||
let result = handler.analyze(data, 1000);
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), FormatError::InvalidData(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_truncated() {
|
||||
let handler = FlacHandler::new();
|
||||
let data = b"fLaC"; // Just the marker, no blocks
|
||||
|
||||
let result = handler.analyze(data, 4);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthesize_creates_valid_flac_header() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = make_test_meta();
|
||||
|
||||
// Create layout with STREAMINFO
|
||||
let original_data = make_minimal_flac_header();
|
||||
let layout = handler
|
||||
.analyze(&original_data, original_data.len() as u64)
|
||||
.unwrap();
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok(), "synthesize failed: {:?}", result.err());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
|
||||
// Verify FLAC marker
|
||||
assert!(bytes.len() >= 4);
|
||||
assert_eq!(&bytes[0..4], b"fLaC");
|
||||
|
||||
// Verify STREAMINFO block header
|
||||
assert_eq!(bytes[4] & 0x7F, BLOCK_TYPE_STREAMINFO); // Type 0
|
||||
assert_eq!(bytes[4] & 0x80, 0); // Not last
|
||||
|
||||
// Verify STREAMINFO size
|
||||
let streaminfo_size =
|
||||
((bytes[5] as usize) << 16) | ((bytes[6] as usize) << 8) | (bytes[7] as usize);
|
||||
assert_eq!(streaminfo_size, STREAMINFO_DATA_SIZE);
|
||||
|
||||
// Verify VORBIS_COMMENT block follows
|
||||
let vorbis_offset = 4 + 4 + STREAMINFO_DATA_SIZE;
|
||||
assert_eq!(bytes[vorbis_offset] & 0x7F, BLOCK_TYPE_VORBIS_COMMENT);
|
||||
assert_eq!(bytes[vorbis_offset] & 0x80, 0x80); // Is last
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthesize_preserves_streaminfo() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = AudioMeta::default();
|
||||
|
||||
// Create layout with specific STREAMINFO
|
||||
let original_data = make_minimal_flac_header();
|
||||
let layout = handler
|
||||
.analyze(&original_data, original_data.len() as u64)
|
||||
.unwrap();
|
||||
let original_streaminfo = layout.format_data.as_ref().unwrap().clone();
|
||||
|
||||
let synthesized = handler.synthesize(&meta, &layout).unwrap();
|
||||
|
||||
// Extract STREAMINFO from synthesized header
|
||||
let streaminfo_start = 4 + 4; // After marker and header
|
||||
let streaminfo_end = streaminfo_start + STREAMINFO_DATA_SIZE;
|
||||
let synthesized_streaminfo = &synthesized[streaminfo_start..streaminfo_end];
|
||||
|
||||
assert_eq!(synthesized_streaminfo, original_streaminfo.as_slice());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthesize_missing_streaminfo() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = AudioMeta::default();
|
||||
let layout = FormatLayout {
|
||||
audio_start: 42,
|
||||
audio_end: 1000,
|
||||
format: AudioFormat::Flac,
|
||||
format_data: None, // Missing STREAMINFO
|
||||
};
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(
|
||||
result.unwrap_err(),
|
||||
FormatError::SynthesisFailed(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_from_flac() {
|
||||
let handler = FlacHandler::new();
|
||||
let data = make_flac_with_vorbis_comments();
|
||||
|
||||
let result = handler.extract(&data);
|
||||
assert!(result.is_ok(), "extract failed: {:?}", result.err());
|
||||
|
||||
let meta = result.unwrap();
|
||||
assert_eq!(meta.title, Some("Test Song".to_string()));
|
||||
assert_eq!(meta.artist, Some("Test Artist".to_string()));
|
||||
assert_eq!(meta.album, Some("Test Album".to_string()));
|
||||
assert_eq!(meta.track, Some(3));
|
||||
assert_eq!(meta.format, AudioFormat::Flac);
|
||||
|
||||
// Check ReplayGain parsing
|
||||
let gain = meta.replaygain_track_gain.unwrap();
|
||||
assert!((gain - (-5.5)).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_and_extract_vorbis_comments() {
|
||||
let original_meta = make_test_meta();
|
||||
let tag = FlacHandler::build_vorbis_comments(&original_meta);
|
||||
let extracted = FlacHandler::extract_from_vorbis_comments(&tag);
|
||||
|
||||
assert_eq!(extracted.title, original_meta.title);
|
||||
assert_eq!(extracted.artist, original_meta.artist);
|
||||
assert_eq!(extracted.album, original_meta.album);
|
||||
assert_eq!(extracted.album_artist, original_meta.album_artist);
|
||||
assert_eq!(extracted.genre, original_meta.genre);
|
||||
assert_eq!(extracted.track, original_meta.track);
|
||||
assert_eq!(extracted.track_total, original_meta.track_total);
|
||||
assert_eq!(extracted.disc, original_meta.disc);
|
||||
assert_eq!(extracted.disc_total, original_meta.disc_total);
|
||||
assert_eq!(extracted.composer, original_meta.composer);
|
||||
assert_eq!(extracted.comment, original_meta.comment);
|
||||
assert_eq!(extracted.lyrics, original_meta.lyrics);
|
||||
assert_eq!(extracted.copyright, original_meta.copyright);
|
||||
assert_eq!(extracted.compilation, original_meta.compilation);
|
||||
assert_eq!(extracted.title_sort, original_meta.title_sort);
|
||||
assert_eq!(extracted.artist_sort, original_meta.artist_sort);
|
||||
assert_eq!(extracted.album_sort, original_meta.album_sort);
|
||||
assert_eq!(extracted.album_artist_sort, original_meta.album_artist_sort);
|
||||
assert_eq!(extracted.mb_recording_id, original_meta.mb_recording_id);
|
||||
assert_eq!(extracted.mb_album_id, original_meta.mb_album_id);
|
||||
assert_eq!(extracted.mb_artist_id, original_meta.mb_artist_id);
|
||||
assert_eq!(
|
||||
extracted.mb_album_artist_id,
|
||||
original_meta.mb_album_artist_id
|
||||
);
|
||||
assert_eq!(
|
||||
extracted.mb_release_group_id,
|
||||
original_meta.mb_release_group_id
|
||||
);
|
||||
assert_eq!(extracted.encoder, original_meta.encoder);
|
||||
|
||||
// ReplayGain values (with tolerance for formatting)
|
||||
let orig_track_gain = original_meta.replaygain_track_gain.unwrap();
|
||||
let ext_track_gain = extracted.replaygain_track_gain.unwrap();
|
||||
assert!((orig_track_gain - ext_track_gain).abs() < 0.01);
|
||||
|
||||
let orig_track_peak = original_meta.replaygain_track_peak.unwrap();
|
||||
let ext_track_peak = extracted.replaygain_track_peak.unwrap();
|
||||
assert!((orig_track_peak - ext_track_peak).abs() < 0.0001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_replaygain_value() {
|
||||
assert_eq!(FlacHandler::parse_replaygain_value("-6.50 dB"), Some(-6.50));
|
||||
assert_eq!(FlacHandler::parse_replaygain_value("-6.50dB"), Some(-6.50));
|
||||
assert_eq!(FlacHandler::parse_replaygain_value("-6.50"), Some(-6.50));
|
||||
assert_eq!(FlacHandler::parse_replaygain_value(" 3.2 dB "), Some(3.2));
|
||||
assert_eq!(FlacHandler::parse_replaygain_value("invalid"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_block_header() {
|
||||
// Not last, type 0, size 34
|
||||
let header = [0x00, 0x00, 0x00, 0x22];
|
||||
let (is_last, block_type, size) = FlacHandler::parse_block_header(&header).unwrap();
|
||||
assert!(!is_last);
|
||||
assert_eq!(block_type, 0);
|
||||
assert_eq!(size, 34);
|
||||
|
||||
// Last, type 4, size 256
|
||||
let header = [0x84, 0x00, 0x01, 0x00];
|
||||
let (is_last, block_type, size) = FlacHandler::parse_block_header(&header).unwrap();
|
||||
assert!(is_last);
|
||||
assert_eq!(block_type, 4);
|
||||
assert_eq!(size, 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_block_header() {
|
||||
let header = FlacHandler::write_block_header(false, 0, 34);
|
||||
assert_eq!(header, [0x00, 0x00, 0x00, 0x22]);
|
||||
|
||||
let header = FlacHandler::write_block_header(true, 4, 256);
|
||||
assert_eq!(header, [0x84, 0x00, 0x01, 0x00]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_metadata_produces_minimal_vorbis() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = AudioMeta::default();
|
||||
|
||||
let original_data = make_minimal_flac_header();
|
||||
let layout = handler
|
||||
.analyze(&original_data, original_data.len() as u64)
|
||||
.unwrap();
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
// Should have: fLaC (4) + STREAMINFO (38) + VORBIS_COMMENT (header + minimal data)
|
||||
assert!(bytes.len() >= 42 + 4 + 8); // At least vendor string overhead
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_round_trip_synthesize_analyze() {
|
||||
let handler = FlacHandler::new();
|
||||
let meta = make_test_meta();
|
||||
|
||||
// Create initial layout
|
||||
let original_data = make_minimal_flac_header();
|
||||
let layout = handler
|
||||
.analyze(&original_data, original_data.len() as u64)
|
||||
.unwrap();
|
||||
|
||||
// Synthesize new header
|
||||
let synthesized = handler.synthesize(&meta, &layout).unwrap();
|
||||
|
||||
// Analyze synthesized header
|
||||
let new_layout = handler
|
||||
.analyze(&synthesized, synthesized.len() as u64)
|
||||
.unwrap();
|
||||
|
||||
// STREAMINFO should be preserved
|
||||
assert_eq!(new_layout.format_data, layout.format_data);
|
||||
assert_eq!(new_layout.format, AudioFormat::Flac);
|
||||
}
|
||||
}
|
||||
@@ -1,631 +0,0 @@
|
||||
use crate::{FormatError, FormatHandler, FormatLayout};
|
||||
use lofty::config::{ParseOptions, WriteOptions};
|
||||
use lofty::file::AudioFile;
|
||||
use lofty::id3::v2::{
|
||||
CommentFrame, Frame, FrameId, Id3v2Tag, TextInformationFrame, UnsynchronizedTextFrame,
|
||||
};
|
||||
use lofty::mpeg::MpegFile;
|
||||
use lofty::tag::{Accessor, TagExt};
|
||||
use lofty::TextEncoding;
|
||||
use musicfs_core::{AudioFormat, AudioMeta};
|
||||
use std::borrow::Cow;
|
||||
use std::io::Cursor;
|
||||
|
||||
const ID3V2_HEADER_SIZE: usize = 10;
|
||||
const ID3V1_TAG_SIZE: usize = 128;
|
||||
|
||||
pub struct Id3v2Handler;
|
||||
|
||||
impl Id3v2Handler {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
fn parse_id3v2_header(data: &[u8]) -> Option<usize> {
|
||||
if data.len() < ID3V2_HEADER_SIZE {
|
||||
return None;
|
||||
}
|
||||
|
||||
if &data[0..3] != b"ID3" {
|
||||
return None;
|
||||
}
|
||||
|
||||
let size = syncsafe_decode(&data[6..10]);
|
||||
Some(ID3V2_HEADER_SIZE + size)
|
||||
}
|
||||
|
||||
fn has_id3v1_tag(data: &[u8], file_size: u64) -> bool {
|
||||
if file_size < ID3V1_TAG_SIZE as u64 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let tag_start = (file_size as usize).saturating_sub(ID3V1_TAG_SIZE);
|
||||
if tag_start >= data.len() {
|
||||
return false;
|
||||
}
|
||||
|
||||
&data[tag_start..tag_start + 3] == b"TAG"
|
||||
}
|
||||
|
||||
fn set_text_frame(tag: &mut Id3v2Tag, frame_id: &'static str, value: &str) {
|
||||
let id = FrameId::Valid(Cow::Borrowed(frame_id));
|
||||
let frame = Frame::Text(TextInformationFrame::new(
|
||||
id,
|
||||
TextEncoding::UTF8,
|
||||
value.to_string(),
|
||||
));
|
||||
tag.insert(frame);
|
||||
}
|
||||
|
||||
fn set_track_disc_frame(
|
||||
tag: &mut Id3v2Tag,
|
||||
frame_id: &'static str,
|
||||
num: u32,
|
||||
total: Option<u32>,
|
||||
) {
|
||||
let value = match total {
|
||||
Some(t) => format!("{}/{}", num, t),
|
||||
None => num.to_string(),
|
||||
};
|
||||
Self::set_text_frame(tag, frame_id, &value);
|
||||
}
|
||||
|
||||
fn set_comment_frame(tag: &mut Id3v2Tag, value: &str) {
|
||||
let frame = Frame::Comment(CommentFrame::new(
|
||||
TextEncoding::UTF8,
|
||||
*b"eng",
|
||||
String::new(),
|
||||
value.to_string(),
|
||||
));
|
||||
tag.insert(frame);
|
||||
}
|
||||
|
||||
fn set_lyrics_frame(tag: &mut Id3v2Tag, value: &str) {
|
||||
let frame = Frame::UnsynchronizedText(UnsynchronizedTextFrame::new(
|
||||
TextEncoding::UTF8,
|
||||
*b"eng",
|
||||
String::new(),
|
||||
value.to_string(),
|
||||
));
|
||||
tag.insert(frame);
|
||||
}
|
||||
|
||||
fn build_tag_from_meta(metadata: &AudioMeta) -> Id3v2Tag {
|
||||
let mut tag = Id3v2Tag::new();
|
||||
|
||||
if let Some(ref title) = metadata.title {
|
||||
tag.set_title(title.clone());
|
||||
}
|
||||
if let Some(ref artist) = metadata.artist {
|
||||
tag.set_artist(artist.clone());
|
||||
}
|
||||
if let Some(ref album) = metadata.album {
|
||||
tag.set_album(album.clone());
|
||||
}
|
||||
if let Some(ref album_artist) = metadata.album_artist {
|
||||
Self::set_text_frame(&mut tag, "TPE2", album_artist);
|
||||
}
|
||||
if let Some(year) = metadata.year {
|
||||
Self::set_text_frame(&mut tag, "TDRC", &year.to_string());
|
||||
}
|
||||
if let Some(ref genre) = metadata.genre {
|
||||
tag.set_genre(genre.clone());
|
||||
}
|
||||
|
||||
if let Some(track) = metadata.track {
|
||||
Self::set_track_disc_frame(&mut tag, "TRCK", track, metadata.track_total);
|
||||
}
|
||||
if let Some(disc) = metadata.disc {
|
||||
Self::set_track_disc_frame(&mut tag, "TPOS", disc, metadata.disc_total);
|
||||
}
|
||||
|
||||
if let Some(ref date) = metadata.date {
|
||||
Self::set_text_frame(&mut tag, "TDRC", date);
|
||||
}
|
||||
if let Some(ref composer) = metadata.composer {
|
||||
Self::set_text_frame(&mut tag, "TCOM", composer);
|
||||
}
|
||||
if let Some(ref comment) = metadata.comment {
|
||||
Self::set_comment_frame(&mut tag, comment);
|
||||
}
|
||||
if let Some(ref lyrics) = metadata.lyrics {
|
||||
Self::set_lyrics_frame(&mut tag, lyrics);
|
||||
}
|
||||
if let Some(ref copyright) = metadata.copyright {
|
||||
Self::set_text_frame(&mut tag, "TCOP", copyright);
|
||||
}
|
||||
if let Some(compilation) = metadata.compilation {
|
||||
Self::set_text_frame(&mut tag, "TCMP", if compilation { "1" } else { "0" });
|
||||
}
|
||||
|
||||
if let Some(ref title_sort) = metadata.title_sort {
|
||||
Self::set_text_frame(&mut tag, "TSOT", title_sort);
|
||||
}
|
||||
if let Some(ref artist_sort) = metadata.artist_sort {
|
||||
Self::set_text_frame(&mut tag, "TSOP", artist_sort);
|
||||
}
|
||||
if let Some(ref album_sort) = metadata.album_sort {
|
||||
Self::set_text_frame(&mut tag, "TSOA", album_sort);
|
||||
}
|
||||
if let Some(ref album_artist_sort) = metadata.album_artist_sort {
|
||||
Self::set_text_frame(&mut tag, "TSO2", album_artist_sort);
|
||||
}
|
||||
|
||||
if let Some(ref mb_recording_id) = metadata.mb_recording_id {
|
||||
tag.insert_user_text(
|
||||
"MusicBrainz Recording Id".to_string(),
|
||||
mb_recording_id.clone(),
|
||||
);
|
||||
}
|
||||
if let Some(ref mb_album_id) = metadata.mb_album_id {
|
||||
tag.insert_user_text("MusicBrainz Album Id".to_string(), mb_album_id.clone());
|
||||
}
|
||||
if let Some(ref mb_artist_id) = metadata.mb_artist_id {
|
||||
tag.insert_user_text("MusicBrainz Artist Id".to_string(), mb_artist_id.clone());
|
||||
}
|
||||
if let Some(ref mb_album_artist_id) = metadata.mb_album_artist_id {
|
||||
tag.insert_user_text(
|
||||
"MusicBrainz Album Artist Id".to_string(),
|
||||
mb_album_artist_id.clone(),
|
||||
);
|
||||
}
|
||||
if let Some(ref mb_release_group_id) = metadata.mb_release_group_id {
|
||||
tag.insert_user_text(
|
||||
"MusicBrainz Release Group Id".to_string(),
|
||||
mb_release_group_id.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(gain) = metadata.replaygain_track_gain {
|
||||
tag.insert_user_text(
|
||||
"REPLAYGAIN_TRACK_GAIN".to_string(),
|
||||
format!("{:.2} dB", gain),
|
||||
);
|
||||
}
|
||||
if let Some(peak) = metadata.replaygain_track_peak {
|
||||
tag.insert_user_text("REPLAYGAIN_TRACK_PEAK".to_string(), format!("{:.6}", peak));
|
||||
}
|
||||
if let Some(gain) = metadata.replaygain_album_gain {
|
||||
tag.insert_user_text(
|
||||
"REPLAYGAIN_ALBUM_GAIN".to_string(),
|
||||
format!("{:.2} dB", gain),
|
||||
);
|
||||
}
|
||||
if let Some(peak) = metadata.replaygain_album_peak {
|
||||
tag.insert_user_text("REPLAYGAIN_ALBUM_PEAK".to_string(), format!("{:.6}", peak));
|
||||
}
|
||||
|
||||
if let Some(ref encoder) = metadata.encoder {
|
||||
Self::set_text_frame(&mut tag, "TSSE", encoder);
|
||||
}
|
||||
|
||||
tag
|
||||
}
|
||||
|
||||
fn extract_text_frame(tag: &Id3v2Tag, frame_id: &str) -> Option<String> {
|
||||
let id = FrameId::new(frame_id).ok()?;
|
||||
tag.get_text(&id).map(|s| s.to_string())
|
||||
}
|
||||
|
||||
fn parse_track_disc(value: &str) -> (Option<u32>, Option<u32>) {
|
||||
let parts: Vec<&str> = value.split('/').collect();
|
||||
let num = parts.first().and_then(|s| s.parse().ok());
|
||||
let total = parts.get(1).and_then(|s| s.parse().ok());
|
||||
(num, total)
|
||||
}
|
||||
|
||||
fn parse_replaygain_value(value: &str) -> Option<f32> {
|
||||
value
|
||||
.trim()
|
||||
.trim_end_matches(" dB")
|
||||
.trim_end_matches("dB")
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn extract_from_tag(tag: &Id3v2Tag) -> AudioMeta {
|
||||
let mut meta = AudioMeta::default();
|
||||
meta.format = AudioFormat::Mp3;
|
||||
|
||||
meta.title = tag.title().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.artist = tag.artist().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.album = tag.album().map(|c: Cow<'_, str>| c.into_owned());
|
||||
meta.album_artist = Self::extract_text_frame(tag, "TPE2");
|
||||
meta.genre = tag.genre().map(|c: Cow<'_, str>| c.into_owned());
|
||||
|
||||
if let Some(track_str) = Self::extract_text_frame(tag, "TRCK") {
|
||||
let (track, track_total) = Self::parse_track_disc(&track_str);
|
||||
meta.track = track;
|
||||
meta.track_total = track_total;
|
||||
} else {
|
||||
meta.track = tag.track();
|
||||
meta.track_total = tag.track_total();
|
||||
}
|
||||
|
||||
if let Some(disc_str) = Self::extract_text_frame(tag, "TPOS") {
|
||||
let (disc, disc_total) = Self::parse_track_disc(&disc_str);
|
||||
meta.disc = disc;
|
||||
meta.disc_total = disc_total;
|
||||
} else {
|
||||
meta.disc = tag.disk();
|
||||
meta.disc_total = tag.disk_total();
|
||||
}
|
||||
|
||||
meta.date = Self::extract_text_frame(tag, "TDRC");
|
||||
if let Some(ref date) = meta.date {
|
||||
if let Some(year_str) = date.split('-').next() {
|
||||
meta.year = year_str.parse().ok();
|
||||
}
|
||||
}
|
||||
|
||||
meta.composer = Self::extract_text_frame(tag, "TCOM");
|
||||
meta.comment = tag.comment().map(|c: Cow<'_, str>| c.into_owned());
|
||||
|
||||
if let Some(uslt) = tag.unsync_text().next() {
|
||||
meta.lyrics = Some(uslt.content.to_string());
|
||||
}
|
||||
|
||||
meta.copyright = Self::extract_text_frame(tag, "TCOP");
|
||||
|
||||
if let Some(tcmp) = Self::extract_text_frame(tag, "TCMP") {
|
||||
meta.compilation = Some(tcmp == "1");
|
||||
}
|
||||
|
||||
meta.title_sort = Self::extract_text_frame(tag, "TSOT");
|
||||
meta.artist_sort = Self::extract_text_frame(tag, "TSOP");
|
||||
meta.album_sort = Self::extract_text_frame(tag, "TSOA");
|
||||
meta.album_artist_sort = Self::extract_text_frame(tag, "TSO2");
|
||||
|
||||
meta.mb_recording_id = tag
|
||||
.get_user_text("MusicBrainz Recording Id")
|
||||
.map(String::from);
|
||||
meta.mb_album_id = tag.get_user_text("MusicBrainz Album Id").map(String::from);
|
||||
meta.mb_artist_id = tag.get_user_text("MusicBrainz Artist Id").map(String::from);
|
||||
meta.mb_album_artist_id = tag
|
||||
.get_user_text("MusicBrainz Album Artist Id")
|
||||
.map(String::from);
|
||||
meta.mb_release_group_id = tag
|
||||
.get_user_text("MusicBrainz Release Group Id")
|
||||
.map(String::from);
|
||||
|
||||
if let Some(gain_str) = tag.get_user_text("REPLAYGAIN_TRACK_GAIN") {
|
||||
meta.replaygain_track_gain = Self::parse_replaygain_value(gain_str);
|
||||
}
|
||||
if let Some(peak_str) = tag.get_user_text("REPLAYGAIN_TRACK_PEAK") {
|
||||
meta.replaygain_track_peak = peak_str.parse::<f32>().ok();
|
||||
}
|
||||
if let Some(gain_str) = tag.get_user_text("REPLAYGAIN_ALBUM_GAIN") {
|
||||
meta.replaygain_album_gain = Self::parse_replaygain_value(gain_str);
|
||||
}
|
||||
if let Some(peak_str) = tag.get_user_text("REPLAYGAIN_ALBUM_PEAK") {
|
||||
meta.replaygain_album_peak = peak_str.parse::<f32>().ok();
|
||||
}
|
||||
|
||||
meta.encoder = Self::extract_text_frame(tag, "TSSE");
|
||||
|
||||
meta
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Id3v2Handler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl FormatHandler for Id3v2Handler {
|
||||
fn id(&self) -> &'static str {
|
||||
"id3v2"
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"ID3v2 (MP3)"
|
||||
}
|
||||
|
||||
fn extensions(&self) -> &[&'static str] {
|
||||
&["mp3"]
|
||||
}
|
||||
|
||||
fn mime_types(&self) -> &[&'static str] {
|
||||
&["audio/mpeg"]
|
||||
}
|
||||
|
||||
fn analyze(&self, data: &[u8], file_size: u64) -> Result<FormatLayout, FormatError> {
|
||||
let audio_start = Self::parse_id3v2_header(data).unwrap_or(0) as u64;
|
||||
|
||||
let audio_end = if Self::has_id3v1_tag(data, file_size) {
|
||||
file_size - ID3V1_TAG_SIZE as u64
|
||||
} else {
|
||||
file_size
|
||||
};
|
||||
|
||||
Ok(FormatLayout {
|
||||
audio_start,
|
||||
audio_end,
|
||||
format: AudioFormat::Mp3,
|
||||
format_data: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn synthesize(
|
||||
&self,
|
||||
metadata: &AudioMeta,
|
||||
_layout: &FormatLayout,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let tag = Self::build_tag_from_meta(metadata);
|
||||
|
||||
let mut buffer = Cursor::new(Vec::new());
|
||||
let write_options = WriteOptions::new().preferred_padding(1024);
|
||||
|
||||
tag.dump_to(&mut buffer, write_options)
|
||||
.map_err(|e| FormatError::SynthesisFailed(e.to_string()))?;
|
||||
|
||||
Ok(buffer.into_inner())
|
||||
}
|
||||
|
||||
fn extract(&self, data: &[u8]) -> Result<AudioMeta, FormatError> {
|
||||
let mut cursor = Cursor::new(data);
|
||||
|
||||
let mpeg_file = MpegFile::read_from(&mut cursor, ParseOptions::new())
|
||||
.map_err(|e| FormatError::InvalidData(e.to_string()))?;
|
||||
|
||||
let tag = mpeg_file
|
||||
.id3v2()
|
||||
.ok_or_else(|| FormatError::InvalidData("No ID3v2 tag found".to_string()))?;
|
||||
|
||||
Ok(Self::extract_from_tag(tag))
|
||||
}
|
||||
|
||||
fn estimate_header_size(&self, _metadata: &AudioMeta) -> usize {
|
||||
4096 + 1024
|
||||
}
|
||||
}
|
||||
|
||||
fn syncsafe_decode(bytes: &[u8]) -> usize {
|
||||
((bytes[0] as usize) << 21)
|
||||
| ((bytes[1] as usize) << 14)
|
||||
| ((bytes[2] as usize) << 7)
|
||||
| (bytes[3] as usize)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_test_meta() -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: Some("Test Title".to_string()),
|
||||
artist: Some("Test Artist".to_string()),
|
||||
album: Some("Test Album".to_string()),
|
||||
album_artist: Some("Test Album Artist".to_string()),
|
||||
genre: Some("Rock".to_string()),
|
||||
year: Some(2024),
|
||||
track: Some(5),
|
||||
track_total: Some(12),
|
||||
disc: Some(1),
|
||||
disc_total: Some(2),
|
||||
format: AudioFormat::Mp3,
|
||||
date: Some("2024-03-15".to_string()),
|
||||
composer: Some("Test Composer".to_string()),
|
||||
comment: Some("Test Comment".to_string()),
|
||||
lyrics: Some("Test Lyrics\nLine 2".to_string()),
|
||||
copyright: Some("2024 Test Copyright".to_string()),
|
||||
compilation: Some(false),
|
||||
title_sort: Some("Title, Test".to_string()),
|
||||
artist_sort: Some("Artist, Test".to_string()),
|
||||
album_sort: Some("Album, Test".to_string()),
|
||||
album_artist_sort: Some("Album Artist, Test".to_string()),
|
||||
mb_recording_id: Some("rec-12345".to_string()),
|
||||
mb_album_id: Some("alb-12345".to_string()),
|
||||
mb_artist_id: Some("art-12345".to_string()),
|
||||
mb_album_artist_id: Some("albart-12345".to_string()),
|
||||
mb_release_group_id: Some("rg-12345".to_string()),
|
||||
replaygain_track_gain: Some(-6.5),
|
||||
replaygain_track_peak: Some(0.987654),
|
||||
replaygain_album_gain: Some(-5.2),
|
||||
replaygain_album_peak: Some(0.999999),
|
||||
encoder: Some("LAME 3.100".to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_id_and_name() {
|
||||
let handler = Id3v2Handler::new();
|
||||
assert_eq!(handler.id(), "id3v2");
|
||||
assert_eq!(handler.name(), "ID3v2 (MP3)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extensions_and_mime_types() {
|
||||
let handler = Id3v2Handler::new();
|
||||
assert_eq!(handler.extensions(), &["mp3"]);
|
||||
assert_eq!(handler.mime_types(), &["audio/mpeg"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_header_size() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let meta = AudioMeta::default();
|
||||
assert_eq!(handler.estimate_header_size(&meta), 5120);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthesize_creates_valid_id3v2() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let meta = make_test_meta();
|
||||
let layout = FormatLayout {
|
||||
audio_start: 0,
|
||||
audio_end: 1000,
|
||||
format: AudioFormat::Mp3,
|
||||
format_data: None,
|
||||
};
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
assert!(bytes.len() >= 10);
|
||||
assert_eq!(&bytes[0..3], b"ID3");
|
||||
assert_eq!(bytes[3], 0x04);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_no_id3v2() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let data = vec![0xFF, 0xFB, 0x90, 0x00];
|
||||
let file_size = 1000;
|
||||
|
||||
let result = handler.analyze(&data, file_size);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let layout = result.unwrap();
|
||||
assert_eq!(layout.audio_start, 0);
|
||||
assert_eq!(layout.audio_end, 1000);
|
||||
assert_eq!(layout.format, AudioFormat::Mp3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_with_id3v2() {
|
||||
let handler = Id3v2Handler::new();
|
||||
|
||||
let mut data = vec![b'I', b'D', b'3', 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64];
|
||||
data.extend(vec![0u8; 100]);
|
||||
let file_size = data.len() as u64;
|
||||
|
||||
let result = handler.analyze(&data, file_size);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let layout = result.unwrap();
|
||||
assert_eq!(layout.audio_start, 110);
|
||||
assert_eq!(layout.audio_end, file_size);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_with_id3v1() {
|
||||
let handler = Id3v2Handler::new();
|
||||
|
||||
let mut data = vec![0xFF, 0xFB, 0x90, 0x00];
|
||||
data.extend(vec![0u8; 100]);
|
||||
data.extend(b"TAG");
|
||||
data.extend(vec![0u8; 125]);
|
||||
let file_size = data.len() as u64;
|
||||
|
||||
let result = handler.analyze(&data, file_size);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let layout = result.unwrap();
|
||||
assert_eq!(layout.audio_start, 0);
|
||||
assert_eq!(layout.audio_end, file_size - 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_syncsafe_decode() {
|
||||
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x00, 0x7F]), 127);
|
||||
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x01, 0x00]), 128);
|
||||
assert_eq!(syncsafe_decode(&[0x00, 0x00, 0x00, 0x64]), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_track_disc() {
|
||||
assert_eq!(Id3v2Handler::parse_track_disc("5/12"), (Some(5), Some(12)));
|
||||
assert_eq!(Id3v2Handler::parse_track_disc("5"), (Some(5), None));
|
||||
assert_eq!(Id3v2Handler::parse_track_disc(""), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_replaygain_value() {
|
||||
assert_eq!(
|
||||
Id3v2Handler::parse_replaygain_value("-6.50 dB"),
|
||||
Some(-6.50)
|
||||
);
|
||||
assert_eq!(Id3v2Handler::parse_replaygain_value("-6.50dB"), Some(-6.50));
|
||||
assert_eq!(Id3v2Handler::parse_replaygain_value("-6.50"), Some(-6.50));
|
||||
assert_eq!(Id3v2Handler::parse_replaygain_value("invalid"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_metadata_produces_empty_tag() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let meta = AudioMeta::default();
|
||||
let layout = FormatLayout {
|
||||
audio_start: 0,
|
||||
audio_end: 1000,
|
||||
format: AudioFormat::Mp3,
|
||||
format_data: None,
|
||||
};
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
assert!(bytes.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_minimal_metadata_produces_valid_tag() {
|
||||
let handler = Id3v2Handler::new();
|
||||
let mut meta = AudioMeta::default();
|
||||
meta.title = Some("Test".to_string());
|
||||
let layout = FormatLayout {
|
||||
audio_start: 0,
|
||||
audio_end: 1000,
|
||||
format: AudioFormat::Mp3,
|
||||
format_data: None,
|
||||
};
|
||||
|
||||
let result = handler.synthesize(&meta, &layout);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let bytes = result.unwrap();
|
||||
assert!(bytes.len() >= 10);
|
||||
assert_eq!(&bytes[0..3], b"ID3");
|
||||
assert_eq!(bytes[3], 0x04);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_and_extract_tag() {
|
||||
let original_meta = make_test_meta();
|
||||
let tag = Id3v2Handler::build_tag_from_meta(&original_meta);
|
||||
let extracted = Id3v2Handler::extract_from_tag(&tag);
|
||||
|
||||
assert_eq!(extracted.title, original_meta.title);
|
||||
assert_eq!(extracted.artist, original_meta.artist);
|
||||
assert_eq!(extracted.album, original_meta.album);
|
||||
assert_eq!(extracted.album_artist, original_meta.album_artist);
|
||||
assert_eq!(extracted.genre, original_meta.genre);
|
||||
assert_eq!(extracted.track, original_meta.track);
|
||||
assert_eq!(extracted.track_total, original_meta.track_total);
|
||||
assert_eq!(extracted.disc, original_meta.disc);
|
||||
assert_eq!(extracted.disc_total, original_meta.disc_total);
|
||||
assert_eq!(extracted.composer, original_meta.composer);
|
||||
assert_eq!(extracted.comment, original_meta.comment);
|
||||
assert_eq!(extracted.lyrics, original_meta.lyrics);
|
||||
assert_eq!(extracted.copyright, original_meta.copyright);
|
||||
assert_eq!(extracted.compilation, original_meta.compilation);
|
||||
assert_eq!(extracted.title_sort, original_meta.title_sort);
|
||||
assert_eq!(extracted.artist_sort, original_meta.artist_sort);
|
||||
assert_eq!(extracted.album_sort, original_meta.album_sort);
|
||||
assert_eq!(extracted.album_artist_sort, original_meta.album_artist_sort);
|
||||
assert_eq!(extracted.mb_recording_id, original_meta.mb_recording_id);
|
||||
assert_eq!(extracted.mb_album_id, original_meta.mb_album_id);
|
||||
assert_eq!(extracted.mb_artist_id, original_meta.mb_artist_id);
|
||||
assert_eq!(
|
||||
extracted.mb_album_artist_id,
|
||||
original_meta.mb_album_artist_id
|
||||
);
|
||||
assert_eq!(
|
||||
extracted.mb_release_group_id,
|
||||
original_meta.mb_release_group_id
|
||||
);
|
||||
assert_eq!(extracted.encoder, original_meta.encoder);
|
||||
|
||||
let orig_track_gain = original_meta.replaygain_track_gain.unwrap();
|
||||
let ext_track_gain = extracted.replaygain_track_gain.unwrap();
|
||||
assert!((orig_track_gain - ext_track_gain).abs() < 0.01);
|
||||
|
||||
let orig_track_peak = original_meta.replaygain_track_peak.unwrap();
|
||||
let ext_track_peak = extracted.replaygain_track_peak.unwrap();
|
||||
assert!((orig_track_peak - ext_track_peak).abs() < 0.0001);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
//! Format-specific metadata handlers for audio file synthesis.
|
||||
//!
|
||||
//! Each handler implements the `FormatHandler` trait to support:
|
||||
//! - Analyzing original files to find audio boundaries
|
||||
//! - Synthesizing new headers from database metadata
|
||||
//! - Extracting metadata from existing files
|
||||
|
||||
mod flac;
|
||||
mod id3v2;
|
||||
|
||||
pub use flac::FlacHandler;
|
||||
pub use id3v2::Id3v2Handler;
|
||||
@@ -1,26 +0,0 @@
|
||||
mod artwork;
|
||||
mod db;
|
||||
mod eviction;
|
||||
mod format_handler;
|
||||
mod format_layout;
|
||||
pub mod handlers;
|
||||
mod metadata;
|
||||
mod overlay;
|
||||
mod patterns;
|
||||
mod prefetch;
|
||||
mod tree;
|
||||
|
||||
pub use artwork::{ArtworkCache, ArtworkError, CachedArtwork};
|
||||
pub use db::{Database, EnrichmentUpdate, TrashedFile, TrashedFilter};
|
||||
pub use eviction::{EvictionError, EvictionPolicy, LruEviction};
|
||||
pub use format_handler::{FormatError, FormatHandler, FormatHandlerRegistry};
|
||||
pub use format_layout::FormatLayout;
|
||||
pub use handlers::{FlacHandler, Id3v2Handler};
|
||||
pub use metadata::MetadataCache;
|
||||
pub use overlay::{OverlayError, OverlayReader};
|
||||
pub use patterns::{AccessContext, AccessPattern, PatternError, PatternStore};
|
||||
pub use prefetch::{PrefetchConfig, PrefetchEngine, PrefetchHandle};
|
||||
pub use tree::{
|
||||
DirNode, FileNode, Inode, RefreshPolicy, RemoveError, RenameError, TreeBuilder, VirtualNode,
|
||||
VirtualTree, ROOT_INODE,
|
||||
};
|
||||
@@ -1,138 +0,0 @@
|
||||
use crate::db::Database;
|
||||
use musicfs_core::{AudioMeta, FileMeta, OriginId, Result, VirtualPath};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tracing::trace;
|
||||
|
||||
pub struct MetadataCache {
|
||||
db: Arc<Database>,
|
||||
}
|
||||
|
||||
impl MetadataCache {
|
||||
pub fn new(db: Arc<Database>) -> Self {
|
||||
Self { db }
|
||||
}
|
||||
|
||||
pub fn store(
|
||||
&self,
|
||||
origin_id: &OriginId,
|
||||
real_path: &Path,
|
||||
virtual_path: &VirtualPath,
|
||||
audio_meta: &AudioMeta,
|
||||
origin_mtime: SystemTime,
|
||||
origin_size: u64,
|
||||
) -> Result<()> {
|
||||
self.db.upsert_file(
|
||||
origin_id,
|
||||
real_path,
|
||||
virtual_path,
|
||||
audio_meta,
|
||||
origin_mtime,
|
||||
origin_size,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn lookup(&self, path: &VirtualPath) -> Result<Option<FileMeta>> {
|
||||
let result = self.db.get_file_by_virtual_path(path)?;
|
||||
let hit = result.is_some();
|
||||
trace!(path = path.as_str(), hit, "metadata cache lookup");
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn is_fresh(
|
||||
&self,
|
||||
origin_id: &OriginId,
|
||||
real_path: &Path,
|
||||
current_mtime: SystemTime,
|
||||
) -> Result<bool> {
|
||||
if let Some(cached_mtime) = self.db.get_mtime_by_real_path(origin_id, real_path)? {
|
||||
let current_secs = current_mtime
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs();
|
||||
let cached_secs = cached_mtime
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs();
|
||||
let hit = current_secs == cached_secs;
|
||||
trace!(path = ?real_path, hit, "metadata freshness check");
|
||||
Ok(hit)
|
||||
} else {
|
||||
trace!(path = ?real_path, hit = false, "metadata freshness check");
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn invalidate(&self, path: &VirtualPath) -> Result<()> {
|
||||
if let Some(meta) = self.db.get_file_by_virtual_path(path)? {
|
||||
self.db.delete_file(meta.id)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_core::AudioFormat;
|
||||
|
||||
#[test]
|
||||
fn test_metadata_cache_store_and_lookup() {
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
let cache = MetadataCache::new(db);
|
||||
|
||||
let origin_id = OriginId::from("local");
|
||||
let real_path = Path::new("/music/song.flac");
|
||||
let virtual_path = VirtualPath::new("/Artist/Album/Song.flac");
|
||||
let meta = AudioMeta {
|
||||
title: Some("Song".to_string()),
|
||||
artist: Some("Artist".to_string()),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
cache
|
||||
.store(
|
||||
&origin_id,
|
||||
real_path,
|
||||
&virtual_path,
|
||||
&meta,
|
||||
UNIX_EPOCH,
|
||||
5000,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let retrieved = cache.lookup(&virtual_path).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
retrieved.audio.as_ref().unwrap().title,
|
||||
Some("Song".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_metadata_cache_invalidate() {
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
let cache = MetadataCache::new(db);
|
||||
|
||||
let virtual_path = VirtualPath::new("/Test.flac");
|
||||
|
||||
cache
|
||||
.store(
|
||||
&OriginId::from("local"),
|
||||
Path::new("/test.flac"),
|
||||
&virtual_path,
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(cache.lookup(&virtual_path).unwrap().is_some());
|
||||
|
||||
cache.invalidate(&virtual_path).unwrap();
|
||||
|
||||
assert!(cache.lookup(&virtual_path).unwrap().is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,467 +0,0 @@
|
||||
//! OverlayReader: On-the-fly metadata overlay with header/audio splice logic.
|
||||
//!
|
||||
//! This module provides the core read path for metadata overlay. It synthesizes
|
||||
//! headers on-the-fly from database metadata and splices them with original audio
|
||||
//! data from the CAS.
|
||||
|
||||
use crate::{Database, FormatError, FormatHandlerRegistry};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use musicfs_cas::{FileReader, ReaderError};
|
||||
use musicfs_core::{AudioFormat, FileId};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, trace};
|
||||
|
||||
/// Error types for overlay operations
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum OverlayError {
|
||||
#[error("Database error: {0}")]
|
||||
Database(#[from] musicfs_core::Error),
|
||||
|
||||
#[error("Format handler error: {0}")]
|
||||
Handler(#[from] FormatError),
|
||||
|
||||
#[error("CAS error: {0}")]
|
||||
Cas(#[from] ReaderError),
|
||||
|
||||
#[error("File not found: {0:?}")]
|
||||
NotFound(FileId),
|
||||
|
||||
#[error("No handler for format: {0:?}")]
|
||||
NoHandler(AudioFormat),
|
||||
}
|
||||
|
||||
/// OverlayReader provides on-the-fly metadata overlay for audio files.
|
||||
///
|
||||
/// It synthesizes headers from database metadata and splices them with
|
||||
/// original audio data from the CAS, presenting a virtual file that
|
||||
/// reflects the current metadata state.
|
||||
pub struct OverlayReader {
|
||||
db: Arc<Database>,
|
||||
registry: Arc<FormatHandlerRegistry>,
|
||||
cas_reader: Arc<FileReader>,
|
||||
}
|
||||
|
||||
impl OverlayReader {
|
||||
/// Create a new OverlayReader with the given dependencies.
|
||||
pub fn new(
|
||||
db: Arc<Database>,
|
||||
registry: Arc<FormatHandlerRegistry>,
|
||||
cas_reader: Arc<FileReader>,
|
||||
) -> Self {
|
||||
Self {
|
||||
db,
|
||||
registry,
|
||||
cas_reader,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read bytes from a virtual file with metadata overlay.
|
||||
///
|
||||
/// This method implements the three-region splice logic:
|
||||
/// - Region 1: Synthetic header (offset < header_len)
|
||||
/// - Region 2: Audio data from CAS (offset >= header_len)
|
||||
/// - Region 3: Boundary crossing (spans header/audio)
|
||||
///
|
||||
/// If no format_layout exists for the file, delegates directly to CAS reader.
|
||||
pub async fn read(
|
||||
&self,
|
||||
file_id: FileId,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> Result<Bytes, OverlayError> {
|
||||
// Get format layout - if None, passthrough to CAS
|
||||
let layout = match self.db.get_format_layout(file_id)? {
|
||||
Some(layout) => layout,
|
||||
None => {
|
||||
trace!(file_id = ?file_id, "No format_layout, passthrough to CAS");
|
||||
return Ok(self.cas_reader.read(file_id, offset, size).await?);
|
||||
}
|
||||
};
|
||||
|
||||
// Get metadata for synthesis
|
||||
let metadata = self.db.get_file_metadata_row(file_id)?;
|
||||
|
||||
// Get handler for this format (handler IDs are lowercase)
|
||||
let format_id = format!("{:?}", layout.format).to_lowercase();
|
||||
let handler = self
|
||||
.registry
|
||||
.get_by_format(&format_id)
|
||||
.ok_or_else(|| OverlayError::NoHandler(layout.format))?;
|
||||
|
||||
// Synthesize header on-the-fly
|
||||
let header = handler.synthesize(&metadata, &layout)?;
|
||||
let header_len = header.len() as u64;
|
||||
let audio_len = layout.audio_end - layout.audio_start;
|
||||
let virtual_size = header_len + audio_len;
|
||||
|
||||
trace!(
|
||||
file_id = ?file_id,
|
||||
header_len,
|
||||
audio_len,
|
||||
virtual_size,
|
||||
offset,
|
||||
size,
|
||||
"Overlay read"
|
||||
);
|
||||
|
||||
// Handle EOF
|
||||
if offset >= virtual_size {
|
||||
return Ok(Bytes::new());
|
||||
}
|
||||
|
||||
let virtual_end = (offset + size as u64).min(virtual_size);
|
||||
let mut result = BytesMut::with_capacity((virtual_end - offset) as usize);
|
||||
|
||||
// Region 1: Synthetic header
|
||||
if offset < header_len {
|
||||
let end = virtual_end.min(header_len);
|
||||
result.extend_from_slice(&header[offset as usize..end as usize]);
|
||||
trace!(
|
||||
file_id = ?file_id,
|
||||
start = offset,
|
||||
end,
|
||||
bytes = end - offset,
|
||||
"Read from synthetic header"
|
||||
);
|
||||
}
|
||||
|
||||
// Region 2: Origin audio data (from CAS)
|
||||
if virtual_end > header_len {
|
||||
let audio_start_in_virtual = header_len.max(offset);
|
||||
let audio_offset_in_origin = layout.audio_start + (audio_start_in_virtual - header_len);
|
||||
let audio_bytes_needed = (virtual_end - audio_start_in_virtual) as u32;
|
||||
|
||||
trace!(
|
||||
file_id = ?file_id,
|
||||
audio_offset_in_origin,
|
||||
audio_bytes_needed,
|
||||
"Read from CAS audio"
|
||||
);
|
||||
|
||||
let audio = self
|
||||
.cas_reader
|
||||
.read(file_id, audio_offset_in_origin, audio_bytes_needed)
|
||||
.await?;
|
||||
result.extend_from_slice(&audio);
|
||||
}
|
||||
|
||||
debug!(
|
||||
file_id = ?file_id,
|
||||
offset,
|
||||
size,
|
||||
returned = result.len(),
|
||||
"Overlay read complete"
|
||||
);
|
||||
|
||||
Ok(result.freeze())
|
||||
}
|
||||
|
||||
/// Estimate the virtual size of a file for getattr.
|
||||
///
|
||||
/// Returns the estimated size based on format layout. If no layout exists,
|
||||
/// returns None to indicate the caller should use the original file size.
|
||||
pub fn estimate_virtual_size(&self, file_id: FileId) -> Result<Option<u64>, OverlayError> {
|
||||
// Get format layout - if None, return None to indicate passthrough
|
||||
let layout = match self.db.get_format_layout(file_id)? {
|
||||
Some(layout) => layout,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
// Get metadata for header size estimation
|
||||
let metadata = self.db.get_file_metadata_row(file_id)?;
|
||||
|
||||
let format_id = format!("{:?}", layout.format).to_lowercase();
|
||||
let handler = self
|
||||
.registry
|
||||
.get_by_format(&format_id)
|
||||
.ok_or_else(|| OverlayError::NoHandler(layout.format))?;
|
||||
|
||||
// Estimate header size
|
||||
let estimated_header = handler.estimate_header_size(&metadata) as u64;
|
||||
let audio_len = layout.audio_end - layout.audio_start;
|
||||
let virtual_size = estimated_header + audio_len;
|
||||
|
||||
trace!(
|
||||
file_id = ?file_id,
|
||||
estimated_header,
|
||||
audio_len,
|
||||
virtual_size,
|
||||
"Estimated virtual size"
|
||||
);
|
||||
|
||||
Ok(Some(virtual_size))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::handlers::FlacHandler;
|
||||
use crate::FormatLayout;
|
||||
use musicfs_cas::{CasConfig, CasStore, ChunkManifest, ChunkRef};
|
||||
use musicfs_core::{AudioFormat, AudioMeta, OriginId, VirtualPath};
|
||||
use std::path::Path;
|
||||
use std::time::UNIX_EPOCH;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_test_metadata() -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: Some("Test Track".to_string()),
|
||||
artist: Some("Test Artist".to_string()),
|
||||
album: Some("Test Album".to_string()),
|
||||
track: Some(1),
|
||||
format: AudioFormat::Flac,
|
||||
sample_rate: Some(44100),
|
||||
bits_per_sample: Some(16),
|
||||
channels: Some(2),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn make_test_layout() -> FormatLayout {
|
||||
// Simulate a file with minimal FLAC header, audio from 42 to 102442 (100KB audio)
|
||||
// STREAMINFO data (34 bytes) - minimal valid values for FLAC synthesis
|
||||
let streaminfo_data = vec![
|
||||
0x10, 0x00, // min_block_size = 4096
|
||||
0x10, 0x00, // max_block_size = 4096
|
||||
0x00, 0x00, 0x00, // min_frame_size = 0
|
||||
0x00, 0x00, 0x00, // max_frame_size = 0
|
||||
0x0A, 0xC4, 0x42, 0xF0, // sample_rate=44100, channels=2, bits=16
|
||||
0x00, 0x00, 0x00, 0x00, // total_samples
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MD5 (16 bytes)
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
];
|
||||
FormatLayout {
|
||||
audio_start: 42, // fLaC (4) + STREAMINFO block (38)
|
||||
audio_end: 42 + 100 * 1024, // 100KB audio
|
||||
format: AudioFormat::Flac,
|
||||
format_data: Some(streaminfo_data),
|
||||
}
|
||||
}
|
||||
|
||||
async fn setup_test_env() -> (
|
||||
TempDir,
|
||||
Arc<Database>,
|
||||
Arc<FormatHandlerRegistry>,
|
||||
Arc<FileReader>,
|
||||
FileId,
|
||||
) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
// Setup database
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
|
||||
// Setup registry with FLAC handler
|
||||
let mut registry = FormatHandlerRegistry::new();
|
||||
registry.register(Arc::new(FlacHandler::new()));
|
||||
let registry = Arc::new(registry);
|
||||
|
||||
// Setup CAS store and reader
|
||||
let cas_config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
|
||||
|
||||
// Create test audio data (simulating 100KB of audio)
|
||||
let audio_data: Vec<u8> = (0..100 * 1024).map(|i| (i % 256) as u8).collect();
|
||||
let hash = store.put(&audio_data).await.unwrap();
|
||||
|
||||
let reader = Arc::new(FileReader::new(store));
|
||||
|
||||
// Register manifest for the test file
|
||||
// The manifest represents the ORIGINAL file in CAS, with audio starting at offset 42
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: 42 + 100 * 1024, // Original file size (42 byte header + 100KB audio)
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 42, // Audio starts at offset 42 in the original file
|
||||
size: audio_data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let file_id = db
|
||||
.upsert_file_with_layout(
|
||||
&OriginId::from("test"),
|
||||
Path::new("/test.flac"),
|
||||
&VirtualPath::new("/Test Artist/Test Album/01 - Test Track.flac"),
|
||||
&make_test_metadata(),
|
||||
UNIX_EPOCH,
|
||||
42 + 100 * 1024,
|
||||
Some(&make_test_layout()),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
(dir, db, registry, reader, file_id)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_header_region() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
// Read first 100 bytes (should be from synthetic header)
|
||||
let result = overlay.read(file_id, 0, 100).await.unwrap();
|
||||
|
||||
// Should return data (synthetic header)
|
||||
assert!(!result.is_empty());
|
||||
assert!(result.len() <= 100);
|
||||
|
||||
// FLAC files start with "fLaC" magic
|
||||
assert_eq!(&result[0..4], b"fLaC");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_audio_region() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db.clone(), registry.clone(), reader.clone());
|
||||
|
||||
// First, get the actual header size by reading it
|
||||
let _header_result = overlay.read(file_id, 0, 64 * 1024).await.unwrap();
|
||||
|
||||
// Get the layout to know where audio starts in virtual file
|
||||
let layout = db.get_format_layout(file_id).unwrap().unwrap();
|
||||
let metadata = db.get_file_metadata_row(file_id).unwrap();
|
||||
let handler = registry.get_by_format("flac").unwrap();
|
||||
let header = handler.synthesize(&metadata, &layout).unwrap();
|
||||
let header_len = header.len() as u64;
|
||||
|
||||
// Read from well into the audio region
|
||||
let audio_offset = header_len + 1000;
|
||||
let result = overlay.read(file_id, audio_offset, 1000).await.unwrap();
|
||||
|
||||
// Should return audio data
|
||||
assert!(!result.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_boundary() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db.clone(), registry.clone(), reader.clone());
|
||||
|
||||
// Get the actual header size
|
||||
let layout = db.get_format_layout(file_id).unwrap().unwrap();
|
||||
let metadata = db.get_file_metadata_row(file_id).unwrap();
|
||||
let handler = registry.get_by_format("flac").unwrap();
|
||||
let header = handler.synthesize(&metadata, &layout).unwrap();
|
||||
let header_len = header.len() as u64;
|
||||
|
||||
// Read across the header/audio boundary
|
||||
let boundary_offset = header_len - 50;
|
||||
let result = overlay.read(file_id, boundary_offset, 100).await.unwrap();
|
||||
|
||||
// Should return 100 bytes spanning both regions
|
||||
assert_eq!(result.len(), 100);
|
||||
|
||||
// First 50 bytes should be from header
|
||||
assert_eq!(&result[0..50], &header[(header_len - 50) as usize..]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_passthrough() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
let registry = Arc::new(FormatHandlerRegistry::new());
|
||||
|
||||
let cas_config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
|
||||
|
||||
let test_data = b"Hello, World! This is test data.";
|
||||
let hash = store.put(test_data).await.unwrap();
|
||||
|
||||
// Insert file WITHOUT format_layout first to get the file_id
|
||||
let file_id = db
|
||||
.upsert_file(
|
||||
&OriginId::from("test"),
|
||||
Path::new("/test.txt"),
|
||||
&VirtualPath::new("/test.txt"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
test_data.len() as u64,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let reader = Arc::new(FileReader::new(store));
|
||||
// Register manifest with the actual file_id from database
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id,
|
||||
total_size: test_data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 0,
|
||||
size: test_data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
let result = overlay
|
||||
.read(file_id, 0, test_data.len() as u32)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(&result[..], test_data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_estimate_virtual_size() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
// Should return estimated size
|
||||
let size = overlay.estimate_virtual_size(file_id).unwrap();
|
||||
assert!(size.is_some());
|
||||
|
||||
let virtual_size = size.unwrap();
|
||||
// Virtual size should be header + audio (100KB audio)
|
||||
assert!(virtual_size > 100 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_estimate_virtual_size_passthrough() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
let registry = Arc::new(FormatHandlerRegistry::new());
|
||||
let cas_config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(cas_config).await.unwrap());
|
||||
let reader = Arc::new(FileReader::new(store));
|
||||
|
||||
// Insert file WITHOUT format_layout
|
||||
let file_id = db
|
||||
.upsert_file(
|
||||
&OriginId::from("test"),
|
||||
Path::new("/test.txt"),
|
||||
&VirtualPath::new("/test.txt"),
|
||||
&AudioMeta::default(),
|
||||
UNIX_EPOCH,
|
||||
1000,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
// Should return None for passthrough
|
||||
let size = overlay.estimate_virtual_size(file_id).unwrap();
|
||||
assert!(size.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_eof() {
|
||||
let (_dir, db, registry, reader, file_id) = setup_test_env().await;
|
||||
let overlay = OverlayReader::new(db, registry, reader);
|
||||
|
||||
// Read past EOF
|
||||
let result = overlay.read(file_id, 1_000_000, 100).await.unwrap();
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -1,291 +0,0 @@
|
||||
use musicfs_core::FileId;
|
||||
use parking_lot::{Mutex, RwLock};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tracing::{debug, info, trace};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccessPattern {
|
||||
pub file_id: FileId,
|
||||
pub timestamp: SystemTime,
|
||||
pub context: AccessContext,
|
||||
pub hour_of_day: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AccessContext {
|
||||
pub album_id: Option<i64>,
|
||||
pub track_number: Option<u32>,
|
||||
pub artist: Option<String>,
|
||||
}
|
||||
|
||||
pub struct PatternStore {
|
||||
db: Mutex<rusqlite::Connection>,
|
||||
sequence_counts: RwLock<HashMap<(FileId, FileId), u32>>,
|
||||
time_patterns: RwLock<HashMap<u8, Vec<FileId>>>,
|
||||
max_history: usize,
|
||||
}
|
||||
|
||||
impl PatternStore {
|
||||
pub fn new(db_path: &Path, max_history: usize) -> Result<Self, PatternError> {
|
||||
let db = rusqlite::Connection::open(db_path)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS access_log (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL,
|
||||
access_time INTEGER NOT NULL,
|
||||
hour_of_day INTEGER NOT NULL
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_access_log_file ON access_log(file_id)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_access_log_time ON access_log(access_time)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS sequence_counts (
|
||||
from_file_id INTEGER NOT NULL,
|
||||
to_file_id INTEGER NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY (from_file_id, to_file_id)
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
|
||||
let sequence_counts = {
|
||||
let mut map = HashMap::new();
|
||||
let mut stmt =
|
||||
db.prepare("SELECT from_file_id, to_file_id, count FROM sequence_counts")?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
(FileId(row.get::<_, i64>(0)?), FileId(row.get::<_, i64>(1)?)),
|
||||
row.get::<_, u32>(2)?,
|
||||
))
|
||||
})?;
|
||||
for row in rows {
|
||||
let (key, count) = row?;
|
||||
map.insert(key, count);
|
||||
}
|
||||
map
|
||||
};
|
||||
|
||||
let store = Self {
|
||||
db: Mutex::new(db),
|
||||
sequence_counts: RwLock::new(sequence_counts),
|
||||
time_patterns: RwLock::new(HashMap::new()),
|
||||
max_history,
|
||||
};
|
||||
let sequence_count = store.sequence_counts.read().len();
|
||||
info!(path = ?db_path, sequence_count = sequence_count, max_history = max_history, "Pattern store opened");
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub fn record(&self, file_id: FileId, _context: AccessContext) -> Result<(), PatternError> {
|
||||
trace!(file_id = file_id.0, "Recording access pattern");
|
||||
let now = SystemTime::now();
|
||||
let timestamp = now.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64;
|
||||
let hour = (timestamp / 3600 % 24) as u8;
|
||||
|
||||
let db = self.db.lock();
|
||||
|
||||
db.execute(
|
||||
"INSERT INTO access_log (file_id, access_time, hour_of_day) VALUES (?1, ?2, ?3)",
|
||||
rusqlite::params![file_id.0, timestamp, hour],
|
||||
)?;
|
||||
|
||||
{
|
||||
let mut time_patterns = self.time_patterns.write();
|
||||
time_patterns.entry(hour).or_default().push(file_id);
|
||||
}
|
||||
|
||||
let prev_file_id: Option<i64> = db
|
||||
.query_row(
|
||||
"SELECT file_id FROM access_log WHERE id = (SELECT MAX(id) - 1 FROM access_log)",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.ok();
|
||||
|
||||
if let Some(prev_id) = prev_file_id {
|
||||
let prev = FileId(prev_id);
|
||||
|
||||
{
|
||||
let mut sequences = self.sequence_counts.write();
|
||||
*sequences.entry((prev, file_id)).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
db.execute(
|
||||
"INSERT INTO sequence_counts (from_file_id, to_file_id, count)
|
||||
VALUES (?1, ?2, 1)
|
||||
ON CONFLICT(from_file_id, to_file_id) DO UPDATE SET count = count + 1",
|
||||
rusqlite::params![prev_id, file_id.0],
|
||||
)?;
|
||||
}
|
||||
|
||||
let cutoff = timestamp - (self.max_history as i64 * 86400);
|
||||
db.execute("DELETE FROM access_log WHERE access_time < ?1", [cutoff])?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn predict_next(&self, current: FileId, limit: usize) -> Vec<FileId> {
|
||||
let sequences = self.sequence_counts.read();
|
||||
|
||||
let mut predictions: Vec<_> = sequences
|
||||
.iter()
|
||||
.filter(|((from, _), count)| *from == current && **count >= 2)
|
||||
.map(|((_, to), count)| (*to, *count))
|
||||
.collect();
|
||||
|
||||
predictions.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
let result: Vec<FileId> = predictions
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|(id, _)| id)
|
||||
.collect();
|
||||
debug!(
|
||||
file_id = current.0,
|
||||
predictions = result.len(),
|
||||
"Predicted next files"
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
pub fn predict_for_time(&self, hour: u8, limit: usize) -> Vec<FileId> {
|
||||
let time_patterns = self.time_patterns.read();
|
||||
|
||||
time_patterns
|
||||
.get(&hour)
|
||||
.map(|files| files.iter().rev().take(limit).copied().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn recently_played(&self, days: u32) -> Result<Vec<FileId>, PatternError> {
|
||||
let cutoff = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs() as i64
|
||||
- (days as i64 * 86400);
|
||||
|
||||
let db = self.db.lock();
|
||||
let mut stmt = db.prepare(
|
||||
"SELECT DISTINCT file_id FROM access_log WHERE access_time >= ?1 ORDER BY access_time DESC",
|
||||
)?;
|
||||
|
||||
let files: Vec<FileId> = stmt
|
||||
.query_map([cutoff], |row| Ok(FileId(row.get(0)?)))?
|
||||
.filter_map(|r| r.ok())
|
||||
.collect();
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
pub fn most_played(&self, limit: u32) -> Result<Vec<FileId>, PatternError> {
|
||||
let db = self.db.lock();
|
||||
let mut stmt = db.prepare(
|
||||
"SELECT file_id, COUNT(*) as play_count FROM access_log
|
||||
GROUP BY file_id ORDER BY play_count DESC LIMIT ?1",
|
||||
)?;
|
||||
|
||||
let files: Vec<FileId> = stmt
|
||||
.query_map([limit], |row| Ok(FileId(row.get(0)?)))?
|
||||
.filter_map(|r| r.ok())
|
||||
.collect();
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum PatternError {
|
||||
#[error("database error: {0}")]
|
||||
Database(#[from] rusqlite::Error),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_pattern_prediction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("patterns.db");
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
let ctx = AccessContext::default();
|
||||
|
||||
for _ in 0..5 {
|
||||
store.record(FileId(1), ctx.clone()).unwrap();
|
||||
store.record(FileId(2), ctx.clone()).unwrap();
|
||||
store.record(FileId(3), ctx.clone()).unwrap();
|
||||
}
|
||||
|
||||
let predictions = store.predict_next(FileId(1), 3);
|
||||
assert!(!predictions.is_empty());
|
||||
assert_eq!(predictions[0], FileId(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pattern_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("patterns.db");
|
||||
let ctx = AccessContext::default();
|
||||
|
||||
{
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
for _ in 0..3 {
|
||||
store.record(FileId(1), ctx.clone()).unwrap();
|
||||
store.record(FileId(2), ctx.clone()).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
let predictions = store.predict_next(FileId(1), 3);
|
||||
assert!(!predictions.is_empty());
|
||||
assert_eq!(predictions[0], FileId(2));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_recently_played() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("patterns.db");
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
let ctx = AccessContext::default();
|
||||
|
||||
store.record(FileId(100), ctx.clone()).unwrap();
|
||||
store.record(FileId(200), ctx.clone()).unwrap();
|
||||
|
||||
let recent = store.recently_played(7).unwrap();
|
||||
assert!(recent.contains(&FileId(100)));
|
||||
assert!(recent.contains(&FileId(200)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_most_played() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db_path = dir.path().join("patterns.db");
|
||||
let store = PatternStore::new(&db_path, 30).unwrap();
|
||||
let ctx = AccessContext::default();
|
||||
|
||||
for _ in 0..5 {
|
||||
store.record(FileId(1), ctx.clone()).unwrap();
|
||||
}
|
||||
for _ in 0..2 {
|
||||
store.record(FileId(2), ctx.clone()).unwrap();
|
||||
}
|
||||
|
||||
let most = store.most_played(10).unwrap();
|
||||
assert_eq!(most[0], FileId(1));
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
use crate::patterns::{AccessContext, PatternStore};
|
||||
use musicfs_cas::ContentFetcher;
|
||||
use musicfs_core::{Event, EventBus, FileId};
|
||||
use parking_lot::Mutex as ParkingMutex;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const DEFAULT_PREFETCH_LOOKAHEAD: usize = 3;
|
||||
const DEFAULT_MAX_CONCURRENT: usize = 2;
|
||||
const DEFAULT_COOLDOWN_MS: u64 = 100;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PrefetchConfig {
|
||||
pub lookahead: usize,
|
||||
pub max_concurrent: usize,
|
||||
pub cooldown: Duration,
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for PrefetchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
lookahead: DEFAULT_PREFETCH_LOOKAHEAD,
|
||||
max_concurrent: DEFAULT_MAX_CONCURRENT,
|
||||
cooldown: Duration::from_millis(DEFAULT_COOLDOWN_MS),
|
||||
enabled: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PrefetchEngine {
|
||||
config: PrefetchConfig,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
in_flight: Arc<ParkingMutex<HashSet<FileId>>>,
|
||||
semaphore: Arc<Semaphore>,
|
||||
running: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
pub struct PrefetchHandle {
|
||||
handle: JoinHandle<()>,
|
||||
running: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl PrefetchHandle {
|
||||
pub async fn stop(self) {
|
||||
self.running.store(false, Ordering::SeqCst);
|
||||
let _ = self.handle.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl PrefetchEngine {
|
||||
pub fn new(
|
||||
config: PrefetchConfig,
|
||||
_pattern_store: Arc<PatternStore>,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
) -> Self {
|
||||
let semaphore = Arc::new(Semaphore::new(config.max_concurrent));
|
||||
|
||||
Self {
|
||||
config,
|
||||
fetcher,
|
||||
in_flight: Arc::new(ParkingMutex::new(HashSet::new())),
|
||||
semaphore,
|
||||
running: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(
|
||||
self: Arc<Self>,
|
||||
event_bus: Arc<EventBus>,
|
||||
pattern_store: Arc<PatternStore>,
|
||||
) -> PrefetchHandle {
|
||||
self.running.store(true, Ordering::SeqCst);
|
||||
let running = self.running.clone();
|
||||
|
||||
let config = self.config.clone();
|
||||
let fetcher = self.fetcher.clone();
|
||||
let in_flight = self.in_flight.clone();
|
||||
let semaphore = self.semaphore.clone();
|
||||
let running_inner = running.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut rx = event_bus.subscribe();
|
||||
|
||||
while running_inner.load(Ordering::SeqCst) {
|
||||
match tokio::time::timeout(Duration::from_secs(1), rx.recv()).await {
|
||||
Ok(Ok(event)) => {
|
||||
if let Event::FileAccessed { file_id, .. } = event {
|
||||
if config.enabled {
|
||||
let ctx = AccessContext::default();
|
||||
if let Err(e) = pattern_store.record(file_id, ctx) {
|
||||
warn!("Failed to record access pattern: {}", e);
|
||||
continue;
|
||||
}
|
||||
|
||||
let predictions =
|
||||
pattern_store.predict_next(file_id, config.lookahead);
|
||||
|
||||
for predicted_id in predictions {
|
||||
prefetch_file(predicted_id, &fetcher, &in_flight, &semaphore)
|
||||
.await;
|
||||
}
|
||||
|
||||
tokio::time::sleep(config.cooldown).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(_)) => break,
|
||||
Err(_) => continue,
|
||||
}
|
||||
}
|
||||
|
||||
info!("Prefetch engine stopped");
|
||||
});
|
||||
|
||||
PrefetchHandle { handle, running }
|
||||
}
|
||||
|
||||
pub fn is_running(&self) -> bool {
|
||||
self.running.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn in_flight_count(&self) -> usize {
|
||||
self.in_flight.lock().len()
|
||||
}
|
||||
|
||||
pub fn update_config(&mut self, config: PrefetchConfig) {
|
||||
self.config = config;
|
||||
}
|
||||
}
|
||||
|
||||
async fn prefetch_file(
|
||||
file_id: FileId,
|
||||
fetcher: &Arc<ContentFetcher>,
|
||||
in_flight: &Arc<ParkingMutex<HashSet<FileId>>>,
|
||||
semaphore: &Arc<Semaphore>,
|
||||
) {
|
||||
{
|
||||
let mut guard = in_flight.lock();
|
||||
if guard.contains(&file_id) {
|
||||
debug!("Skipping prefetch for {:?} - already in flight", file_id);
|
||||
return;
|
||||
}
|
||||
guard.insert(file_id);
|
||||
}
|
||||
|
||||
let permit = match semaphore.clone().try_acquire_owned() {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
debug!("Skipping prefetch for {:?} - concurrency limit", file_id);
|
||||
in_flight.lock().remove(&file_id);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let fetcher = fetcher.clone();
|
||||
let in_flight = in_flight.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
debug!("Prefetching file {:?}", file_id);
|
||||
|
||||
match fetcher.ensure_cached(file_id).await {
|
||||
Ok(manifest) => {
|
||||
info!(
|
||||
"Prefetched {:?}: {} chunks, {} bytes",
|
||||
file_id,
|
||||
manifest.chunks.len(),
|
||||
manifest.total_size
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Prefetch failed for {:?}: {}", file_id, e);
|
||||
}
|
||||
}
|
||||
|
||||
in_flight.lock().remove(&file_id);
|
||||
drop(permit);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_prefetch_config_defaults() {
|
||||
let config = PrefetchConfig::default();
|
||||
assert_eq!(config.lookahead, 3);
|
||||
assert_eq!(config.max_concurrent, 2);
|
||||
assert!(config.enabled);
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA foreign_keys = ON;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
id INTEGER PRIMARY KEY,
|
||||
origin_id TEXT NOT NULL,
|
||||
real_path TEXT NOT NULL,
|
||||
virtual_path TEXT NOT NULL,
|
||||
|
||||
title TEXT,
|
||||
artist TEXT,
|
||||
album TEXT,
|
||||
album_artist TEXT,
|
||||
genre TEXT,
|
||||
year INTEGER,
|
||||
track INTEGER,
|
||||
disc INTEGER,
|
||||
duration_ms INTEGER,
|
||||
bitrate INTEGER,
|
||||
sample_rate INTEGER,
|
||||
format TEXT,
|
||||
track_total INTEGER,
|
||||
disc_total INTEGER,
|
||||
date TEXT,
|
||||
composer TEXT,
|
||||
comment TEXT,
|
||||
lyrics TEXT,
|
||||
copyright TEXT,
|
||||
compilation INTEGER,
|
||||
artist_sort TEXT,
|
||||
album_artist_sort TEXT,
|
||||
album_sort TEXT,
|
||||
title_sort TEXT,
|
||||
mb_recording_id TEXT,
|
||||
mb_album_id TEXT,
|
||||
mb_artist_id TEXT,
|
||||
mb_album_artist_id TEXT,
|
||||
mb_release_group_id TEXT,
|
||||
replaygain_track_gain REAL,
|
||||
replaygain_track_peak REAL,
|
||||
replaygain_album_gain REAL,
|
||||
replaygain_album_peak REAL,
|
||||
channels INTEGER,
|
||||
bits_per_sample INTEGER,
|
||||
encoder TEXT,
|
||||
custom_tags TEXT,
|
||||
format_layout BLOB,
|
||||
|
||||
label TEXT,
|
||||
album_type TEXT,
|
||||
cover_url TEXT,
|
||||
genres_json TEXT,
|
||||
enrichment_source TEXT,
|
||||
enriched_at INTEGER,
|
||||
enrichment_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_enrichment_error TEXT,
|
||||
|
||||
origin_mtime INTEGER NOT NULL,
|
||||
origin_size INTEGER NOT NULL,
|
||||
content_hash TEXT,
|
||||
chunk_manifest BLOB,
|
||||
last_sync INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
|
||||
|
||||
trashed INTEGER NOT NULL DEFAULT 0,
|
||||
original_path TEXT,
|
||||
trashed_at INTEGER,
|
||||
|
||||
UNIQUE(origin_id, real_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artwork (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
||||
art_type TEXT NOT NULL,
|
||||
chunk_hash TEXT NOT NULL,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
mime_type TEXT,
|
||||
UNIQUE(file_id, art_type)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS collections (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
query_json TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
|
||||
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_files_virtual ON files(virtual_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_artist_album ON files(artist, album);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_content_hash ON files(content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_real ON files(origin_id, real_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_origin ON files(origin_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_last_sync ON files(last_sync);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_mb_album ON files(mb_album_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_mb_artist ON files(mb_artist_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_genre ON files(genre);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_year ON files(year);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_composer ON files(composer);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_file ON artwork(file_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS directories (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_directories_path ON directories(path);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_trashed ON files(trashed) WHERE trashed = 1;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,29 +0,0 @@
|
||||
[package]
|
||||
name = "musicfs-cas"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
failpoints = ["fail/failpoints"]
|
||||
|
||||
[dependencies]
|
||||
fail = { workspace = true, optional = true }
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-origins = { path = "../musicfs-origins" }
|
||||
musicfs-sync = { path = "../musicfs-sync" }
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
serde.workspace = true
|
||||
sled.workspace = true
|
||||
xxhash-rust.workspace = true
|
||||
bytes.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
hex.workspace = true
|
||||
dirs.workspace = true
|
||||
thiserror.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
musicfs-cache = { path = "../musicfs-cache" }
|
||||
@@ -1,45 +0,0 @@
|
||||
use musicfs_core::ChunkHash;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkLocation {
|
||||
pub path: PathBuf,
|
||||
pub size: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkRef {
|
||||
pub hash: ChunkHash,
|
||||
pub offset: u64,
|
||||
pub size: u32,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_chunk_hash_from_bytes() {
|
||||
let data = b"hello world";
|
||||
let hash = ChunkHash::from_bytes(data);
|
||||
assert_eq!(hash.as_hex().len(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunk_hash_deterministic() {
|
||||
let data = b"test data";
|
||||
let hash1 = ChunkHash::from_bytes(data);
|
||||
let hash2 = ChunkHash::from_bytes(data);
|
||||
assert_eq!(hash1, hash2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunk_hash_hex_roundtrip() {
|
||||
let data = b"roundtrip test";
|
||||
let hash = ChunkHash::from_bytes(data);
|
||||
let hex = hash.as_hex();
|
||||
let restored = ChunkHash::from_hex(&hex).unwrap();
|
||||
assert_eq!(hash, restored);
|
||||
}
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
use crate::{CasStore, ChunkManifest, ChunkRef};
|
||||
use musicfs_core::{Event, EventBus, FileId, FileMeta, OriginId};
|
||||
use musicfs_origins::Origin;
|
||||
use musicfs_sync::CdcChunker;
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
pub struct ContentFetcher {
|
||||
store: Arc<CasStore>,
|
||||
origins: RwLock<HashMap<OriginId, Arc<dyn Origin>>>,
|
||||
file_meta: RwLock<HashMap<FileId, FileMeta>>,
|
||||
event_bus: Option<Arc<EventBus>>,
|
||||
chunker: CdcChunker,
|
||||
}
|
||||
|
||||
impl ContentFetcher {
|
||||
pub fn new(store: Arc<CasStore>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
origins: RwLock::new(HashMap::new()),
|
||||
file_meta: RwLock::new(HashMap::new()),
|
||||
event_bus: None,
|
||||
chunker: CdcChunker::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_event_bus(store: Arc<CasStore>, event_bus: Arc<EventBus>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
origins: RwLock::new(HashMap::new()),
|
||||
file_meta: RwLock::new(HashMap::new()),
|
||||
event_bus: Some(event_bus),
|
||||
chunker: CdcChunker::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_origin(&self, origin: Arc<dyn Origin>) {
|
||||
let id = origin.id().clone();
|
||||
self.origins.write().insert(id, origin);
|
||||
}
|
||||
|
||||
pub fn register_file(&self, meta: FileMeta) {
|
||||
self.file_meta.write().insert(meta.id, meta);
|
||||
}
|
||||
|
||||
pub fn register_files(&self, files: impl IntoIterator<Item = FileMeta>) {
|
||||
let mut map = self.file_meta.write();
|
||||
for meta in files {
|
||||
map.insert(meta.id, meta);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn fetch_file(&self, file_id: FileId) -> Result<ChunkManifest, FetchError> {
|
||||
let meta = {
|
||||
let files = self.file_meta.read();
|
||||
files
|
||||
.get(&file_id)
|
||||
.cloned()
|
||||
.ok_or(FetchError::FileNotFound(file_id))?
|
||||
};
|
||||
|
||||
let origin = {
|
||||
let origins = self.origins.read();
|
||||
origins
|
||||
.get(&meta.real_path.origin_id)
|
||||
.cloned()
|
||||
.ok_or_else(|| FetchError::OriginNotFound(meta.real_path.origin_id.clone()))?
|
||||
};
|
||||
|
||||
info!("Fetching file {:?} from origin {}", file_id, origin.id());
|
||||
|
||||
let data = origin
|
||||
.read_full(&meta.real_path.path)
|
||||
.await
|
||||
.map_err(|e| FetchError::OriginRead(e.to_string()))?;
|
||||
|
||||
let mtime = meta
|
||||
.mtime
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0);
|
||||
|
||||
let chunks = self.chunker.chunk_refs(&data);
|
||||
info!("Chunked {:?} into {} chunks", file_id, chunks.len());
|
||||
|
||||
let mut chunk_refs = Vec::with_capacity(chunks.len());
|
||||
for chunk in chunks {
|
||||
if !self.store.exists(&chunk.hash) {
|
||||
if let Err(e) = self.store.put(chunk.data).await {
|
||||
warn!(hash = %chunk.hash, error = %e, "CAS write failed, continuing in passthrough mode");
|
||||
}
|
||||
}
|
||||
|
||||
chunk_refs.push(ChunkRef {
|
||||
hash: chunk.hash,
|
||||
offset: chunk.offset,
|
||||
size: chunk.length,
|
||||
});
|
||||
}
|
||||
|
||||
let manifest = ChunkManifest {
|
||||
file_id,
|
||||
total_size: meta.size,
|
||||
mtime,
|
||||
chunks: chunk_refs,
|
||||
};
|
||||
|
||||
debug!(
|
||||
"Created manifest for {:?}: {} bytes, {} chunks",
|
||||
file_id,
|
||||
meta.size,
|
||||
manifest.chunks.len()
|
||||
);
|
||||
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
pub async fn ensure_cached(&self, file_id: FileId) -> Result<ChunkManifest, FetchError> {
|
||||
self.fetch_file(file_id).await
|
||||
}
|
||||
|
||||
pub fn get_file_meta(&self, file_id: FileId) -> Option<FileMeta> {
|
||||
self.file_meta.read().get(&file_id).cloned()
|
||||
}
|
||||
|
||||
pub fn emit_access_event(&self, meta: &FileMeta, offset: u64, size: u32) {
|
||||
if let Some(bus) = &self.event_bus {
|
||||
bus.publish(Event::FileAccessed {
|
||||
file_id: meta.id,
|
||||
path: meta.virtual_path.clone(),
|
||||
origin_id: meta.real_path.origin_id.clone(),
|
||||
offset,
|
||||
size,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FetchError {
|
||||
#[error("File not found: {0:?}")]
|
||||
FileNotFound(FileId),
|
||||
|
||||
#[error("Origin not found: {0}")]
|
||||
OriginNotFound(OriginId),
|
||||
|
||||
#[error("Origin read error: {0}")]
|
||||
OriginRead(String),
|
||||
|
||||
#[error("Store error: {0}")]
|
||||
Store(#[from] crate::CasError),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::CasConfig;
|
||||
use musicfs_core::{RealPath, VirtualPath};
|
||||
use musicfs_origins::LocalOrigin;
|
||||
use std::path::PathBuf;
|
||||
use std::time::SystemTime;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_file() {
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
|
||||
std::fs::write(origin_dir.path().join("test.flac"), b"fake audio data").unwrap();
|
||||
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
let fetcher = ContentFetcher::new(store.clone());
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new("local", origin_dir.path()));
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/Artist/Album/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("local"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: 15,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(meta);
|
||||
|
||||
let manifest = fetcher.fetch_file(FileId(1)).await.unwrap();
|
||||
assert_eq!(manifest.total_size, 15);
|
||||
assert_eq!(manifest.chunks.len(), 1);
|
||||
|
||||
let data = store.get(&manifest.chunks[0].hash).await.unwrap();
|
||||
assert_eq!(&data[..], b"fake audio data");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_file_not_found() {
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
let fetcher = ContentFetcher::new(store);
|
||||
|
||||
let result = fetcher.fetch_file(FileId(999)).await;
|
||||
assert!(matches!(result, Err(FetchError::FileNotFound(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_emits_event() {
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
std::fs::write(origin_dir.path().join("test.flac"), b"audio").unwrap();
|
||||
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
let event_bus = Arc::new(EventBus::default());
|
||||
let mut rx = event_bus.subscribe();
|
||||
|
||||
let fetcher = ContentFetcher::with_event_bus(store, event_bus);
|
||||
let origin = Arc::new(LocalOrigin::new("local", origin_dir.path()));
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/Artist/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("local"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: 5,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(meta.clone());
|
||||
|
||||
fetcher.emit_access_event(&meta, 0, 5);
|
||||
|
||||
let event = rx.try_recv().unwrap();
|
||||
assert!(matches!(event, Event::FileAccessed { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_origin_not_found() {
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
let fetcher = ContentFetcher::new(store);
|
||||
|
||||
let meta = FileMeta {
|
||||
id: FileId(1),
|
||||
virtual_path: VirtualPath::new("/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("nonexistent"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: 100,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(meta);
|
||||
|
||||
let result = fetcher.fetch_file(FileId(1)).await;
|
||||
assert!(matches!(result, Err(FetchError::OriginNotFound(_))));
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
mod chunks;
|
||||
mod fetcher;
|
||||
mod reader;
|
||||
mod store;
|
||||
|
||||
pub use chunks::{ChunkLocation, ChunkRef};
|
||||
pub use fetcher::{ContentFetcher, FetchError};
|
||||
pub use reader::{ChunkManifest, FileReader, ReaderError};
|
||||
pub use store::{CasConfig, CasError, CasStore, DedupStats};
|
||||
@@ -1,332 +0,0 @@
|
||||
use crate::chunks::ChunkRef;
|
||||
use crate::fetcher::{ContentFetcher, FetchError};
|
||||
use crate::store::{CasError, CasStore};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use musicfs_core::FileId;
|
||||
use parking_lot::RwLock;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkManifest {
|
||||
pub file_id: FileId,
|
||||
pub total_size: u64,
|
||||
pub mtime: i64,
|
||||
pub chunks: Vec<ChunkRef>,
|
||||
}
|
||||
|
||||
impl ChunkManifest {
|
||||
pub fn chunks_to_bytes(&self) -> Vec<u8> {
|
||||
rmp_serde::to_vec(&self.chunks).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn chunks_from_bytes(data: &[u8]) -> Option<Vec<ChunkRef>> {
|
||||
rmp_serde::from_slice(data).ok()
|
||||
}
|
||||
|
||||
pub fn from_db(
|
||||
file_id: FileId,
|
||||
total_size: u64,
|
||||
mtime: i64,
|
||||
chunk_blob: &[u8],
|
||||
) -> Option<Self> {
|
||||
let chunks = Self::chunks_from_bytes(chunk_blob)?;
|
||||
Some(Self {
|
||||
file_id,
|
||||
total_size,
|
||||
mtime,
|
||||
chunks,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FileReader {
|
||||
store: Arc<CasStore>,
|
||||
fetcher: Option<Arc<ContentFetcher>>,
|
||||
manifests: RwLock<HashMap<FileId, ChunkManifest>>,
|
||||
}
|
||||
|
||||
impl FileReader {
|
||||
pub fn new(store: Arc<CasStore>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
fetcher: None,
|
||||
manifests: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_fetcher(store: Arc<CasStore>, fetcher: Arc<ContentFetcher>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
fetcher: Some(fetcher),
|
||||
manifests: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_manifest(&self, manifest: ChunkManifest) {
|
||||
let mut manifests = self.manifests.write();
|
||||
manifests.insert(manifest.file_id, manifest);
|
||||
}
|
||||
|
||||
async fn get_or_fetch_manifest(&self, file_id: FileId) -> Result<ChunkManifest, ReaderError> {
|
||||
{
|
||||
let manifests = self.manifests.read();
|
||||
if let Some(m) = manifests.get(&file_id) {
|
||||
trace!(file_id = ?file_id, "manifest cache hit");
|
||||
return Ok(m.clone());
|
||||
}
|
||||
}
|
||||
|
||||
trace!(file_id = ?file_id, "manifest cache miss");
|
||||
let Some(fetcher) = &self.fetcher else {
|
||||
return Err(ReaderError::ManifestNotFound(file_id));
|
||||
};
|
||||
|
||||
let manifest = fetcher.ensure_cached(file_id).await?;
|
||||
self.manifests.write().insert(file_id, manifest.clone());
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
pub async fn read(
|
||||
&self,
|
||||
file_id: FileId,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> Result<Bytes, ReaderError> {
|
||||
let manifest = self.get_or_fetch_manifest(file_id).await?;
|
||||
|
||||
if let Some(fetcher) = &self.fetcher {
|
||||
if let Some(meta) = fetcher.get_file_meta(file_id) {
|
||||
fetcher.emit_access_event(&meta, offset, size);
|
||||
}
|
||||
}
|
||||
|
||||
if offset >= manifest.total_size {
|
||||
return Ok(Bytes::new());
|
||||
}
|
||||
|
||||
let end = std::cmp::min(offset + size as u64, manifest.total_size);
|
||||
let mut result = BytesMut::with_capacity((end - offset) as usize);
|
||||
let mut chunks_read = 0u32;
|
||||
|
||||
for chunk_ref in &manifest.chunks {
|
||||
let chunk_start = chunk_ref.offset;
|
||||
let chunk_end = chunk_ref.offset + chunk_ref.size as u64;
|
||||
|
||||
if chunk_end <= offset || chunk_start >= end {
|
||||
continue;
|
||||
}
|
||||
|
||||
let chunk_data = match self.store.get(&chunk_ref.hash).await {
|
||||
Ok(data) => data,
|
||||
Err(CasError::IntegrityError { .. }) => {
|
||||
warn!(hash = %chunk_ref.hash, "Chunk corrupt, deleting and re-fetching");
|
||||
let _ = self.store.delete(&chunk_ref.hash).await;
|
||||
if let Some(fetcher) = &self.fetcher {
|
||||
let new_manifest = fetcher.fetch_file(file_id).await?;
|
||||
self.manifests.write().insert(file_id, new_manifest);
|
||||
self.store.get(&chunk_ref.hash).await?
|
||||
} else {
|
||||
return Err(ReaderError::Cas(CasError::NotFound(
|
||||
chunk_ref.hash.as_hex(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(CasError::NotFound(_)) => {
|
||||
warn!(hash = %chunk_ref.hash, "Chunk missing, attempting re-fetch");
|
||||
if let Some(fetcher) = &self.fetcher {
|
||||
let new_manifest = fetcher.fetch_file(file_id).await?;
|
||||
self.manifests.write().insert(file_id, new_manifest);
|
||||
self.store.get(&chunk_ref.hash).await?
|
||||
} else {
|
||||
return Err(ReaderError::Cas(CasError::NotFound(
|
||||
chunk_ref.hash.as_hex(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(ReaderError::Cas(e)),
|
||||
};
|
||||
|
||||
let read_start = if offset > chunk_start {
|
||||
(offset - chunk_start) as usize
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let read_end = if end < chunk_end {
|
||||
(end - chunk_start) as usize
|
||||
} else {
|
||||
chunk_ref.size as usize
|
||||
};
|
||||
|
||||
result.extend_from_slice(&chunk_data[read_start..read_end]);
|
||||
chunks_read += 1;
|
||||
}
|
||||
|
||||
let bytes_read = result.len() as u64;
|
||||
debug!(file_id = ?file_id, offset, size, chunks_read, bytes_read, "read completed");
|
||||
Ok(result.freeze())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ReaderError {
|
||||
#[error("Manifest not found for file {0:?}")]
|
||||
ManifestNotFound(FileId),
|
||||
|
||||
#[error("Fetch error: {0}")]
|
||||
Fetch(#[from] FetchError),
|
||||
|
||||
#[error("CAS error: {0}")]
|
||||
Cas(#[from] crate::store::CasError),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::CasConfig;
|
||||
use musicfs_core::ChunkHash;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_reader_simple() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let data = b"Hello, World!";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let reader = FileReader::new(store);
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 0,
|
||||
size: data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let result = reader.read(FileId(1), 0, data.len() as u32).await.unwrap();
|
||||
assert_eq!(&result[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_reader_partial() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let data = b"ABCDEFGHIJ";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let reader = FileReader::new(store);
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 0,
|
||||
size: data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let result = reader.read(FileId(1), 3, 4).await.unwrap();
|
||||
assert_eq!(&result[..], b"DEFG");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_reader_multi_chunk() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let chunk1 = b"AAAA";
|
||||
let chunk2 = b"BBBB";
|
||||
let hash1 = store.put(chunk1).await.unwrap();
|
||||
let hash2 = store.put(chunk2).await.unwrap();
|
||||
|
||||
let reader = FileReader::new(store);
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: 8,
|
||||
mtime: 0,
|
||||
chunks: vec![
|
||||
ChunkRef {
|
||||
hash: hash1,
|
||||
offset: 0,
|
||||
size: 4,
|
||||
},
|
||||
ChunkRef {
|
||||
hash: hash2,
|
||||
offset: 4,
|
||||
size: 4,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
let result = reader.read(FileId(1), 2, 4).await.unwrap();
|
||||
assert_eq!(&result[..], b"AABB");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_reader_eof() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let data = b"short";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let reader = FileReader::new(store);
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash,
|
||||
offset: 0,
|
||||
size: data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let result = reader.read(FileId(1), 100, 10).await.unwrap();
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunk_manifest_serialization() {
|
||||
let manifest = ChunkManifest {
|
||||
file_id: FileId(42),
|
||||
total_size: 1024,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash: ChunkHash::from_bytes(b"test"),
|
||||
offset: 0,
|
||||
size: 1024,
|
||||
}],
|
||||
};
|
||||
|
||||
let bytes = manifest.chunks_to_bytes();
|
||||
let restored = ChunkManifest::chunks_from_bytes(&bytes).unwrap();
|
||||
assert_eq!(restored.len(), 1);
|
||||
assert_eq!(restored[0].size, 1024);
|
||||
}
|
||||
}
|
||||
@@ -1,396 +0,0 @@
|
||||
use crate::chunks::ChunkLocation;
|
||||
use bytes::Bytes;
|
||||
use musicfs_core::ChunkHash;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::fs;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
use fail::fail_point;
|
||||
|
||||
const DEFAULT_MAX_SIZE_10GB: u64 = 10 * 1024 * 1024 * 1024;
|
||||
const DEFAULT_SHARD_LEVELS_256_SUBDIRS: u8 = 2;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CasConfig {
|
||||
pub chunks_dir: PathBuf,
|
||||
pub max_size: u64,
|
||||
pub shard_levels: u8,
|
||||
}
|
||||
|
||||
impl Default for CasConfig {
|
||||
fn default() -> Self {
|
||||
let cache_dir = dirs::cache_dir()
|
||||
.unwrap_or_else(|| PathBuf::from(".cache"))
|
||||
.join("musicfs")
|
||||
.join("chunks");
|
||||
|
||||
Self {
|
||||
chunks_dir: cache_dir,
|
||||
max_size: DEFAULT_MAX_SIZE_10GB,
|
||||
shard_levels: DEFAULT_SHARD_LEVELS_256_SUBDIRS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CasStore {
|
||||
config: CasConfig,
|
||||
index: sled::Db,
|
||||
current_size: AtomicU64,
|
||||
}
|
||||
|
||||
impl CasStore {
|
||||
pub async fn open(config: CasConfig) -> Result<Self, CasError> {
|
||||
fs::create_dir_all(&config.chunks_dir).await?;
|
||||
|
||||
let index_path = config.chunks_dir.join("index.sled");
|
||||
let index = match sled::open(&index_path) {
|
||||
Ok(db) => db,
|
||||
Err(e) => {
|
||||
warn!(error = %e, path = ?index_path, "sled index corrupted, attempting recovery");
|
||||
|
||||
match sled::Config::new().path(&index_path).open() {
|
||||
Ok(db) => {
|
||||
info!("sled index repaired successfully");
|
||||
db
|
||||
}
|
||||
Err(repair_err) => {
|
||||
warn!(error = %repair_err, "sled repair failed, recreating index");
|
||||
if index_path.exists() {
|
||||
std::fs::remove_dir_all(&index_path).map_err(CasError::Io)?;
|
||||
}
|
||||
sled::open(&index_path)?
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let current_size = Self::calculate_size(&config.chunks_dir).await;
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
index,
|
||||
current_size: AtomicU64::new(current_size),
|
||||
})
|
||||
}
|
||||
|
||||
async fn calculate_size(dir: &Path) -> u64 {
|
||||
Self::calculate_size_recursive(dir).await
|
||||
}
|
||||
|
||||
fn calculate_size_recursive(
|
||||
dir: &Path,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = u64> + Send + '_>> {
|
||||
Box::pin(async move {
|
||||
let mut size = 0u64;
|
||||
if let Ok(mut entries) = fs::read_dir(dir).await {
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
if let Ok(meta) = entry.metadata().await {
|
||||
if meta.is_file() {
|
||||
size += meta.len();
|
||||
} else if meta.is_dir() {
|
||||
// Skip sled index directory
|
||||
let name = entry.file_name();
|
||||
if name != "index.sled" {
|
||||
size += Self::calculate_size_recursive(&entry.path()).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
size
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn put(&self, data: &[u8]) -> Result<ChunkHash, CasError> {
|
||||
let hash = ChunkHash::from_bytes(data);
|
||||
let path = self.chunk_path(&hash);
|
||||
|
||||
if path.exists() {
|
||||
trace!(hash = %hash, size_bytes = data.len(), "dedup hit");
|
||||
return Ok(hash);
|
||||
}
|
||||
|
||||
if self.config.max_size > 0 {
|
||||
let new_size = self.current_size.load(Ordering::SeqCst) + data.len() as u64;
|
||||
if new_size > self.config.max_size {
|
||||
warn!(
|
||||
current_size = self.current_size.load(Ordering::SeqCst),
|
||||
chunk_size = data.len(),
|
||||
max_size = self.config.max_size,
|
||||
"CAS store full, rejecting write"
|
||||
);
|
||||
return Err(CasError::StoreFull {
|
||||
current: self.current_size.load(Ordering::SeqCst),
|
||||
max: self.config.max_size,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
fail_point!("cas-put-before-write", |_| {
|
||||
Err(CasError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Failpoint: cas-put-before-write",
|
||||
)))
|
||||
});
|
||||
|
||||
fs::write(&path, data).await?;
|
||||
|
||||
#[cfg(feature = "failpoints")]
|
||||
fail_point!("cas-put-after-write-before-index", |_| {
|
||||
Err(CasError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Failpoint: cas-put-after-write-before-index",
|
||||
)))
|
||||
});
|
||||
|
||||
let location = ChunkLocation {
|
||||
path: path.clone(),
|
||||
size: data.len() as u32,
|
||||
};
|
||||
self.index.insert(
|
||||
hash.0.as_slice(),
|
||||
rmp_serde::to_vec(&location).map_err(|e| CasError::Serialization(e.to_string()))?,
|
||||
)?;
|
||||
|
||||
self.current_size
|
||||
.fetch_add(data.len() as u64, Ordering::SeqCst);
|
||||
|
||||
debug!(hash = %hash, size_bytes = data.len(), "chunk stored");
|
||||
Ok(hash)
|
||||
}
|
||||
|
||||
pub async fn get(&self, hash: &ChunkHash) -> Result<Bytes, CasError> {
|
||||
let path = self.chunk_path(hash);
|
||||
|
||||
if !path.exists() {
|
||||
return Err(CasError::NotFound(hash.as_hex()));
|
||||
}
|
||||
|
||||
let data = fs::read(&path).await?;
|
||||
|
||||
if self.config.max_size > 0 {
|
||||
self.verify_integrity(hash, &data)?;
|
||||
}
|
||||
|
||||
debug!(hash = %hash, size_bytes = data.len(), "chunk retrieved");
|
||||
Ok(Bytes::from(data))
|
||||
}
|
||||
|
||||
pub fn exists(&self, hash: &ChunkHash) -> bool {
|
||||
self.chunk_path(hash).exists()
|
||||
}
|
||||
|
||||
fn verify_integrity(&self, expected: &ChunkHash, data: &[u8]) -> Result<(), CasError> {
|
||||
let actual = ChunkHash::from_bytes(data);
|
||||
if actual != *expected {
|
||||
warn!(
|
||||
"Chunk integrity failure: expected {}, got {}",
|
||||
expected, actual
|
||||
);
|
||||
return Err(CasError::IntegrityError {
|
||||
expected: expected.as_hex(),
|
||||
actual: actual.as_hex(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn chunk_path(&self, hash: &ChunkHash) -> PathBuf {
|
||||
let hex = hash.as_hex();
|
||||
let mut path = self.config.chunks_dir.clone();
|
||||
|
||||
for i in 0..self.config.shard_levels as usize {
|
||||
let start = i * 2;
|
||||
let end = start + 2;
|
||||
if end <= hex.len() {
|
||||
path = path.join(&hex[start..end]);
|
||||
}
|
||||
}
|
||||
|
||||
path.join(&hex)
|
||||
}
|
||||
|
||||
pub async fn delete(&self, hash: &ChunkHash) -> Result<(), CasError> {
|
||||
let path = self.chunk_path(hash);
|
||||
|
||||
if path.exists() {
|
||||
let meta = fs::metadata(&path).await?;
|
||||
fs::remove_file(&path).await?;
|
||||
self.index.remove(hash.0.as_slice())?;
|
||||
self.current_size.fetch_sub(meta.len(), Ordering::SeqCst);
|
||||
debug!(hash = %hash, size_bytes = meta.len(), "chunk deleted");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn current_size(&self) -> u64 {
|
||||
self.current_size.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn max_size(&self) -> u64 {
|
||||
self.config.max_size
|
||||
}
|
||||
|
||||
pub fn list_chunks(&self) -> impl Iterator<Item = ChunkHash> + '_ {
|
||||
self.index.iter().filter_map(|r| {
|
||||
r.ok().and_then(|(k, _)| {
|
||||
if k.len() == 8 {
|
||||
let mut arr = [0u8; 8];
|
||||
arr.copy_from_slice(&k);
|
||||
Some(ChunkHash(arr))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn dedup_stats(&self) -> DedupStats {
|
||||
let chunks_stored = self.index.len() as u64;
|
||||
let size_bytes = self.current_size();
|
||||
|
||||
DedupStats {
|
||||
chunks_stored,
|
||||
chunks_unique: chunks_stored,
|
||||
size_bytes,
|
||||
size_limit_bytes: self.config.max_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DedupStats {
|
||||
pub chunks_stored: u64,
|
||||
pub chunks_unique: u64,
|
||||
pub size_bytes: u64,
|
||||
pub size_limit_bytes: u64,
|
||||
}
|
||||
|
||||
impl DedupStats {
|
||||
pub fn dedup_ratio(&self) -> f64 {
|
||||
if self.chunks_stored == 0 {
|
||||
0.0
|
||||
} else {
|
||||
1.0 - (self.chunks_unique as f64 / self.chunks_stored as f64)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CasError {
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("Sled error: {0}")]
|
||||
Sled(#[from] sled::Error),
|
||||
|
||||
#[error("Chunk not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
#[error("Integrity error: expected {expected}, got {actual}")]
|
||||
IntegrityError { expected: String, actual: String },
|
||||
|
||||
#[error("Serialization error: {0}")]
|
||||
Serialization(String),
|
||||
|
||||
#[error("Store full: {current} / {max} bytes")]
|
||||
StoreFull { current: u64, max: u64 },
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
async fn test_store() -> (CasStore, TempDir) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
max_size: 1024 * 1024,
|
||||
shard_levels: 2,
|
||||
};
|
||||
let store = CasStore::open(config).await.unwrap();
|
||||
(store, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_put_get() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"test chunk data";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let retrieved = store.get(&hash).await.unwrap();
|
||||
assert_eq!(&retrieved[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_dedup() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"duplicate data";
|
||||
let hash1 = store.put(data).await.unwrap();
|
||||
let hash2 = store.put(data).await.unwrap();
|
||||
|
||||
assert_eq!(hash1, hash2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_exists() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"existence test";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
assert!(store.exists(&hash));
|
||||
|
||||
let fake_hash = ChunkHash::from_bytes(b"nonexistent");
|
||||
assert!(!store.exists(&fake_hash));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_delete() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"delete me";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
assert!(store.exists(&hash));
|
||||
|
||||
store.delete(&hash).await.unwrap();
|
||||
|
||||
assert!(!store.exists(&hash));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_integrity() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
let data = b"integrity test";
|
||||
let hash = store.put(data).await.unwrap();
|
||||
|
||||
let retrieved = store.get(&hash).await.unwrap();
|
||||
assert_eq!(&retrieved[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_dedup_stats() {
|
||||
let (store, _dir) = test_store().await;
|
||||
|
||||
store.put(b"chunk1").await.unwrap();
|
||||
store.put(b"chunk2").await.unwrap();
|
||||
store.put(b"chunk1").await.unwrap();
|
||||
|
||||
let stats = store.dedup_stats();
|
||||
assert_eq!(stats.chunks_stored, 2);
|
||||
assert_eq!(stats.chunks_unique, 2);
|
||||
}
|
||||
}
|
||||
@@ -1,203 +0,0 @@
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_cas::{CasConfig, CasStore, ChunkManifest, ChunkRef, ContentFetcher, FileReader};
|
||||
use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath};
|
||||
use musicfs_origins::LocalOrigin;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::SystemTime;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from("/test"),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cas_and_tree_integration() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let file_data = b"This is test audio file content for testing.";
|
||||
let chunk_hash = store.put(file_data).await.unwrap();
|
||||
|
||||
let mut builder = TreeBuilder::new();
|
||||
builder.add_file(&make_file_meta(
|
||||
1,
|
||||
"/Artist/Album/Track.flac",
|
||||
file_data.len() as u64,
|
||||
));
|
||||
let _tree = Arc::new(RwLock::new(builder.build()));
|
||||
|
||||
let reader = Arc::new(FileReader::new(store.clone()));
|
||||
reader.register_manifest(ChunkManifest {
|
||||
file_id: FileId(1),
|
||||
total_size: file_data.len() as u64,
|
||||
mtime: 0,
|
||||
chunks: vec![ChunkRef {
|
||||
hash: chunk_hash,
|
||||
offset: 0,
|
||||
size: file_data.len() as u32,
|
||||
}],
|
||||
});
|
||||
|
||||
let result = reader
|
||||
.read(FileId(1), 0, file_data.len() as u32)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(&result[..], file_data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let data = b"persistent data";
|
||||
let hash = {
|
||||
let store = CasStore::open(config.clone()).await.unwrap();
|
||||
store.put(data).await.unwrap()
|
||||
};
|
||||
|
||||
let store = CasStore::open(config).await.unwrap();
|
||||
let retrieved = store.get(&hash).await.unwrap();
|
||||
assert_eq!(&retrieved[..], data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deduplication() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let config = CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = CasStore::open(config).await.unwrap();
|
||||
|
||||
let data = b"duplicate this content";
|
||||
|
||||
let hash1 = store.put(data).await.unwrap();
|
||||
let size_after_first = store.current_size();
|
||||
|
||||
let hash2 = store.put(data).await.unwrap();
|
||||
let size_after_second = store.current_size();
|
||||
|
||||
assert_eq!(hash1, hash2);
|
||||
assert_eq!(size_after_first, size_after_second);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetcher_cache_miss_flow() {
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
|
||||
let test_content = b"This is audio content that will be fetched on cache miss";
|
||||
let test_file_path = origin_dir.path().join("test.flac");
|
||||
std::fs::write(&test_file_path, test_content).unwrap();
|
||||
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let origin_id = OriginId::from("test-origin");
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
origin_id.clone(),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
|
||||
let fetcher = ContentFetcher::new(store.clone());
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let file_id = FileId(42);
|
||||
let file_meta = FileMeta {
|
||||
id: file_id,
|
||||
virtual_path: VirtualPath::new("/Artist/Album/test.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id,
|
||||
path: PathBuf::from("/test.flac"),
|
||||
},
|
||||
size: test_content.len() as u64,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(file_meta);
|
||||
|
||||
let manifest = fetcher.fetch_file(file_id).await.unwrap();
|
||||
|
||||
assert_eq!(manifest.file_id, file_id);
|
||||
assert_eq!(manifest.total_size, test_content.len() as u64);
|
||||
assert_eq!(manifest.chunks.len(), 1);
|
||||
|
||||
let chunk_data = store.get(&manifest.chunks[0].hash).await.unwrap();
|
||||
assert_eq!(&chunk_data[..], test_content);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_reader_with_fetcher_integration() {
|
||||
let origin_dir = TempDir::new().unwrap();
|
||||
let cas_dir = TempDir::new().unwrap();
|
||||
|
||||
let test_content = b"Audio file content for reader integration test";
|
||||
let test_file_path = origin_dir.path().join("song.flac");
|
||||
std::fs::write(&test_file_path, test_content).unwrap();
|
||||
|
||||
let config = CasConfig {
|
||||
chunks_dir: cas_dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(CasStore::open(config).await.unwrap());
|
||||
|
||||
let origin_id = OriginId::from("local");
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
origin_id.clone(),
|
||||
origin_dir.path().to_path_buf(),
|
||||
));
|
||||
|
||||
let fetcher = ContentFetcher::new(store.clone());
|
||||
fetcher.register_origin(origin);
|
||||
|
||||
let file_id = FileId(100);
|
||||
let file_meta = FileMeta {
|
||||
id: file_id,
|
||||
virtual_path: VirtualPath::new("/Test/song.flac"),
|
||||
real_path: RealPath {
|
||||
origin_id,
|
||||
path: PathBuf::from("/song.flac"),
|
||||
},
|
||||
size: test_content.len() as u64,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
};
|
||||
fetcher.register_file(file_meta);
|
||||
|
||||
let reader = FileReader::with_fetcher(store, Arc::new(fetcher));
|
||||
|
||||
let result = reader
|
||||
.read(file_id, 0, test_content.len() as u32)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(&result[..], test_content);
|
||||
|
||||
let result2 = reader.read(file_id, 0, 10).await.unwrap();
|
||||
assert_eq!(&result2[..], &test_content[..10]);
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
[package]
|
||||
name = "musicfs-cli"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "musicfs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
musicfs-core.path = "../musicfs-core"
|
||||
musicfs-origins.path = "../musicfs-origins"
|
||||
musicfs-cache.path = "../musicfs-cache"
|
||||
musicfs-cas.path = "../musicfs-cas"
|
||||
musicfs-fuse.path = "../musicfs-fuse"
|
||||
musicfs-metadata.path = "../musicfs-metadata"
|
||||
musicfs-grpc.path = "../musicfs-grpc"
|
||||
|
||||
clap.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
tonic.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
tracing-appender.workspace = true
|
||||
anyhow.workspace = true
|
||||
dirs.workspace = true
|
||||
toml.workspace = true
|
||||
parking_lot.workspace = true
|
||||
libc.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
tracing-journald.workspace = true
|
||||
sd-notify.workspace = true
|
||||
@@ -1 +0,0 @@
|
||||
#![allow(dead_code)]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,638 +0,0 @@
|
||||
//! CLI subcommands for metadata overlay management.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Subcommand;
|
||||
use musicfs_grpc::proto::musicfs::v1::{
|
||||
metadata_service_client::MetadataServiceClient, ClearOverlayRequest, GetMetadataRequest,
|
||||
ImportMetadataRequest, UpdateMetadataRequest,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use tokio_stream::StreamExt;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Metadata overlay management subcommands.
|
||||
#[derive(Subcommand)]
|
||||
pub enum MetadataCommand {
|
||||
/// Get metadata for a file (prints as JSON)
|
||||
Get {
|
||||
/// Virtual path of the file
|
||||
path: String,
|
||||
/// Print only a specific field
|
||||
#[arg(long)]
|
||||
field: Option<String>,
|
||||
},
|
||||
/// Set metadata fields for a file
|
||||
Set {
|
||||
/// Virtual path of the file
|
||||
path: String,
|
||||
/// Track title
|
||||
#[arg(long)]
|
||||
title: Option<String>,
|
||||
/// Artist name
|
||||
#[arg(long)]
|
||||
artist: Option<String>,
|
||||
/// Album name
|
||||
#[arg(long)]
|
||||
album: Option<String>,
|
||||
/// Album artist
|
||||
#[arg(long)]
|
||||
album_artist: Option<String>,
|
||||
/// Track number
|
||||
#[arg(long)]
|
||||
track: Option<u32>,
|
||||
/// Disc number
|
||||
#[arg(long)]
|
||||
disc: Option<u32>,
|
||||
/// Genre
|
||||
#[arg(long)]
|
||||
genre: Option<String>,
|
||||
/// Date (YYYY-MM-DD or YYYY)
|
||||
#[arg(long)]
|
||||
date: Option<String>,
|
||||
/// Composer
|
||||
#[arg(long)]
|
||||
composer: Option<String>,
|
||||
/// Comment
|
||||
#[arg(long)]
|
||||
comment: Option<String>,
|
||||
/// Set metadata from JSON string
|
||||
#[arg(long, conflicts_with_all = ["title", "artist", "album", "album_artist", "track", "disc", "genre", "date", "composer", "comment"])]
|
||||
json: Option<String>,
|
||||
},
|
||||
/// Clear metadata overlay (revert to original)
|
||||
Clear {
|
||||
/// Virtual path of the file
|
||||
path: String,
|
||||
},
|
||||
/// Show difference between current and original metadata
|
||||
Diff {
|
||||
/// Virtual path of the file
|
||||
path: String,
|
||||
},
|
||||
/// Import metadata from CSV or JSON file
|
||||
Import {
|
||||
/// Import file path
|
||||
file: PathBuf,
|
||||
/// File format (csv or json, auto-detected if not specified)
|
||||
#[arg(long)]
|
||||
format: Option<String>,
|
||||
},
|
||||
/// Export metadata to file
|
||||
Export {
|
||||
/// Output file path
|
||||
#[arg(long, short)]
|
||||
output: PathBuf,
|
||||
/// Filter by search query
|
||||
#[arg(long)]
|
||||
query: Option<String>,
|
||||
/// Output format (csv or json, auto-detected from extension)
|
||||
#[arg(long)]
|
||||
format: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Metadata fields for JSON serialization.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct MetadataFields {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub file_id: Option<i64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub artist: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub album: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub album_artist: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub year: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub track: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub disc: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub genre: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub format: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub duration_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bitrate: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub track_total: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub disc_total: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub date: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub composer: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub comment: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lyrics: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub copyright: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub compilation: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub artist_sort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub album_artist_sort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub album_sort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title_sort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_recording_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_album_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_artist_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_album_artist_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mb_release_group_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub replaygain_track_gain: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub replaygain_track_peak: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub replaygain_album_gain: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub replaygain_album_peak: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub channels: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bits_per_sample: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub encoder: Option<String>,
|
||||
#[serde(skip_serializing_if = "HashMap::is_empty", default)]
|
||||
pub custom_tags: HashMap<String, String>,
|
||||
}
|
||||
|
||||
/// Execute a metadata subcommand.
|
||||
pub async fn run_metadata(command: MetadataCommand, endpoint: &str) -> Result<()> {
|
||||
match command {
|
||||
MetadataCommand::Get { path, field } => run_get(endpoint, &path, field.as_deref()).await,
|
||||
MetadataCommand::Set {
|
||||
path,
|
||||
title,
|
||||
artist,
|
||||
album,
|
||||
album_artist,
|
||||
track,
|
||||
disc,
|
||||
genre,
|
||||
date,
|
||||
composer,
|
||||
comment,
|
||||
json,
|
||||
} => {
|
||||
run_set(
|
||||
endpoint,
|
||||
&path,
|
||||
title,
|
||||
artist,
|
||||
album,
|
||||
album_artist,
|
||||
track,
|
||||
disc,
|
||||
genre,
|
||||
date,
|
||||
composer,
|
||||
comment,
|
||||
json,
|
||||
)
|
||||
.await
|
||||
}
|
||||
MetadataCommand::Clear { path } => run_clear(endpoint, &path).await,
|
||||
MetadataCommand::Diff { path } => run_diff(endpoint, &path).await,
|
||||
MetadataCommand::Import { file, format } => run_import(endpoint, &file, format).await,
|
||||
MetadataCommand::Export {
|
||||
output,
|
||||
query,
|
||||
format,
|
||||
} => run_export(endpoint, &output, query, format).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect(endpoint: &str) -> Result<MetadataServiceClient<Channel>> {
|
||||
MetadataServiceClient::connect(endpoint.to_string())
|
||||
.await
|
||||
.context("Failed to connect to gRPC server")
|
||||
}
|
||||
|
||||
async fn run_get(endpoint: &str, path: &str, field: Option<&str>) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let response = client
|
||||
.get_metadata(GetMetadataRequest {
|
||||
virtual_path: path.to_string(),
|
||||
})
|
||||
.await
|
||||
.context("GetMetadata RPC failed")?;
|
||||
|
||||
let meta = response.into_inner();
|
||||
let fields = MetadataFields {
|
||||
file_id: Some(meta.file_id),
|
||||
title: meta.title,
|
||||
artist: meta.artist,
|
||||
album: meta.album,
|
||||
album_artist: meta.album_artist,
|
||||
year: meta.year,
|
||||
track: meta.track,
|
||||
disc: meta.disc,
|
||||
genre: meta.genre,
|
||||
format: meta.format,
|
||||
duration_ms: meta.duration_ms,
|
||||
bitrate: meta.bitrate,
|
||||
track_total: meta.track_total,
|
||||
disc_total: meta.disc_total,
|
||||
date: meta.date,
|
||||
composer: meta.composer,
|
||||
comment: meta.comment,
|
||||
lyrics: meta.lyrics,
|
||||
copyright: meta.copyright,
|
||||
compilation: meta.compilation,
|
||||
artist_sort: meta.artist_sort,
|
||||
album_artist_sort: meta.album_artist_sort,
|
||||
album_sort: meta.album_sort,
|
||||
title_sort: meta.title_sort,
|
||||
mb_recording_id: meta.mb_recording_id,
|
||||
mb_album_id: meta.mb_album_id,
|
||||
mb_artist_id: meta.mb_artist_id,
|
||||
mb_album_artist_id: meta.mb_album_artist_id,
|
||||
mb_release_group_id: meta.mb_release_group_id,
|
||||
replaygain_track_gain: meta.replaygain_track_gain,
|
||||
replaygain_track_peak: meta.replaygain_track_peak,
|
||||
replaygain_album_gain: meta.replaygain_album_gain,
|
||||
replaygain_album_peak: meta.replaygain_album_peak,
|
||||
channels: meta.channels,
|
||||
bits_per_sample: meta.bits_per_sample,
|
||||
encoder: meta.encoder,
|
||||
custom_tags: meta.custom_tags,
|
||||
};
|
||||
|
||||
if let Some(field_name) = field {
|
||||
let value = get_field_value(&fields, field_name)?;
|
||||
println!("{}", value);
|
||||
} else {
|
||||
let json = serde_json::to_string_pretty(&fields)?;
|
||||
println!("{}", json);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_field_value(fields: &MetadataFields, field_name: &str) -> Result<String> {
|
||||
let value = match field_name {
|
||||
"file_id" => fields.file_id.map(|v| v.to_string()),
|
||||
"title" => fields.title.clone(),
|
||||
"artist" => fields.artist.clone(),
|
||||
"album" => fields.album.clone(),
|
||||
"album_artist" => fields.album_artist.clone(),
|
||||
"year" => fields.year.map(|v| v.to_string()),
|
||||
"track" => fields.track.map(|v| v.to_string()),
|
||||
"disc" => fields.disc.map(|v| v.to_string()),
|
||||
"genre" => fields.genre.clone(),
|
||||
"format" => fields.format.clone(),
|
||||
"duration_ms" => fields.duration_ms.map(|v| v.to_string()),
|
||||
"bitrate" => fields.bitrate.map(|v| v.to_string()),
|
||||
"track_total" => fields.track_total.map(|v| v.to_string()),
|
||||
"disc_total" => fields.disc_total.map(|v| v.to_string()),
|
||||
"date" => fields.date.clone(),
|
||||
"composer" => fields.composer.clone(),
|
||||
"comment" => fields.comment.clone(),
|
||||
"lyrics" => fields.lyrics.clone(),
|
||||
"copyright" => fields.copyright.clone(),
|
||||
"compilation" => fields.compilation.map(|v| v.to_string()),
|
||||
"artist_sort" => fields.artist_sort.clone(),
|
||||
"album_artist_sort" => fields.album_artist_sort.clone(),
|
||||
"album_sort" => fields.album_sort.clone(),
|
||||
"title_sort" => fields.title_sort.clone(),
|
||||
"mb_recording_id" => fields.mb_recording_id.clone(),
|
||||
"mb_album_id" => fields.mb_album_id.clone(),
|
||||
"mb_artist_id" => fields.mb_artist_id.clone(),
|
||||
"mb_album_artist_id" => fields.mb_album_artist_id.clone(),
|
||||
"mb_release_group_id" => fields.mb_release_group_id.clone(),
|
||||
"replaygain_track_gain" => fields.replaygain_track_gain.map(|v| v.to_string()),
|
||||
"replaygain_track_peak" => fields.replaygain_track_peak.map(|v| v.to_string()),
|
||||
"replaygain_album_gain" => fields.replaygain_album_gain.map(|v| v.to_string()),
|
||||
"replaygain_album_peak" => fields.replaygain_album_peak.map(|v| v.to_string()),
|
||||
"channels" => fields.channels.map(|v| v.to_string()),
|
||||
"bits_per_sample" => fields.bits_per_sample.map(|v| v.to_string()),
|
||||
"encoder" => fields.encoder.clone(),
|
||||
_ => return Err(anyhow::anyhow!("Unknown field: {}", field_name)),
|
||||
};
|
||||
|
||||
Ok(value.unwrap_or_else(|| "null".to_string()))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_set(
|
||||
endpoint: &str,
|
||||
path: &str,
|
||||
title: Option<String>,
|
||||
artist: Option<String>,
|
||||
album: Option<String>,
|
||||
album_artist: Option<String>,
|
||||
track: Option<u32>,
|
||||
disc: Option<u32>,
|
||||
genre: Option<String>,
|
||||
date: Option<String>,
|
||||
composer: Option<String>,
|
||||
comment: Option<String>,
|
||||
json: Option<String>,
|
||||
) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let get_response = client
|
||||
.get_metadata(GetMetadataRequest {
|
||||
virtual_path: path.to_string(),
|
||||
})
|
||||
.await
|
||||
.context("Failed to get file metadata")?;
|
||||
|
||||
let file_id = get_response.into_inner().file_id;
|
||||
|
||||
let request = if let Some(json_str) = json {
|
||||
let fields: MetadataFields =
|
||||
serde_json::from_str(&json_str).context("Failed to parse JSON metadata")?;
|
||||
UpdateMetadataRequest {
|
||||
file_id,
|
||||
title: fields.title,
|
||||
artist: fields.artist,
|
||||
album: fields.album,
|
||||
album_artist: fields.album_artist,
|
||||
track_number: fields.track,
|
||||
disc_number: fields.disc,
|
||||
genre: fields.genre,
|
||||
date: fields.date,
|
||||
composer: fields.composer,
|
||||
comment: fields.comment,
|
||||
lyrics: fields.lyrics,
|
||||
copyright: fields.copyright,
|
||||
compilation: fields.compilation,
|
||||
artist_sort: fields.artist_sort,
|
||||
album_artist_sort: fields.album_artist_sort,
|
||||
album_sort: fields.album_sort,
|
||||
title_sort: fields.title_sort,
|
||||
mb_recording_id: fields.mb_recording_id,
|
||||
mb_album_id: fields.mb_album_id,
|
||||
mb_artist_id: fields.mb_artist_id,
|
||||
replaygain_track_gain: fields.replaygain_track_gain,
|
||||
replaygain_track_peak: fields.replaygain_track_peak,
|
||||
replaygain_album_gain: fields.replaygain_album_gain,
|
||||
replaygain_album_peak: fields.replaygain_album_peak,
|
||||
label: None,
|
||||
album_type: None,
|
||||
cover_url: None,
|
||||
custom_tags: fields.custom_tags,
|
||||
}
|
||||
} else {
|
||||
UpdateMetadataRequest {
|
||||
file_id,
|
||||
title,
|
||||
artist,
|
||||
album,
|
||||
album_artist,
|
||||
track_number: track,
|
||||
disc_number: disc,
|
||||
genre,
|
||||
date,
|
||||
composer,
|
||||
comment,
|
||||
lyrics: None,
|
||||
copyright: None,
|
||||
compilation: None,
|
||||
artist_sort: None,
|
||||
album_artist_sort: None,
|
||||
album_sort: None,
|
||||
title_sort: None,
|
||||
mb_recording_id: None,
|
||||
mb_album_id: None,
|
||||
mb_artist_id: None,
|
||||
replaygain_track_gain: None,
|
||||
replaygain_track_peak: None,
|
||||
replaygain_album_gain: None,
|
||||
replaygain_album_peak: None,
|
||||
label: None,
|
||||
album_type: None,
|
||||
cover_url: None,
|
||||
custom_tags: HashMap::new(),
|
||||
}
|
||||
};
|
||||
|
||||
let response = client
|
||||
.update_metadata(request)
|
||||
.await
|
||||
.context("UpdateMetadata RPC failed")?;
|
||||
|
||||
let result = response.into_inner();
|
||||
if result.success {
|
||||
info!(file_id = result.file_id, "Metadata updated successfully");
|
||||
println!("Metadata updated for file_id={}", result.file_id);
|
||||
} else {
|
||||
let msg = result
|
||||
.error_message
|
||||
.unwrap_or_else(|| "Unknown error".to_string());
|
||||
anyhow::bail!("Failed to update metadata: {}", msg);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_clear(endpoint: &str, path: &str) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let get_response = client
|
||||
.get_metadata(GetMetadataRequest {
|
||||
virtual_path: path.to_string(),
|
||||
})
|
||||
.await
|
||||
.context("Failed to get file metadata")?;
|
||||
|
||||
let file_id = get_response.into_inner().file_id;
|
||||
|
||||
let response = client
|
||||
.clear_overlay(ClearOverlayRequest { file_id })
|
||||
.await
|
||||
.context("ClearOverlay RPC failed")?;
|
||||
|
||||
let result = response.into_inner();
|
||||
if result.success {
|
||||
info!(file_id = result.file_id, "Overlay cleared successfully");
|
||||
println!("Metadata overlay cleared for file_id={}", result.file_id);
|
||||
} else {
|
||||
let msg = result
|
||||
.error_message
|
||||
.unwrap_or_else(|| "Unknown error".to_string());
|
||||
anyhow::bail!("Failed to clear overlay: {}", msg);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_diff(endpoint: &str, path: &str) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let response = client
|
||||
.get_metadata(GetMetadataRequest {
|
||||
virtual_path: path.to_string(),
|
||||
})
|
||||
.await
|
||||
.context("GetMetadata RPC failed")?;
|
||||
|
||||
let meta = response.into_inner();
|
||||
debug!(file_id = meta.file_id, "Retrieved metadata for diff");
|
||||
|
||||
println!("Current metadata for: {}", path);
|
||||
println!("---");
|
||||
|
||||
let fields = MetadataFields {
|
||||
file_id: Some(meta.file_id),
|
||||
title: meta.title,
|
||||
artist: meta.artist,
|
||||
album: meta.album,
|
||||
album_artist: meta.album_artist,
|
||||
year: meta.year,
|
||||
track: meta.track,
|
||||
disc: meta.disc,
|
||||
genre: meta.genre,
|
||||
format: meta.format,
|
||||
duration_ms: meta.duration_ms,
|
||||
bitrate: meta.bitrate,
|
||||
track_total: meta.track_total,
|
||||
disc_total: meta.disc_total,
|
||||
date: meta.date,
|
||||
composer: meta.composer,
|
||||
comment: meta.comment,
|
||||
lyrics: meta.lyrics,
|
||||
copyright: meta.copyright,
|
||||
compilation: meta.compilation,
|
||||
artist_sort: meta.artist_sort,
|
||||
album_artist_sort: meta.album_artist_sort,
|
||||
album_sort: meta.album_sort,
|
||||
title_sort: meta.title_sort,
|
||||
mb_recording_id: meta.mb_recording_id,
|
||||
mb_album_id: meta.mb_album_id,
|
||||
mb_artist_id: meta.mb_artist_id,
|
||||
mb_album_artist_id: meta.mb_album_artist_id,
|
||||
mb_release_group_id: meta.mb_release_group_id,
|
||||
replaygain_track_gain: meta.replaygain_track_gain,
|
||||
replaygain_track_peak: meta.replaygain_track_peak,
|
||||
replaygain_album_gain: meta.replaygain_album_gain,
|
||||
replaygain_album_peak: meta.replaygain_album_peak,
|
||||
channels: meta.channels,
|
||||
bits_per_sample: meta.bits_per_sample,
|
||||
encoder: meta.encoder,
|
||||
custom_tags: meta.custom_tags,
|
||||
};
|
||||
|
||||
let json = serde_json::to_string_pretty(&fields)?;
|
||||
println!("{}", json);
|
||||
println!("---");
|
||||
println!("Note: Original metadata comparison requires re-parsing the source file.");
|
||||
println!("Use 'musicfs metadata clear <path>' to revert to original metadata.");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_import(endpoint: &str, file: &PathBuf, format: Option<String>) -> Result<()> {
|
||||
let mut client = connect(endpoint).await?;
|
||||
|
||||
let file_format = format.or_else(|| {
|
||||
file.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|s| s.to_lowercase())
|
||||
});
|
||||
|
||||
let source_path = file
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| file.clone())
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
|
||||
info!(source_path = %source_path, format = ?file_format, "Starting metadata import");
|
||||
|
||||
let response = client
|
||||
.import_metadata(ImportMetadataRequest {
|
||||
source_path,
|
||||
format: file_format,
|
||||
})
|
||||
.await
|
||||
.context("ImportMetadata RPC failed")?;
|
||||
|
||||
let mut stream = response.into_inner();
|
||||
let mut last_imported = 0u32;
|
||||
let mut last_total = 0u32;
|
||||
let mut errors = Vec::new();
|
||||
|
||||
while let Some(progress) = stream.next().await {
|
||||
let progress = progress.context("Stream error")?;
|
||||
last_imported = progress.imported;
|
||||
last_total = progress.total;
|
||||
|
||||
if let Some(ref err) = progress.error_message {
|
||||
let file = progress.current_file.as_deref().unwrap_or("unknown");
|
||||
errors.push(format!("{}: {}", file, err));
|
||||
}
|
||||
|
||||
if let Some(ref current) = progress.current_file {
|
||||
print!(
|
||||
"\rImporting: {}/{} - {}",
|
||||
progress.imported, progress.total, current
|
||||
);
|
||||
std::io::Write::flush(&mut std::io::stdout())?;
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
println!(
|
||||
"Import complete: {}/{} files imported",
|
||||
last_imported, last_total
|
||||
);
|
||||
|
||||
if !errors.is_empty() {
|
||||
println!("\nErrors ({}):", errors.len());
|
||||
for err in errors.iter().take(10) {
|
||||
println!(" - {}", err);
|
||||
}
|
||||
if errors.len() > 10 {
|
||||
println!(" ... and {} more", errors.len() - 10);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_export(
|
||||
_endpoint: &str,
|
||||
output: &PathBuf,
|
||||
query: Option<String>,
|
||||
format: Option<String>,
|
||||
) -> Result<()> {
|
||||
let output_format = format.or_else(|| {
|
||||
output
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|s| s.to_lowercase())
|
||||
});
|
||||
|
||||
println!("Export metadata to: {}", output.display());
|
||||
if let Some(ref q) = query {
|
||||
println!("Filter query: {}", q);
|
||||
}
|
||||
println!("Format: {}", output_format.as_deref().unwrap_or("json"));
|
||||
println!();
|
||||
println!("Note: Export requires file listing capability.");
|
||||
println!("This feature requires integration with the Search service.");
|
||||
println!(
|
||||
"Use 'musicfs search <query>' to find files, then 'musicfs metadata get <path>' for each."
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
[package]
|
||||
name = "musicfs-client"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
default-run = "musicfs"
|
||||
|
||||
[lib]
|
||||
name = "musicfs"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "musicfs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
musicfs-core.workspace = true
|
||||
musicfs-proto.workspace = true
|
||||
fuser.workspace = true
|
||||
libc.workspace = true
|
||||
sea-orm.workspace = true
|
||||
tokio.workspace = true
|
||||
anyhow.workspace = true
|
||||
async-trait.workspace = true
|
||||
clap.workspace = true
|
||||
notify.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
tonic.workspace = true
|
||||
tonic-health.workspace = true
|
||||
tonic-reflection.workspace = true
|
||||
bytes.workspace = true
|
||||
http.workspace = true
|
||||
tracing.workspace = true
|
||||
log.workspace = true
|
||||
twox-hash.workspace = true
|
||||
chrono.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
sea-orm = { workspace = true, features = ["mock"] }
|
||||
@@ -0,0 +1,248 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use musicfs_core::music::encoder::MusicMetadataEncoderFactory;
|
||||
use musicfs_proto::{
|
||||
ClientControl, FileEntry, FileMetadata, GetMusicMetadataRequest, GetMusicMetadataResponse,
|
||||
ListFilesRequest, ListFilesResponse, MusicMetadata as ProtoMusicMetadata, PictureDataRange,
|
||||
UpdateMusicMetadataRequest, UpdateMusicMetadataResponse,
|
||||
};
|
||||
use sea_orm::{ActiveModelTrait, DatabaseConnection, EntityTrait};
|
||||
use tonic::{Request, Response, Status};
|
||||
|
||||
use crate::db::entities;
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::music::db::save_music_metadata;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::virtual_dirs::{
|
||||
compute_new_layout, ensure_virtual_dirs, find_orphaned_dirs, parent_inode_from_path,
|
||||
};
|
||||
|
||||
pub struct ClientControlServiceImpl {
|
||||
files: Arc<Mutex<BTreeMap<INodeNo, Item>>>,
|
||||
db: DatabaseConnection,
|
||||
}
|
||||
|
||||
impl ClientControlServiceImpl {
|
||||
pub fn new(files: Arc<Mutex<BTreeMap<INodeNo, Item>>>, db: DatabaseConnection) -> Self {
|
||||
ClientControlServiceImpl { files, db }
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl ClientControl for ClientControlServiceImpl {
|
||||
async fn get_music_metadata(
|
||||
&self,
|
||||
request: Request<GetMusicMetadataRequest>,
|
||||
) -> Result<Response<GetMusicMetadataResponse>, Status> {
|
||||
let inode = INodeNo(request.into_inner().inode);
|
||||
let metadata = {
|
||||
let files = self.files.lock().unwrap();
|
||||
let item = files
|
||||
.get(&inode)
|
||||
.ok_or_else(|| Status::not_found(format!("inode {} not found", inode.0)))?;
|
||||
item.music_metadata.clone().ok_or_else(|| {
|
||||
Status::not_found(format!("inode {} has no music metadata", inode.0))
|
||||
})?
|
||||
};
|
||||
Ok(Response::new(GetMusicMetadataResponse {
|
||||
metadata: Some(music_metadata_to_proto(metadata)),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn update_music_metadata(
|
||||
&self,
|
||||
request: Request<UpdateMusicMetadataRequest>,
|
||||
) -> Result<Response<UpdateMusicMetadataResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let inode = INodeNo(req.inode);
|
||||
|
||||
let outcome = {
|
||||
let mut files = self.files.lock().unwrap();
|
||||
|
||||
let (old_local_path, current_name, source_for_virtual_dirs) = {
|
||||
let root = files.get(&INodeNo::ROOT);
|
||||
let item = files
|
||||
.get(&inode)
|
||||
.ok_or_else(|| Status::not_found(format!("inode {} not found", inode.0)))?;
|
||||
let source = root
|
||||
.map(|r| r.original_path.clone())
|
||||
.unwrap_or_else(|| PathBuf::from("/"));
|
||||
(item.local_path.clone(), item.name.clone(), source)
|
||||
};
|
||||
|
||||
// Phase 1: mutate the music metadata and re-encode the header.
|
||||
// Done in its own scope so the mutable borrow on `files` is
|
||||
// released before we touch `files` again for layout changes.
|
||||
let (updated_mm, _encoder_extension_source) = {
|
||||
let item = files
|
||||
.get_mut(&inode)
|
||||
.ok_or_else(|| Status::not_found(format!("inode {} not found", inode.0)))?;
|
||||
let mm = item.music_metadata.as_mut().ok_or_else(|| {
|
||||
Status::not_found(format!("inode {} has no music metadata", inode.0))
|
||||
})?;
|
||||
apply_update(mm, req);
|
||||
let encoder =
|
||||
MusicMetadataEncoderFactory::for_path(&item.local_path).ok_or_else(|| {
|
||||
Status::failed_precondition(format!(
|
||||
"no encoder for file type: {}",
|
||||
item.local_path.display()
|
||||
))
|
||||
})?;
|
||||
encoder.encode(mm);
|
||||
(mm.clone(), item.local_path.clone())
|
||||
};
|
||||
|
||||
// Phase 2: compute new layout, write it onto the item, then
|
||||
// ensure new virtual dirs and prune orphans. A fresh mutable
|
||||
// borrow is fine because phase 1's borrow has been released.
|
||||
let (new_name, new_local_path) = compute_new_layout(¤t_name, &updated_mm);
|
||||
{
|
||||
let item = files
|
||||
.get_mut(&inode)
|
||||
.ok_or_else(|| Status::not_found(format!("inode {} not found", inode.0)))?;
|
||||
item.name = new_name.clone();
|
||||
item.local_path = new_local_path.clone();
|
||||
item.parent_inode = parent_inode_from_path(&new_local_path);
|
||||
}
|
||||
ensure_virtual_dirs(&new_local_path, &source_for_virtual_dirs, &mut files);
|
||||
|
||||
let orphaned = find_orphaned_dirs(&files, inode, &old_local_path);
|
||||
for dir_inode in &orphaned {
|
||||
files.remove(dir_inode);
|
||||
}
|
||||
|
||||
UpdateOutcome {
|
||||
metadata: updated_mm,
|
||||
new_name,
|
||||
new_local_path,
|
||||
orphaned_dirs: orphaned,
|
||||
}
|
||||
};
|
||||
|
||||
save_music_metadata(inode.0 as i64, &outcome.metadata, &self.db)
|
||||
.await
|
||||
.map_err(|e| Status::internal(format!("DB save failed: {e}")))?;
|
||||
|
||||
persist_layout_changes(
|
||||
&self.db,
|
||||
inode.0 as i64,
|
||||
&outcome.new_name,
|
||||
&outcome.new_local_path,
|
||||
&outcome.orphaned_dirs,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Status::internal(format!("DB layout save failed: {e}")))?;
|
||||
|
||||
tracing::debug!(
|
||||
ino = inode.0,
|
||||
orphaned_dirs = outcome.orphaned_dirs.len(),
|
||||
"control: music metadata updated, layout recomputed"
|
||||
);
|
||||
|
||||
Ok(Response::new(UpdateMusicMetadataResponse {
|
||||
metadata: Some(music_metadata_to_proto(outcome.metadata)),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn list_files(
|
||||
&self,
|
||||
_request: Request<ListFilesRequest>,
|
||||
) -> Result<Response<ListFilesResponse>, Status> {
|
||||
let entries: Vec<FileEntry> = {
|
||||
let files = self.files.lock().unwrap();
|
||||
files
|
||||
.values()
|
||||
.filter(|item| item.file_type == FileType::File)
|
||||
.map(|item| FileEntry {
|
||||
inode: item.inode.0,
|
||||
original_path: item.original_path.to_string_lossy().into_owned(),
|
||||
local_path: item.local_path.to_string_lossy().into_owned(),
|
||||
name: item.name.clone(),
|
||||
metadata: item
|
||||
.music_metadata
|
||||
.as_ref()
|
||||
.map(|mm| music_metadata_to_file_metadata(mm)),
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
Ok(Response::new(ListFilesResponse { files: entries }))
|
||||
}
|
||||
}
|
||||
|
||||
fn music_metadata_to_file_metadata(mm: &MusicMetadata) -> FileMetadata {
|
||||
FileMetadata {
|
||||
artist: mm.artist.clone(),
|
||||
album_artist: mm.album_artist.clone(),
|
||||
album: mm.album.clone(),
|
||||
track_number: mm.track_number,
|
||||
track_title: mm.track_title.clone(),
|
||||
other_tags: mm.other_tags.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn music_metadata_to_proto(mm: crate::music::metadata::MusicMetadata) -> ProtoMusicMetadata {
|
||||
ProtoMusicMetadata {
|
||||
artist: mm.artist,
|
||||
album_artist: mm.album_artist,
|
||||
album: mm.album,
|
||||
track_number: mm.track_number,
|
||||
track_title: mm.track_title,
|
||||
other_tags: mm.other_tags,
|
||||
header: mm.header,
|
||||
picture_block_headers: mm.picture_block_headers,
|
||||
picture_data_ranges: mm
|
||||
.picture_data_ranges
|
||||
.into_iter()
|
||||
.map(|(offset, length)| PictureDataRange { offset, length })
|
||||
.collect(),
|
||||
real_audio_start: mm.real_audio_start,
|
||||
vorbis_comment_offset: mm.vorbis_comment_offset,
|
||||
vorbis_comment_length: mm.vorbis_comment_length,
|
||||
}
|
||||
}
|
||||
|
||||
struct UpdateOutcome {
|
||||
metadata: MusicMetadata,
|
||||
new_name: String,
|
||||
new_local_path: PathBuf,
|
||||
orphaned_dirs: Vec<INodeNo>,
|
||||
}
|
||||
|
||||
fn apply_update(mm: &mut MusicMetadata, req: UpdateMusicMetadataRequest) {
|
||||
mm.artist = req.artist;
|
||||
mm.album_artist = req.album_artist;
|
||||
mm.album = req.album;
|
||||
mm.track_number = req.track_number;
|
||||
mm.track_title = req.track_title;
|
||||
mm.other_tags = req.other_tags;
|
||||
}
|
||||
|
||||
async fn persist_layout_changes(
|
||||
db: &DatabaseConnection,
|
||||
inode: i64,
|
||||
new_name: &str,
|
||||
new_local_path: &std::path::Path,
|
||||
orphaned_dirs: &[INodeNo],
|
||||
) -> Result<(), sea_orm::DbErr> {
|
||||
use sea_orm::ActiveValue::Set;
|
||||
let new_local_path_str = new_local_path.to_string_lossy().into_owned();
|
||||
let update = entities::ActiveModel {
|
||||
inode: Set(inode),
|
||||
name: Set(new_name.to_string()),
|
||||
local_path: Set(new_local_path_str),
|
||||
..Default::default()
|
||||
};
|
||||
update.update(db).await?;
|
||||
|
||||
for dir_inode in orphaned_dirs {
|
||||
let _ = entities::Entity::delete_by_id(dir_inode.0 as i64)
|
||||
.exec(db)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
use sea_orm::{
|
||||
ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter,
|
||||
sea_query::OnConflict,
|
||||
};
|
||||
use tracing::{debug, error};
|
||||
|
||||
use crate::db::entities::cached_file_bytes::{ActiveModel, Column, Entity, Model};
|
||||
|
||||
/// Return the cached bytes for `inode` if present. `None` means "not cached";
|
||||
/// the caller should fetch from the server and call [`put_cached_bytes`].
|
||||
pub async fn get_cached_bytes(inode: i64, db: &DatabaseConnection) -> Option<Vec<u8>> {
|
||||
let row: Option<Model> = match Entity::find_by_id(inode).one(db).await {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
error!(%inode, error = %e, "get_cached_bytes: DB read failed");
|
||||
None
|
||||
}
|
||||
};
|
||||
return row.map(|m| m.data);
|
||||
}
|
||||
|
||||
/// Insert or replace the cached bytes for `inode`. Called after a successful
|
||||
/// `GetFile` round-trip so subsequent reads of the same range hit the cache.
|
||||
pub async fn put_cached_bytes(inode: i64, data: Vec<u8>, db: &DatabaseConnection) {
|
||||
let active = ActiveModel {
|
||||
inode: Set(inode),
|
||||
data: Set(data),
|
||||
fetched_at: Set(chrono::Utc::now()),
|
||||
};
|
||||
if let Err(e) = Entity::insert(active)
|
||||
.on_conflict(
|
||||
OnConflict::column(Column::Inode)
|
||||
.update_columns([Column::Data, Column::FetchedAt])
|
||||
.to_owned(),
|
||||
)
|
||||
.exec(db)
|
||||
.await
|
||||
{
|
||||
error!(%inode, error = %e, "put_cached_bytes: DB upsert failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove cache rows for the given inodes. Called by reconcile when a file's
|
||||
/// hash has changed (the cached bytes are now stale).
|
||||
pub async fn delete_cached_bytes_for(inodes: &[i64], db: &DatabaseConnection) {
|
||||
if inodes.is_empty() {
|
||||
return;
|
||||
}
|
||||
debug!(count = inodes.len(), "deleting cached bytes");
|
||||
if let Err(e) = Entity::delete_many()
|
||||
.filter(Column::Inode.is_in(inodes.to_vec()))
|
||||
.exec(db)
|
||||
.await
|
||||
{
|
||||
error!(error = %e, "delete_cached_bytes_for: DB delete failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use sea_orm::ConnectionTrait;
|
||||
|
||||
async fn connect() -> Option<DatabaseConnection> {
|
||||
let pg_host = std::env::var("PGHOST").ok()?;
|
||||
let url = format!("postgresql://fujin@localhost/musicfs?host={pg_host}");
|
||||
let db = sea_orm::Database::connect(&url).await.ok()?;
|
||||
db.execute_unprepared("SELECT 1 FROM cached_file_bytes LIMIT 0")
|
||||
.await
|
||||
.ok()?;
|
||||
Some(db)
|
||||
}
|
||||
|
||||
async fn setup_parent_item(db: &DatabaseConnection, inode: i64) {
|
||||
db.execute_unprepared(&format!(
|
||||
"INSERT INTO items (inode, name, original_path, local_path, file_type, hash) \
|
||||
VALUES ({inode}, 'test', '/test', '/test', 'file', 0) \
|
||||
ON CONFLICT (inode) DO NOTHING"
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn teardown(db: &DatabaseConnection, inode: i64) {
|
||||
let _ = db
|
||||
.execute_unprepared(&format!("DELETE FROM items WHERE inode = {inode}"))
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_round_trip_with_timestamptz() {
|
||||
let db = match connect().await {
|
||||
Some(db) => db,
|
||||
None => {
|
||||
eprintln!("skip: no database (PGHOST unset or unreachable)");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let inode = -999_999_i64;
|
||||
let test_data = b"regression-test-payload".to_vec();
|
||||
|
||||
setup_parent_item(&db, inode).await;
|
||||
put_cached_bytes(inode, test_data.clone(), &db).await;
|
||||
|
||||
let cached = get_cached_bytes(inode, &db).await;
|
||||
assert_eq!(
|
||||
cached,
|
||||
Some(test_data),
|
||||
"get_cached_bytes must decode fetched_at (TIMESTAMPTZ), not fail with type mismatch"
|
||||
);
|
||||
|
||||
teardown(&db, inode).await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
use crate::item::{FileType, Item};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "items")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
pub name: String,
|
||||
pub original_path: String,
|
||||
pub local_path: String,
|
||||
pub file_type: String,
|
||||
pub hash: i64,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
|
||||
impl From<&Item> for ActiveModel {
|
||||
fn from(item: &Item) -> Self {
|
||||
use sea_orm::ActiveValue::Set;
|
||||
ActiveModel {
|
||||
inode: Set(item.inode.0 as i64),
|
||||
name: Set(item.name.clone()),
|
||||
original_path: Set(item.original_path.to_string_lossy().into_owned()),
|
||||
local_path: Set(item.local_path.to_string_lossy().into_owned()),
|
||||
file_type: Set(match item.file_type {
|
||||
FileType::Directory => "directory".to_string(),
|
||||
FileType::File => "file".to_string(),
|
||||
}),
|
||||
hash: Set(item.hash as i64),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub mod cached_file_bytes {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "cached_file_bytes")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
#[sea_orm(column_type = "Blob")]
|
||||
pub data: Vec<u8>,
|
||||
pub fetched_at: DateTimeUtc,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
use fuser::INodeNo;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn attrs_for_tempdir() -> FileAttrs {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
return FileAttrs::from(&metadata);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_to_active_model_fields_match() {
|
||||
let item = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/original/path"),
|
||||
PathBuf::from("local/path"),
|
||||
FileType::Directory,
|
||||
attrs_for_tempdir(),
|
||||
None,
|
||||
);
|
||||
let am = ActiveModel::from(&item);
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.inode {
|
||||
assert_eq!(*v, 42i64);
|
||||
} else {
|
||||
panic!("inode field not set");
|
||||
}
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.name {
|
||||
assert_eq!(v, "test");
|
||||
} else {
|
||||
panic!("name field not set");
|
||||
}
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.original_path {
|
||||
assert_eq!(v, "/original/path");
|
||||
} else {
|
||||
panic!("original_path field not set");
|
||||
}
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.local_path {
|
||||
assert_eq!(v, "local/path");
|
||||
} else {
|
||||
panic!("local_path field not set");
|
||||
}
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am.hash {
|
||||
assert_eq!(*v, item.hash as i64);
|
||||
} else {
|
||||
panic!("hash field not set");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_type_string_mapping() {
|
||||
let item_dir = Item::new(
|
||||
INodeNo(1),
|
||||
INodeNo::ROOT,
|
||||
"dir".to_string(),
|
||||
PathBuf::from("/original/dir"),
|
||||
PathBuf::from("local/dir"),
|
||||
FileType::Directory,
|
||||
attrs_for_tempdir(),
|
||||
None,
|
||||
);
|
||||
let am_dir = ActiveModel::from(&item_dir);
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am_dir.file_type {
|
||||
assert_eq!(v, "directory");
|
||||
} else {
|
||||
panic!("file_type field not set for directory");
|
||||
}
|
||||
|
||||
let item_file = Item::new(
|
||||
INodeNo(2),
|
||||
INodeNo::ROOT,
|
||||
"file".to_string(),
|
||||
PathBuf::from("/original/file"),
|
||||
PathBuf::from("local/file"),
|
||||
FileType::File,
|
||||
attrs_for_tempdir(),
|
||||
None,
|
||||
);
|
||||
let am_file = ActiveModel::from(&item_file);
|
||||
|
||||
if let sea_orm::ActiveValue::Set(v) = &am_file.file_type {
|
||||
assert_eq!(v, "file");
|
||||
} else {
|
||||
panic!("file_type field not set for file");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod cache;
|
||||
pub mod entities;
|
||||
pub mod sync;
|
||||
@@ -0,0 +1,92 @@
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use tracing::{debug, error};
|
||||
|
||||
use crate::db::entities::{ActiveModel, Entity, Model};
|
||||
use crate::item::Item;
|
||||
use crate::music::db::save_music_metadata;
|
||||
|
||||
pub async fn sync_items_to_db(
|
||||
snapshot: &BTreeMap<INodeNo, Item>,
|
||||
db_items: &HashMap<i64, Model>,
|
||||
client: &sea_orm::DatabaseConnection,
|
||||
) {
|
||||
let mut to_insert: Vec<ActiveModel> = vec![];
|
||||
let mut to_update: Vec<ActiveModel> = vec![];
|
||||
let mut to_delete: Vec<i64> = vec![];
|
||||
let mut to_save_music: Vec<i64> = vec![];
|
||||
|
||||
for (ino, item) in snapshot {
|
||||
let ino_i64 = ino.0 as i64;
|
||||
match db_items.get(&ino_i64) {
|
||||
None => {
|
||||
to_insert.push(ActiveModel::from(item));
|
||||
if item.music_metadata.is_some() {
|
||||
to_save_music.push(ino_i64);
|
||||
}
|
||||
}
|
||||
Some(db_item) if db_item.hash != item.hash as i64 => {
|
||||
to_update.push(ActiveModel::from(item));
|
||||
if item.music_metadata.is_some() {
|
||||
to_save_music.push(ino_i64);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let fresh_inodes: HashSet<i64> = snapshot.keys().map(|i| i.0 as i64).collect();
|
||||
for ino in db_items.keys().filter(|i| !fresh_inodes.contains(i)) {
|
||||
to_delete.push(*ino);
|
||||
}
|
||||
|
||||
debug!(
|
||||
insert = to_insert.len(),
|
||||
update = to_update.len(),
|
||||
delete = to_delete.len(),
|
||||
save_music = to_save_music.len(),
|
||||
"sync_items_to_db"
|
||||
);
|
||||
|
||||
if !to_insert.is_empty() {
|
||||
if let Err(e) = Entity::insert_many(to_insert).exec(client).await {
|
||||
error!(error = %e, "sync: insert_many failed");
|
||||
}
|
||||
}
|
||||
for model in to_update {
|
||||
if let Err(e) = model.update(client).await {
|
||||
error!(error = %e, "sync: update failed");
|
||||
}
|
||||
}
|
||||
for ino in to_delete {
|
||||
if let Err(e) = Entity::delete_by_id(ino).exec(client).await {
|
||||
error!(%ino, error = %e, "sync: delete failed");
|
||||
}
|
||||
}
|
||||
|
||||
for ino_i64 in &to_save_music {
|
||||
let ino = INodeNo(*ino_i64 as u64);
|
||||
if let Some(music_metadata) = snapshot
|
||||
.get(&ino)
|
||||
.and_then(|item| item.music_metadata.as_ref())
|
||||
{
|
||||
if let Err(e) = save_music_metadata(*ino_i64, music_metadata, client).await {
|
||||
error!(ino = *ino_i64, error = %e, "sync: save_music_metadata failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run_db_blocking<F: std::future::Future>(future: F) -> F::Output {
|
||||
return tokio::runtime::Builder::new_current_thread()
|
||||
.enable_io()
|
||||
.enable_time()
|
||||
.build()
|
||||
.unwrap_or_else(|e| {
|
||||
error!(error = %e, "run_db_blocking: failed to build runtime");
|
||||
panic!("run_db_blocking: failed to build runtime: {e}");
|
||||
})
|
||||
.block_on(future);
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
net::SocketAddr,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use musicfs_proto::{
|
||||
ClientControlServer, ClientStatus, ClientStatusResponse, ClientStatusServer,
|
||||
GetClientStatusRequest,
|
||||
};
|
||||
use tokio::time::Duration;
|
||||
use tonic::{Request, Response, transport::Server};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ServerStatus {
|
||||
connected: Arc<AtomicBool>,
|
||||
last_reconcile: Arc<Mutex<Option<i64>>>,
|
||||
origin_type: String,
|
||||
mountpoint: String,
|
||||
server_endpoint: String,
|
||||
}
|
||||
|
||||
impl ServerStatus {
|
||||
pub fn new_network(mountpoint: String, server_endpoint: String) -> Self {
|
||||
ServerStatus {
|
||||
connected: Arc::new(AtomicBool::new(false)),
|
||||
last_reconcile: Arc::new(Mutex::new(None)),
|
||||
origin_type: "network".to_string(),
|
||||
mountpoint,
|
||||
server_endpoint,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_local(mountpoint: String) -> Self {
|
||||
ServerStatus {
|
||||
connected: Arc::new(AtomicBool::new(true)),
|
||||
last_reconcile: Arc::new(Mutex::new(None)),
|
||||
origin_type: "local".to_string(),
|
||||
mountpoint,
|
||||
server_endpoint: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_connected(&self, connected: bool) {
|
||||
self.connected.store(connected, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn mark_reconciled(&self) {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0);
|
||||
*self.last_reconcile.lock().unwrap() = Some(now);
|
||||
}
|
||||
|
||||
pub fn is_server_connected(&self) -> bool {
|
||||
self.connected.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn spawn_grpc_server(
|
||||
addr: SocketAddr,
|
||||
status: ServerStatus,
|
||||
files: Arc<Mutex<BTreeMap<INodeNo, crate::item::Item>>>,
|
||||
db: sea_orm::DatabaseConnection,
|
||||
) {
|
||||
let (reporter, health_service) = tonic_health::server::health_reporter();
|
||||
reporter
|
||||
.set_serving::<ClientStatusServer<ClientStatusServiceImpl>>()
|
||||
.await;
|
||||
|
||||
let reporter_clone = reporter.clone();
|
||||
let status_clone = status.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut was_connected = status_clone.is_server_connected();
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
let now_connected = status_clone.is_server_connected();
|
||||
if now_connected != was_connected {
|
||||
if now_connected {
|
||||
reporter_clone
|
||||
.set_serving::<ClientStatusServer<ClientStatusServiceImpl>>()
|
||||
.await;
|
||||
} else {
|
||||
reporter_clone
|
||||
.set_not_serving::<ClientStatusServer<ClientStatusServiceImpl>>()
|
||||
.await;
|
||||
}
|
||||
was_connected = now_connected;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(musicfs_proto::musicfs::FILE_DESCRIPTOR_SET)
|
||||
.register_encoded_file_descriptor_set(tonic_health::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("build reflection v1");
|
||||
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(musicfs_proto::musicfs::FILE_DESCRIPTOR_SET)
|
||||
.register_encoded_file_descriptor_set(tonic_health::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("build reflection v1alpha");
|
||||
|
||||
let status_svc = ClientStatusServer::new(ClientStatusServiceImpl {
|
||||
status,
|
||||
files: files.clone(),
|
||||
});
|
||||
let control_svc =
|
||||
ClientControlServer::new(crate::control::ClientControlServiceImpl::new(files, db));
|
||||
|
||||
tracing::info!(%addr, "health server listening");
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = Server::builder()
|
||||
.add_service(health_service)
|
||||
.add_service(status_svc)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
.add_service(control_svc)
|
||||
.serve(addr)
|
||||
.await
|
||||
{
|
||||
tracing::error!(error = %e, "health server ended with error");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
struct ClientStatusServiceImpl {
|
||||
status: ServerStatus,
|
||||
files: Arc<Mutex<BTreeMap<INodeNo, crate::item::Item>>>,
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl ClientStatus for ClientStatusServiceImpl {
|
||||
async fn get_client_status(
|
||||
&self,
|
||||
_request: Request<GetClientStatusRequest>,
|
||||
) -> Result<Response<ClientStatusResponse>, tonic::Status> {
|
||||
let file_count = self
|
||||
.files
|
||||
.lock()
|
||||
.unwrap()
|
||||
.values()
|
||||
.filter(|i| i.file_type == crate::item::FileType::File)
|
||||
.count() as u64;
|
||||
|
||||
let last_reconcile_unix = self.status.last_reconcile.lock().unwrap().unwrap_or(0);
|
||||
|
||||
Ok(Response::new(ClientStatusResponse {
|
||||
origin_type: self.status.origin_type.clone(),
|
||||
mountpoint: self.status.mountpoint.clone(),
|
||||
file_count,
|
||||
server_connected: self.status.is_server_connected(),
|
||||
server_endpoint: self.status.server_endpoint.clone(),
|
||||
last_reconcile_unix,
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
|
||||
use fuser::INodeNo;
|
||||
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
|
||||
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
|
||||
pub enum FileType {
|
||||
Directory,
|
||||
File,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Item {
|
||||
pub inode: INodeNo,
|
||||
pub parent_inode: INodeNo,
|
||||
pub name: String,
|
||||
pub original_path: PathBuf,
|
||||
pub local_path: PathBuf,
|
||||
pub file_type: FileType,
|
||||
pub attrs: FileAttrs,
|
||||
pub music_metadata: Option<MusicMetadata>,
|
||||
pub hash: u64,
|
||||
}
|
||||
|
||||
fn secs_since_epoch(time: SystemTime) -> u64 {
|
||||
return time
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
}
|
||||
|
||||
impl Item {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
inode: INodeNo,
|
||||
parent_inode: INodeNo,
|
||||
name: String,
|
||||
original_path: PathBuf,
|
||||
local_path: PathBuf,
|
||||
file_type: FileType,
|
||||
attrs: FileAttrs,
|
||||
music_metadata: Option<MusicMetadata>,
|
||||
) -> Item {
|
||||
let mut item = Item {
|
||||
inode,
|
||||
parent_inode,
|
||||
name,
|
||||
original_path,
|
||||
local_path,
|
||||
file_type,
|
||||
attrs,
|
||||
music_metadata,
|
||||
hash: 0,
|
||||
};
|
||||
item.hash = item.compute_hash();
|
||||
|
||||
return item;
|
||||
}
|
||||
|
||||
pub fn compute_hash(&self) -> u64 {
|
||||
musicfs_core::compute_item_hash(
|
||||
self.inode.0,
|
||||
self.original_path.as_os_str().as_bytes(),
|
||||
secs_since_epoch(self.attrs.ctime),
|
||||
secs_since_epoch(self.attrs.mtime),
|
||||
secs_since_epoch(self.attrs.crtime),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
|
||||
#[test]
|
||||
fn compute_hash_deterministic() {
|
||||
let attrs = file_attrs_for_tempdir();
|
||||
|
||||
let item1 = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/some/path"),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs.clone(),
|
||||
None,
|
||||
);
|
||||
|
||||
let item2 = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/some/path"),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(item1.hash, item2.hash);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_hash_changes_on_path_change() {
|
||||
let attrs = file_attrs_for_tempdir();
|
||||
|
||||
let item1 = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/some/path"),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs.clone(),
|
||||
None,
|
||||
);
|
||||
|
||||
let item2 = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
PathBuf::from("/different/path"),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_ne!(item1.hash, item2.hash);
|
||||
}
|
||||
|
||||
fn file_attrs_for_tempdir() -> FileAttrs {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
return FileAttrs::from(&metadata);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
pub mod control;
|
||||
pub mod db;
|
||||
pub mod health;
|
||||
pub mod item;
|
||||
pub mod music;
|
||||
pub mod origins;
|
||||
pub mod virtual_dirs;
|
||||
|
||||
pub use musicfs_core::logging;
|
||||
pub use musicfs_proto as proto;
|
||||
@@ -0,0 +1,170 @@
|
||||
use std::{collections::HashMap, net::SocketAddr, path::Path, sync::Arc};
|
||||
|
||||
use clap::Parser;
|
||||
use fuser::INodeNo;
|
||||
use musicfs::db::entities::{Entity, Model};
|
||||
use musicfs::db::sync::sync_items_to_db;
|
||||
use musicfs::health::ServerStatus;
|
||||
use musicfs::item::Item;
|
||||
use musicfs::logging::{LogConfig, init};
|
||||
use musicfs::music::db::restore_music_metadata_from_db;
|
||||
use musicfs::origins::local::LocalOrigin;
|
||||
use musicfs::origins::network::NetworkOrigin;
|
||||
use musicfs::origins::{FuseFs, Origin};
|
||||
use musicfs::virtual_dirs::restore_virtual_paths;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about, long_about = None)]
|
||||
struct Args {
|
||||
#[arg(short, long, required = true)]
|
||||
mountpoint: String,
|
||||
|
||||
/// Local directory path (→ LocalOrigin) OR `http://host:port` URL of a
|
||||
/// musicfs-server (→ NetworkOrigin).
|
||||
#[arg(short, long, required = true)]
|
||||
source: String,
|
||||
|
||||
#[arg(short, long, required = true)]
|
||||
database: String,
|
||||
|
||||
/// Directory for daily-rotated log files.
|
||||
#[arg(long, default_value = "./logs")]
|
||||
log_dir: std::path::PathBuf,
|
||||
|
||||
/// gRPC server address for health, status, and control RPCs.
|
||||
#[arg(long, default_value = "127.0.0.1:50052")]
|
||||
listen: SocketAddr,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let args = Args::parse();
|
||||
|
||||
// Bind the guard for the whole process so the non-blocking file writer
|
||||
// flushes on exit. Initialized before anything else so even early
|
||||
// failures land in the log.
|
||||
let _guard = init(LogConfig {
|
||||
log_dir: args.log_dir.clone(),
|
||||
file_prefix: "musicfs".to_string(),
|
||||
max_files: 7,
|
||||
});
|
||||
|
||||
let mountpoint = args.mountpoint.clone();
|
||||
info!(
|
||||
mountpoint = %mountpoint,
|
||||
source = %args.source,
|
||||
database = %args.database,
|
||||
log_dir = %args.log_dir.display(),
|
||||
"musicfs starting"
|
||||
);
|
||||
|
||||
// sea-orm logs every statement at INFO by default, which floods normal
|
||||
// output; demote to DEBUG so queries stay hidden under RUST_LOG=info.
|
||||
let mut db_opts = sea_orm::ConnectOptions::new(args.database.clone());
|
||||
db_opts.sqlx_logging_level(log::LevelFilter::Debug);
|
||||
let db = sea_orm::Database::connect(db_opts)
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
error!(database = %args.database, error = %e, "database connect failed");
|
||||
panic!("database connect: {e}");
|
||||
});
|
||||
info!(database = %args.database, "database connected");
|
||||
|
||||
let (snapshot, byte_source, watcher, server_status) = if looks_like_url(&args.source) {
|
||||
debug!(source = %args.source, "using NetworkOrigin");
|
||||
let origin = NetworkOrigin::new(args.source.clone(), mountpoint.clone(), db.clone())
|
||||
.unwrap_or_else(|e| {
|
||||
error!(source = %args.source, error = %e, "network origin init failed");
|
||||
panic!("network origin init: {e}");
|
||||
});
|
||||
let server_status = origin.server_status();
|
||||
let snapshot = origin.snapshot_async().await.unwrap_or_else(|e| {
|
||||
error!(source = %args.source, error = %e, "network initial snapshot failed");
|
||||
panic!("network initial snapshot: {e}");
|
||||
});
|
||||
info!(files = snapshot.len(), "network snapshot complete");
|
||||
(
|
||||
snapshot,
|
||||
origin.byte_source(),
|
||||
origin.watcher(),
|
||||
server_status,
|
||||
)
|
||||
} else {
|
||||
debug!(source = %args.source, "using LocalOrigin");
|
||||
let origin = LocalOrigin::new(args.source.clone(), mountpoint.clone());
|
||||
let mut snapshot = origin.snapshot().unwrap_or_else(|e| {
|
||||
error!(source = %args.source, error = %e, "local initial snapshot failed");
|
||||
panic!("local initial snapshot: {e}");
|
||||
});
|
||||
|
||||
let db_items: HashMap<i64, Model> = Entity::find()
|
||||
.all(&db)
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
error!(error = %e, "loading db items failed");
|
||||
panic!("loading db items: {e}");
|
||||
})
|
||||
.into_iter()
|
||||
.map(|e| (e.inode, e))
|
||||
.collect();
|
||||
sync_items_to_db(&snapshot, &db_items, &db).await;
|
||||
restore_music_metadata_from_db(&mut snapshot, &db_items, &db).await;
|
||||
restore_virtual_paths(&mut snapshot, &db_items, Path::new(&args.source));
|
||||
|
||||
info!(files = snapshot.len(), "local snapshot complete");
|
||||
let server_status = ServerStatus::new_local(mountpoint.clone());
|
||||
(
|
||||
snapshot,
|
||||
origin.byte_source(),
|
||||
origin.watcher(),
|
||||
server_status,
|
||||
)
|
||||
};
|
||||
|
||||
let files: Arc<std::sync::Mutex<std::collections::BTreeMap<INodeNo, Item>>> =
|
||||
Arc::new(std::sync::Mutex::new(snapshot));
|
||||
let watcher_handle = watcher.watch(files.clone());
|
||||
let health_files = files.clone();
|
||||
let grpc_db = db.clone();
|
||||
|
||||
let fs = FuseFs {
|
||||
files,
|
||||
bytes: byte_source,
|
||||
client: db,
|
||||
runtime_handle: tokio::runtime::Handle::current(),
|
||||
};
|
||||
|
||||
let cfg = fuser::Config::default();
|
||||
let session = fuser::spawn_mount2(fs, &mountpoint, &cfg).unwrap_or_else(|e| {
|
||||
error!(mountpoint = %mountpoint, error = %e, "failed to mount FUSE filesystem");
|
||||
panic!("failed to mount FUSE filesystem: {e}");
|
||||
});
|
||||
info!(mountpoint = %mountpoint, "FUSE mounted");
|
||||
|
||||
musicfs::health::spawn_grpc_server(args.listen, server_status, health_files, grpc_db).await;
|
||||
|
||||
let mut sigint = signal(SignalKind::interrupt()).unwrap_or_else(|e| {
|
||||
error!(error = %e, "failed to register SIGINT handler");
|
||||
panic!("register SIGINT handler: {e}");
|
||||
});
|
||||
let mut sigterm = signal(SignalKind::terminate()).unwrap_or_else(|e| {
|
||||
error!(error = %e, "failed to register SIGTERM handler");
|
||||
panic!("register SIGTERM handler: {e}");
|
||||
});
|
||||
|
||||
tokio::select! {
|
||||
_ = sigint.recv() => info!("received SIGINT, shutting down"),
|
||||
_ = sigterm.recv() => info!("received SIGTERM, shutting down"),
|
||||
}
|
||||
|
||||
info!(mountpoint = %mountpoint, "unmounting");
|
||||
watcher_handle.stop();
|
||||
drop(session);
|
||||
}
|
||||
|
||||
fn looks_like_url(s: &str) -> bool {
|
||||
return s.starts_with("http://") || s.starts_with("https://");
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
use crate::db::entities::Model;
|
||||
use crate::item::Item;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use tracing::{debug, error};
|
||||
|
||||
pub mod entities {
|
||||
pub mod music_metadata {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "music_metadata")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
pub track_title: String,
|
||||
pub album: String,
|
||||
pub track_number: i32,
|
||||
#[sea_orm(column_type = "Blob")]
|
||||
pub header: Vec<u8>,
|
||||
pub real_audio_start: i64,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
pub mod artists {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "music_metadata_artists")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub artist: String,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
pub mod other_tags {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "music_metadata_other_tags")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub position: i32,
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
pub mod pictures {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "music_metadata_pictures")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub inode: i64,
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub position: i32,
|
||||
#[sea_orm(column_type = "Blob")]
|
||||
pub block_header: Vec<u8>,
|
||||
pub data_offset: i64,
|
||||
pub data_length: i64,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
}
|
||||
|
||||
use entities::{artists, music_metadata as music_metadata_entity, other_tags, pictures};
|
||||
|
||||
pub async fn save_music_metadata(
|
||||
inode: i64,
|
||||
music_metadata: &MusicMetadata,
|
||||
client: &sea_orm::DatabaseConnection,
|
||||
) -> Result<(), sea_orm::DbErr> {
|
||||
use sea_orm::ActiveValue::Set;
|
||||
|
||||
music_metadata_entity::Entity::delete_by_id(inode)
|
||||
.exec(client)
|
||||
.await?;
|
||||
|
||||
music_metadata_entity::Entity::insert(music_metadata_entity::ActiveModel {
|
||||
inode: Set(inode),
|
||||
track_title: Set(music_metadata.track_title.clone()),
|
||||
album: Set(music_metadata.album.clone()),
|
||||
track_number: Set(music_metadata.track_number),
|
||||
header: Set(music_metadata.header.clone()),
|
||||
real_audio_start: Set(music_metadata.real_audio_start as i64),
|
||||
})
|
||||
.exec(client)
|
||||
.await?;
|
||||
|
||||
if !music_metadata.artist.is_empty() {
|
||||
artists::Entity::insert_many(music_metadata.artist.iter().map(|a| artists::ActiveModel {
|
||||
inode: Set(inode),
|
||||
artist: Set(a.clone()),
|
||||
}))
|
||||
.exec(client)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if !music_metadata.other_tags.is_empty() {
|
||||
other_tags::Entity::insert_many(music_metadata.other_tags.iter().enumerate().map(
|
||||
|(pos, tag)| other_tags::ActiveModel {
|
||||
inode: Set(inode),
|
||||
position: Set(pos as i32),
|
||||
tag: Set(tag.clone()),
|
||||
},
|
||||
))
|
||||
.exec(client)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if !music_metadata.picture_block_headers.is_empty() {
|
||||
pictures::Entity::insert_many(
|
||||
music_metadata
|
||||
.picture_block_headers
|
||||
.iter()
|
||||
.zip(music_metadata.picture_data_ranges.iter())
|
||||
.enumerate()
|
||||
.map(|(pos, (hdr, (offset, len)))| pictures::ActiveModel {
|
||||
inode: Set(inode),
|
||||
position: Set(pos as i32),
|
||||
block_header: Set(hdr.clone()),
|
||||
data_offset: Set(*offset as i64),
|
||||
data_length: Set(*len as i64),
|
||||
}),
|
||||
)
|
||||
.exec(client)
|
||||
.await?;
|
||||
}
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub async fn restore_music_metadata_from_db(
|
||||
snapshot: &mut BTreeMap<INodeNo, Item>,
|
||||
db_items: &HashMap<i64, Model>,
|
||||
client: &sea_orm::DatabaseConnection,
|
||||
) {
|
||||
use artists::Column as ArtCol;
|
||||
use music_metadata_entity::Column as MmCol;
|
||||
use other_tags::Column as OtCol;
|
||||
use pictures::Column as PicCol;
|
||||
|
||||
let unchanged_music_inodes: Vec<i64> = db_items
|
||||
.values()
|
||||
.filter_map(|db_item| {
|
||||
let ino = INodeNo(db_item.inode as u64);
|
||||
snapshot
|
||||
.get(&ino)
|
||||
.filter(|item| item.hash as i64 == db_item.hash && item.music_metadata.is_some())
|
||||
.map(|_| db_item.inode)
|
||||
})
|
||||
.collect();
|
||||
|
||||
if unchanged_music_inodes.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mm_rows: HashMap<i64, music_metadata_entity::Model> =
|
||||
match music_metadata_entity::Entity::find()
|
||||
.filter(MmCol::Inode.is_in(unchanged_music_inodes.clone()))
|
||||
.all(client)
|
||||
.await
|
||||
{
|
||||
Ok(rows) => rows.into_iter().map(|m| (m.inode, m)).collect(),
|
||||
Err(e) => {
|
||||
error!(error = %e, "restore_music_metadata: mm rows query failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut artists_by_inode: HashMap<i64, Vec<String>> = HashMap::new();
|
||||
let artist_rows = match artists::Entity::find()
|
||||
.filter(ArtCol::Inode.is_in(unchanged_music_inodes.clone()))
|
||||
.all(client)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
error!(error = %e, "restore_music_metadata: artists query failed");
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
for row in artist_rows {
|
||||
artists_by_inode
|
||||
.entry(row.inode)
|
||||
.or_default()
|
||||
.push(row.artist);
|
||||
}
|
||||
|
||||
let mut other_tags_by_inode: HashMap<i64, Vec<(i32, String)>> = HashMap::new();
|
||||
let other_tag_rows = match other_tags::Entity::find()
|
||||
.filter(OtCol::Inode.is_in(unchanged_music_inodes.clone()))
|
||||
.all(client)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
error!(error = %e, "restore_music_metadata: other_tags query failed");
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
for row in other_tag_rows {
|
||||
other_tags_by_inode
|
||||
.entry(row.inode)
|
||||
.or_default()
|
||||
.push((row.position, row.tag));
|
||||
}
|
||||
|
||||
let mut pictures_by_inode: HashMap<i64, Vec<pictures::Model>> = HashMap::new();
|
||||
let picture_rows = match pictures::Entity::find()
|
||||
.filter(PicCol::Inode.is_in(unchanged_music_inodes))
|
||||
.all(client)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
error!(error = %e, "restore_music_metadata: pictures query failed");
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
for row in picture_rows {
|
||||
pictures_by_inode.entry(row.inode).or_default().push(row);
|
||||
}
|
||||
|
||||
debug!(restored = mm_rows.len(), "restored music metadata from db");
|
||||
|
||||
for (inode, mm_row) in mm_rows {
|
||||
let ino = INodeNo(inode as u64);
|
||||
let Some(item) = snapshot.get_mut(&ino) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let mut sorted_tags = other_tags_by_inode.remove(&inode).unwrap_or_default();
|
||||
sorted_tags.sort_by_key(|(pos, _)| *pos);
|
||||
|
||||
let mut sorted_pics = pictures_by_inode.remove(&inode).unwrap_or_default();
|
||||
sorted_pics.sort_by_key(|p| p.position);
|
||||
|
||||
let picture_block_headers: Vec<Vec<u8>> =
|
||||
sorted_pics.iter().map(|p| p.block_header.clone()).collect();
|
||||
|
||||
let picture_data_ranges: Vec<(u64, u64)> = sorted_pics
|
||||
.iter()
|
||||
.map(|p| (p.data_offset as u64, p.data_length as u64))
|
||||
.collect();
|
||||
|
||||
let mut music_metadata = MusicMetadata {
|
||||
artist: artists_by_inode.remove(&inode).unwrap_or_default(),
|
||||
album_artist: None,
|
||||
album: mm_row.album,
|
||||
track_number: mm_row.track_number,
|
||||
track_title: mm_row.track_title,
|
||||
other_tags: sorted_tags.into_iter().map(|(_, tag)| tag).collect(),
|
||||
header: mm_row.header,
|
||||
picture_block_headers,
|
||||
picture_data_ranges,
|
||||
real_audio_start: mm_row.real_audio_start as u64,
|
||||
vorbis_comment_offset: 0,
|
||||
vorbis_comment_length: 0,
|
||||
};
|
||||
music_metadata.find_vorbis_offsets();
|
||||
item.music_metadata = Some(music_metadata);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod db;
|
||||
pub use musicfs_core::music::{metadata, parse};
|
||||
@@ -0,0 +1,215 @@
|
||||
use std::{
|
||||
fs,
|
||||
io::{self, Read, Seek, SeekFrom},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
pub fn read_bytes_at(path: &Path, offset: u64, len: usize) -> io::Result<Vec<u8>> {
|
||||
let mut f = fs::File::open(path)?;
|
||||
f.seek(SeekFrom::Start(offset))?;
|
||||
let mut buf = vec![0u8; len];
|
||||
let n = f.read(&mut buf)?;
|
||||
buf.truncate(n);
|
||||
|
||||
return Ok(buf);
|
||||
}
|
||||
|
||||
fn assemble_virtual_read(
|
||||
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
|
||||
header: &[u8],
|
||||
pic_hdrs: &[Vec<u8>],
|
||||
pic_ranges: &[(u64, u64)],
|
||||
real_audio_start: u64,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let end = offset + size as u64;
|
||||
let header_end = header.len() as u64;
|
||||
|
||||
if end <= header_end {
|
||||
return Ok(header[offset as usize..end as usize].to_vec());
|
||||
}
|
||||
|
||||
let mut buf = Vec::with_capacity(size as usize);
|
||||
if offset < header_end {
|
||||
buf.extend_from_slice(&header[offset as usize..]);
|
||||
}
|
||||
|
||||
let mut virt_pos = header_end;
|
||||
for (pic_hdr, (data_real_offset, data_len)) in pic_hdrs.iter().zip(pic_ranges.iter()) {
|
||||
let pic_hdr_len = pic_hdr.len() as u64;
|
||||
let pic_hdr_end = virt_pos + pic_hdr_len;
|
||||
let pic_end = pic_hdr_end + data_len;
|
||||
|
||||
if end <= virt_pos {
|
||||
break;
|
||||
}
|
||||
if offset >= pic_end {
|
||||
virt_pos = pic_end;
|
||||
continue;
|
||||
}
|
||||
|
||||
let hdr_from = (offset.max(virt_pos) - virt_pos) as usize;
|
||||
let hdr_to = ((end.min(pic_hdr_end)) - virt_pos) as usize;
|
||||
if hdr_from < hdr_to {
|
||||
buf.extend_from_slice(&pic_hdr[hdr_from..hdr_to.min(pic_hdr.len())]);
|
||||
}
|
||||
|
||||
let data_start = offset.max(pic_hdr_end);
|
||||
let data_end = end.min(pic_end);
|
||||
if data_start < data_end {
|
||||
let real_off = data_real_offset + (data_start - pic_hdr_end);
|
||||
let bytes = reader(real_off, (data_end - data_start) as usize)?;
|
||||
buf.extend_from_slice(&bytes);
|
||||
}
|
||||
|
||||
virt_pos = pic_end;
|
||||
}
|
||||
|
||||
let audio_start = offset.max(virt_pos);
|
||||
if audio_start < end {
|
||||
let real_off = real_audio_start + (audio_start - virt_pos);
|
||||
let bytes = reader(real_off, (end - audio_start) as usize)?;
|
||||
buf.extend_from_slice(&bytes);
|
||||
}
|
||||
|
||||
return Ok(buf);
|
||||
}
|
||||
|
||||
pub fn assemble_flac_read(
|
||||
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
|
||||
header: &[u8],
|
||||
pic_hdrs: &[Vec<u8>],
|
||||
pic_ranges: &[(u64, u64)],
|
||||
real_audio_start: u64,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
assemble_virtual_read(
|
||||
reader,
|
||||
header,
|
||||
pic_hdrs,
|
||||
pic_ranges,
|
||||
real_audio_start,
|
||||
offset,
|
||||
size,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn assemble_mp3_read(
|
||||
reader: &dyn Fn(u64, usize) -> io::Result<Vec<u8>>,
|
||||
header: &[u8],
|
||||
pic_hdrs: &[Vec<u8>],
|
||||
pic_ranges: &[(u64, u64)],
|
||||
real_audio_start: u64,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
assemble_virtual_read(
|
||||
reader,
|
||||
header,
|
||||
pic_hdrs,
|
||||
pic_ranges,
|
||||
real_audio_start,
|
||||
offset,
|
||||
size,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn file_reader(path: PathBuf) -> impl Fn(u64, usize) -> io::Result<Vec<u8>> {
|
||||
return move |offset, len| read_bytes_at(&path, offset, len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_bytes_at_full_file() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
f.write_all(b"hello world").unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path();
|
||||
|
||||
let result = read_bytes_at(path, 0, 11).unwrap();
|
||||
assert_eq!(result, b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_bytes_at_offset() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
f.write_all(b"abcdefghij").unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path();
|
||||
|
||||
let result = read_bytes_at(path, 3, 4).unwrap();
|
||||
assert_eq!(result, b"defg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_header_only() {
|
||||
let header = b"HEADERDATA";
|
||||
let noop = |_, _| Ok(vec![]);
|
||||
let result = assemble_flac_read(&noop, header, &[], &[], 0, 2, 4).unwrap();
|
||||
assert_eq!(result, b"ADER");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_audio_region() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let content: Vec<u8> = (0..20).collect();
|
||||
f.write_all(&content).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let reader = file_reader(path);
|
||||
let header = b"HDR";
|
||||
let result = assemble_flac_read(&reader, header, &[], &[], 10, 5, 4).unwrap();
|
||||
// offset=5, header_len=3, so we read from header[3..] (0 bytes) + audio at real_off=10+(5-3)=12
|
||||
// content[12..16] = [12, 13, 14, 15]
|
||||
assert_eq!(result, vec![12, 13, 14, 15]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_header_audio_boundary() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let content: Vec<u8> = (0..110).collect();
|
||||
f.write_all(&content).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let reader = file_reader(path);
|
||||
let header = b"ABCD";
|
||||
let result = assemble_flac_read(&reader, header, &[], &[], 100, 2, 6).unwrap();
|
||||
// offset=2, size=6, header_len=4
|
||||
// First 2 bytes from header[2..4] = "CD"
|
||||
// Remaining 4 bytes from audio at real_off=100+(2+2-4)=100
|
||||
// content[100..104] = [100, 101, 102, 103]
|
||||
let expected = [b'C', b'D', 100, 101, 102, 103];
|
||||
assert_eq!(result, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_mp3_read_header_only() {
|
||||
let header = b"ID3\x04\x00DATAHERE";
|
||||
let noop = |_, _| Ok(vec![]);
|
||||
let result = assemble_mp3_read(&noop, header, &[], &[], 0, 4, 4).unwrap();
|
||||
assert_eq!(result, &header[4..8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_mp3_read_audio_region() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let content: Vec<u8> = (0..20).collect();
|
||||
f.write_all(&content).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let reader = file_reader(path);
|
||||
let header = b"ID3";
|
||||
let result = assemble_mp3_read(&reader, header, &[], &[], 10, 5, 4).unwrap();
|
||||
assert_eq!(result, vec![12, 13, 14, 15]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
pub mod file_io;
|
||||
pub mod snapshot;
|
||||
pub mod watcher;
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
io,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
|
||||
use crate::item::Item;
|
||||
use crate::origins::{ByteSource, FileWatcher, Origin};
|
||||
pub struct LocalOrigin {
|
||||
pub(crate) source: PathBuf,
|
||||
pub(crate) destination: PathBuf,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LocalOrigin {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return f
|
||||
.debug_struct("LocalOrigin")
|
||||
.field("source", &self.source)
|
||||
.field("destination", &self.destination)
|
||||
.finish();
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalOrigin {
|
||||
pub fn new(source: String, destination: String) -> LocalOrigin {
|
||||
return LocalOrigin {
|
||||
source: source.into(),
|
||||
destination: destination.into(),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn source(&self) -> &Path {
|
||||
return &self.source;
|
||||
}
|
||||
}
|
||||
|
||||
impl Origin for LocalOrigin {
|
||||
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
|
||||
return snapshot::build_snapshot(&self.source, &self.destination);
|
||||
}
|
||||
|
||||
fn byte_source(&self) -> Arc<dyn ByteSource> {
|
||||
return Arc::new(LocalByteSource);
|
||||
}
|
||||
|
||||
fn watcher(&self) -> Box<dyn FileWatcher> {
|
||||
return Box::new(watcher::LocalOriginFileWatcher::new(
|
||||
self.source.clone(),
|
||||
self.destination.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LocalByteSource;
|
||||
|
||||
impl ByteSource for LocalByteSource {
|
||||
fn read_at(
|
||||
&self,
|
||||
_inode: INodeNo,
|
||||
locator: &Path,
|
||||
offset: u64,
|
||||
len: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
return file_io::read_bytes_at(locator, offset, len);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs, io,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::music::parse::parse_music_metadata_for_path;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
use crate::virtual_dirs::ensure_virtual_dirs;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub fn fill_fileset(map: &Arc<Mutex<BTreeMap<INodeNo, Item>>>, source: &Path, destination: &Path) {
|
||||
match build_snapshot(source, destination) {
|
||||
Ok(new_snapshot) => {
|
||||
let count = new_snapshot.len();
|
||||
let mut files = map.lock().unwrap();
|
||||
|
||||
files.retain(|ino, _| new_snapshot.contains_key(ino));
|
||||
|
||||
for (ino, new_item) in new_snapshot {
|
||||
match files.get(&ino) {
|
||||
Some(existing) if existing.hash == new_item.hash => {}
|
||||
_ => {
|
||||
files.insert(ino, new_item);
|
||||
}
|
||||
}
|
||||
}
|
||||
info!(count, "snapshot rebuilt");
|
||||
}
|
||||
Err(e) => error!(source = %source.display(), error = %e, "error while reading source"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_snapshot(
|
||||
source: &Path,
|
||||
destination: &Path,
|
||||
) -> Result<BTreeMap<INodeNo, Item>, io::Error> {
|
||||
let mut map = BTreeMap::new();
|
||||
|
||||
let local_root = Item::new(
|
||||
INodeNo::ROOT,
|
||||
INodeNo::ROOT,
|
||||
"/".to_string(),
|
||||
source.to_path_buf(),
|
||||
destination.to_path_buf(),
|
||||
FileType::Directory,
|
||||
FileAttrs::from(&fs::metadata(source)?),
|
||||
None,
|
||||
);
|
||||
map.insert(INodeNo::ROOT, local_root);
|
||||
|
||||
read_into_map(source, destination, &mut map)?;
|
||||
return Ok(map);
|
||||
}
|
||||
|
||||
pub fn read_into_map(
|
||||
source: &Path,
|
||||
destination: &Path,
|
||||
map: &mut BTreeMap<INodeNo, Item>,
|
||||
) -> Result<(), io::Error> {
|
||||
for item in fs::read_dir(source)? {
|
||||
let entry = item?;
|
||||
let item_path = entry.path();
|
||||
|
||||
if entry.file_type()?.is_dir() {
|
||||
read_into_map(&item_path, destination, map)?;
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
let metadata = entry.metadata()?;
|
||||
let music_metadata = parse_music_metadata_for_path(&item_path);
|
||||
|
||||
let mut local_path = PathBuf::new();
|
||||
if let Some(ref mm) = music_metadata {
|
||||
let joined;
|
||||
let artist_dir = match mm.album_artist.as_deref() {
|
||||
Some(a) => a,
|
||||
None => {
|
||||
joined = mm.artist.join("-");
|
||||
&joined
|
||||
}
|
||||
};
|
||||
local_path.push(artist_dir);
|
||||
local_path.push(&mm.album);
|
||||
}
|
||||
local_path.push(&name);
|
||||
|
||||
let inode = INodeNo(metadata.ino());
|
||||
let parent_inode = ensure_virtual_dirs(&local_path, source, map);
|
||||
let local_item = Item::new(
|
||||
inode,
|
||||
parent_inode,
|
||||
name,
|
||||
item_path,
|
||||
local_path,
|
||||
FileType::File,
|
||||
FileAttrs::from(&metadata),
|
||||
music_metadata,
|
||||
);
|
||||
|
||||
map.insert(inode, local_item);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Mutex, mpsc},
|
||||
thread,
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use notify::{Event, EventKind, RecursiveMode, Watcher};
|
||||
|
||||
use crate::item::Item;
|
||||
use crate::origins::{FileWatcher, WatcherHandle};
|
||||
use tracing::{debug, error, info, trace};
|
||||
|
||||
pub struct LocalOriginFileWatcher {
|
||||
source: PathBuf,
|
||||
destination: PathBuf,
|
||||
}
|
||||
|
||||
impl LocalOriginFileWatcher {
|
||||
pub fn new(source: PathBuf, destination: PathBuf) -> Self {
|
||||
return LocalOriginFileWatcher {
|
||||
source,
|
||||
destination,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
impl FileWatcher for LocalOriginFileWatcher {
|
||||
fn watch(&self, files: Arc<Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle {
|
||||
let source = self.source.clone();
|
||||
let destination = self.destination.clone();
|
||||
|
||||
info!(source = %source.display(), "starting file watcher");
|
||||
thread::spawn(move || {
|
||||
let (tx, rx): (
|
||||
mpsc::Sender<Result<Event, notify::Error>>,
|
||||
mpsc::Receiver<Result<Event, notify::Error>>,
|
||||
) = mpsc::channel();
|
||||
|
||||
let mut watcher: notify::INotifyWatcher = match notify::recommended_watcher(tx) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
error!(error = %e, "watcher: failed to create notify watcher");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = watcher.watch(&source, RecursiveMode::Recursive) {
|
||||
error!(source = %source.display(), error = %e, "watcher: failed to watch source");
|
||||
return;
|
||||
}
|
||||
for res in rx {
|
||||
match res {
|
||||
Ok(event) => {
|
||||
trace!(?event, "watcher event");
|
||||
|
||||
match event.kind {
|
||||
EventKind::Any => {
|
||||
debug!("watcher: EventKind::Any, ignoring");
|
||||
}
|
||||
EventKind::Access(_access_kind) => {
|
||||
debug!("watcher: item accessed");
|
||||
}
|
||||
EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) => {
|
||||
debug!("watcher: create/modify/remove; rebuilding snapshot");
|
||||
super::snapshot::fill_fileset(&files, &source, &destination);
|
||||
}
|
||||
EventKind::Other => {
|
||||
debug!("watcher: other event, ignoring");
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => error!(error = %e, "watcher: notify error"),
|
||||
}
|
||||
}
|
||||
});
|
||||
return WatcherHandle::detached();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,631 @@
|
||||
pub use musicfs_core::attrs;
|
||||
pub mod local;
|
||||
pub mod network;
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
io,
|
||||
path::Path,
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use fuser::{Errno, FileAttr, Filesystem, Generation, INodeNo, Request};
|
||||
use sea_orm::{ActiveModelTrait, DatabaseConnection};
|
||||
use tracing::{debug, error, trace};
|
||||
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::music::db::save_music_metadata;
|
||||
use crate::origins::local::file_io;
|
||||
use crate::virtual_dirs::parent_inode_from_path;
|
||||
|
||||
/// Transport-agnostic byte-range reader. `locator` is whatever string the
|
||||
/// origin treats as a file key: a filesystem path for `LocalOrigin`, a remote
|
||||
/// key for the future network origin. Both produce bytes at `(offset, len)`.
|
||||
///
|
||||
/// `inode` is the FUSE-level inode; NetworkOrigin uses it as the server-side
|
||||
/// file id (and as the Postgres cache key). LocalOrigin ignores it.
|
||||
pub trait ByteSource: Send + Sync {
|
||||
fn read_at(
|
||||
&self,
|
||||
inode: INodeNo,
|
||||
locator: &Path,
|
||||
offset: u64,
|
||||
len: usize,
|
||||
) -> io::Result<Vec<u8>>;
|
||||
}
|
||||
|
||||
pub trait FileWatcher: Send + Sync {
|
||||
fn watch(&self, files: Arc<Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle;
|
||||
}
|
||||
|
||||
/// Returned by [`FileWatcher::watch`] so the caller can stop a background
|
||||
/// watcher before the process (and its tokio runtime) shuts down. Without
|
||||
/// this, an async watcher blocked on a timer/stream panics when the runtime is
|
||||
/// torn down out from under it.
|
||||
pub struct WatcherHandle {
|
||||
stop: Option<Box<dyn FnOnce() + Send>>,
|
||||
}
|
||||
|
||||
impl WatcherHandle {
|
||||
pub fn new(stop: impl FnOnce() + Send + 'static) -> Self {
|
||||
return WatcherHandle {
|
||||
stop: Some(Box::new(stop)),
|
||||
};
|
||||
}
|
||||
|
||||
/// A watcher with no shutdown work — its thread exits on its own.
|
||||
pub fn detached() -> Self {
|
||||
return WatcherHandle { stop: None };
|
||||
}
|
||||
|
||||
/// Signal the watcher to stop and wait for it to finish.
|
||||
pub fn stop(mut self) {
|
||||
if let Some(stop) = self.stop.take() {
|
||||
stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Origin: Send + Sync {
|
||||
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>>;
|
||||
fn byte_source(&self) -> Arc<dyn ByteSource>;
|
||||
fn watcher(&self) -> Box<dyn FileWatcher>;
|
||||
}
|
||||
|
||||
pub struct FuseFs {
|
||||
pub files: Arc<Mutex<BTreeMap<INodeNo, Item>>>,
|
||||
pub bytes: Arc<dyn ByteSource>,
|
||||
pub client: DatabaseConnection,
|
||||
pub runtime_handle: tokio::runtime::Handle,
|
||||
}
|
||||
|
||||
pub(crate) fn file_to_attr(item: &Item) -> FileAttr {
|
||||
let attrs = &item.attrs;
|
||||
let kind = match item.file_type {
|
||||
FileType::Directory => fuser::FileType::Directory,
|
||||
FileType::File => fuser::FileType::RegularFile,
|
||||
};
|
||||
|
||||
let size = match &item.music_metadata {
|
||||
Some(mm) if !mm.header.is_empty() => mm.virtual_size(attrs.size),
|
||||
_ => attrs.size,
|
||||
};
|
||||
|
||||
return FileAttr {
|
||||
ino: item.inode,
|
||||
size,
|
||||
blocks: attrs.blocks,
|
||||
atime: attrs.atime,
|
||||
mtime: attrs.mtime,
|
||||
ctime: attrs.ctime,
|
||||
crtime: attrs.crtime,
|
||||
kind,
|
||||
perm: attrs.perm,
|
||||
nlink: attrs.nlink,
|
||||
uid: attrs.uid,
|
||||
gid: attrs.gid,
|
||||
rdev: attrs.rdev,
|
||||
blksize: attrs.blksize,
|
||||
flags: 0,
|
||||
};
|
||||
}
|
||||
|
||||
impl Filesystem for FuseFs {
|
||||
fn open(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_flags: fuser::OpenFlags,
|
||||
reply: fuser::ReplyOpen,
|
||||
) {
|
||||
trace!(%ino, "open");
|
||||
if self.files.lock().unwrap().contains_key(&ino) {
|
||||
reply.opened(fuser::FileHandle(ino.0), fuser::FopenFlags::empty());
|
||||
} else {
|
||||
debug!(%ino, "open: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
fn setattr(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_mode: Option<u32>,
|
||||
_uid: Option<u32>,
|
||||
_gid: Option<u32>,
|
||||
_size: Option<u64>,
|
||||
_atime: Option<fuser::TimeOrNow>,
|
||||
_mtime: Option<fuser::TimeOrNow>,
|
||||
_ctime: Option<std::time::SystemTime>,
|
||||
_fh: Option<fuser::FileHandle>,
|
||||
_crtime: Option<std::time::SystemTime>,
|
||||
_chgtime: Option<std::time::SystemTime>,
|
||||
_bkuptime: Option<std::time::SystemTime>,
|
||||
_flags: Option<fuser::BsdFileFlags>,
|
||||
reply: fuser::ReplyAttr,
|
||||
) {
|
||||
trace!(%ino, "setattr");
|
||||
match self.files.lock().unwrap().get(&ino) {
|
||||
Some(file) => reply.attr(&Duration::new(1, 0), &file_to_attr(file)),
|
||||
None => {
|
||||
debug!(%ino, "setattr: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_fh: fuser::FileHandle,
|
||||
offset: u64,
|
||||
data: &[u8],
|
||||
_write_flags: fuser::WriteFlags,
|
||||
_flags: fuser::OpenFlags,
|
||||
_lock_owner: Option<fuser::LockOwner>,
|
||||
reply: fuser::ReplyWrite,
|
||||
) {
|
||||
trace!(%ino, offset, len = data.len(), "write");
|
||||
let written = data.len() as u32;
|
||||
let write_start = offset;
|
||||
let write_end = write_start + data.len() as u64;
|
||||
|
||||
let updated_music_metadata = {
|
||||
let mut files = self.files.lock().unwrap();
|
||||
let item = match files.get_mut(&ino) {
|
||||
Some(item) => item,
|
||||
None => {
|
||||
debug!(%ino, "write: not found");
|
||||
reply.written(written);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match &mut item.music_metadata {
|
||||
Some(mm) if mm.vorbis_comment_length > 0 => {
|
||||
let vc_data_offset = mm.vorbis_comment_offset;
|
||||
let vc_hdr_offset = vc_data_offset - 4;
|
||||
if write_start <= vc_hdr_offset && write_end >= vc_data_offset {
|
||||
let hdr_from = (vc_hdr_offset - write_start) as usize;
|
||||
let new_length = u32::from_be_bytes([
|
||||
0,
|
||||
data[hdr_from + 1],
|
||||
data[hdr_from + 2],
|
||||
data[hdr_from + 3],
|
||||
]) as u64;
|
||||
let vc_data_end = vc_data_offset + new_length;
|
||||
if write_end >= vc_data_end {
|
||||
let from = (vc_data_offset - write_start) as usize;
|
||||
let to = (vc_data_end - write_start) as usize;
|
||||
mm.update_from_vorbis_comment_data(&data[from..to]);
|
||||
debug!(%ino, "write: vorbis comment tag update detected");
|
||||
Some(mm.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Some(mm)
|
||||
if !mm.header.is_empty()
|
||||
&& write_start == 0
|
||||
&& data.len() >= 3
|
||||
&& &data[0..3] == b"ID3" =>
|
||||
{
|
||||
mm.update_from_id3_data(data);
|
||||
debug!(%ino, "write: ID3v2 tag update detected");
|
||||
Some(mm.clone())
|
||||
}
|
||||
Some(mm) if mm.header.is_empty() && data.len() == 128 && &data[0..3] == b"TAG" => {
|
||||
mm.update_from_id3v1_data(data);
|
||||
debug!(%ino, "write: ID3v1 tag update detected");
|
||||
Some(mm.clone())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(music_metadata) = updated_music_metadata {
|
||||
let client = self.client.clone();
|
||||
let ino_i64 = ino.0 as i64;
|
||||
self.runtime_handle.block_on(async move {
|
||||
if let Err(e) = save_music_metadata(ino_i64, &music_metadata, &client).await {
|
||||
error!(ino = ino_i64, error = %e, "write: save_music_metadata failed");
|
||||
} else {
|
||||
debug!(ino = ino_i64, "write: persisted updated music metadata");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
reply.written(written);
|
||||
}
|
||||
|
||||
fn getattr(
|
||||
&self,
|
||||
_req: &fuser::Request,
|
||||
ino: INodeNo,
|
||||
_fh: Option<fuser::FileHandle>,
|
||||
reply: fuser::ReplyAttr,
|
||||
) {
|
||||
trace!(%ino, "getattr");
|
||||
match self.files.lock().unwrap().get(&ino) {
|
||||
Some(file) => {
|
||||
let ttl = Duration::new(1, 0);
|
||||
let attr = file_to_attr(file);
|
||||
reply.attr(&ttl, &attr);
|
||||
}
|
||||
None => {
|
||||
debug!(%ino, "getattr: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn readdir(
|
||||
&self,
|
||||
_req: &fuser::Request,
|
||||
ino: INodeNo,
|
||||
fh: fuser::FileHandle,
|
||||
offset: u64,
|
||||
mut reply: fuser::ReplyDirectory,
|
||||
) {
|
||||
trace!(%ino, %fh, offset, "readdir");
|
||||
|
||||
let files = self.files.lock().unwrap();
|
||||
|
||||
let parent_inode = match files.get(&ino) {
|
||||
Some(dir) => dir.parent_inode,
|
||||
None => {
|
||||
debug!(%ino, "readdir: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if offset < 1 {
|
||||
if reply.add(ino, 1, fuser::FileType::Directory, ".") {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if offset < 2 {
|
||||
if reply.add(parent_inode, 2, fuser::FileType::Directory, "..") {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let skip_count = if offset <= 2 {
|
||||
0
|
||||
} else {
|
||||
(offset - 2) as usize
|
||||
};
|
||||
|
||||
for (i, (key, value)) in files
|
||||
.iter()
|
||||
.filter(|(_, v)| v.parent_inode == ino && v.inode != ino)
|
||||
.skip(skip_count)
|
||||
.enumerate()
|
||||
{
|
||||
let entry_offset = (skip_count + i + 3) as u64;
|
||||
let file_type = match value.file_type {
|
||||
FileType::Directory => fuser::FileType::Directory,
|
||||
FileType::File => fuser::FileType::RegularFile,
|
||||
};
|
||||
if reply.add(*key, entry_offset, file_type, &value.name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn rename(
|
||||
&self,
|
||||
_req: &Request,
|
||||
parent: INodeNo,
|
||||
name: &std::ffi::OsStr,
|
||||
newparent: INodeNo,
|
||||
newname: &std::ffi::OsStr,
|
||||
_flags: fuser::RenameFlags,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
trace!(
|
||||
%parent,
|
||||
name = %name.display(),
|
||||
%newparent,
|
||||
newname = %newname.display(),
|
||||
"rename"
|
||||
);
|
||||
let name_str = match name.to_str() {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
error!(%parent, "rename: source name is not valid UTF-8");
|
||||
reply.error(Errno::EINVAL);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let newname_str = match newname.to_str() {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
error!(%newparent, "rename: target name is not valid UTF-8");
|
||||
reply.error(Errno::EINVAL);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut files = self.files.lock().unwrap();
|
||||
|
||||
let item_ino = match files
|
||||
.iter()
|
||||
.find(|(_, v)| v.parent_inode == parent && v.name == name_str)
|
||||
{
|
||||
Some((ino, _)) => *ino,
|
||||
None => {
|
||||
debug!(%parent, name = %name_str, "rename: source not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let new_local_path = if newparent == INodeNo::ROOT {
|
||||
std::path::PathBuf::from(newname_str)
|
||||
} else {
|
||||
match files.get(&newparent) {
|
||||
Some(dir) => dir.local_path.join(newname_str),
|
||||
None => {
|
||||
debug!(%newparent, "rename: target parent not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
debug!(ino = %item_ino, from = %name_str, to = %newname_str, "rename");
|
||||
|
||||
let new_parent_inode = parent_inode_from_path(&new_local_path);
|
||||
|
||||
let item = files.get_mut(&item_ino).unwrap();
|
||||
item.name = newname_str.to_string();
|
||||
item.local_path = new_local_path.clone();
|
||||
item.parent_inode = new_parent_inode;
|
||||
|
||||
let inode_i64 = item_ino.0 as i64;
|
||||
let new_name_owned = newname_str.to_string();
|
||||
let new_local_path_str = new_local_path.to_string_lossy().into_owned();
|
||||
let client = self.client.clone();
|
||||
|
||||
drop(files);
|
||||
|
||||
self.runtime_handle.block_on(async move {
|
||||
use sea_orm::ActiveValue::Set;
|
||||
if let Err(e) = (crate::db::entities::ActiveModel {
|
||||
inode: Set(inode_i64),
|
||||
name: Set(new_name_owned),
|
||||
local_path: Set(new_local_path_str),
|
||||
..Default::default()
|
||||
}
|
||||
.update(&client)
|
||||
.await)
|
||||
{
|
||||
error!(ino = inode_i64, error = %e, "rename: db update failed");
|
||||
}
|
||||
});
|
||||
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn read(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_fh: fuser::FileHandle,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
_flags: fuser::OpenFlags,
|
||||
_lock_owner: Option<fuser::LockOwner>,
|
||||
reply: fuser::ReplyData,
|
||||
) {
|
||||
trace!(%ino, offset, size, "read");
|
||||
let (inode, locator, virtual_layout) = {
|
||||
let files = self.files.lock().unwrap();
|
||||
let item = match files.get(&ino) {
|
||||
Some(f) => f,
|
||||
None => {
|
||||
debug!(%ino, "read: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let inode = item.inode;
|
||||
let locator = item.original_path.clone();
|
||||
let virtual_layout = item
|
||||
.music_metadata
|
||||
.as_ref()
|
||||
.filter(|mm| !mm.header.is_empty())
|
||||
.map(|mm| {
|
||||
(
|
||||
mm.header.starts_with(b"ID3"),
|
||||
mm.header.clone(),
|
||||
mm.picture_block_headers.clone(),
|
||||
mm.picture_data_ranges.clone(),
|
||||
mm.real_audio_start,
|
||||
)
|
||||
});
|
||||
(inode, locator, virtual_layout)
|
||||
};
|
||||
|
||||
let Some((is_mp3, header, pic_hdrs, pic_ranges, real_audio_start)) = virtual_layout else {
|
||||
match self.bytes.read_at(inode, &locator, offset, size as usize) {
|
||||
Ok(bytes) => reply.data(&bytes),
|
||||
Err(e) => {
|
||||
error!(%inode, offset, size, error = %e, "read: read_at failed; returning EIO");
|
||||
reply.error(Errno::EIO);
|
||||
}
|
||||
}
|
||||
return;
|
||||
};
|
||||
|
||||
let bytes = &self.bytes;
|
||||
let reader = |off: u64, len: usize| bytes.read_at(inode, &locator, off, len);
|
||||
let result = if is_mp3 {
|
||||
file_io::assemble_mp3_read(
|
||||
&reader,
|
||||
&header,
|
||||
&pic_hdrs,
|
||||
&pic_ranges,
|
||||
real_audio_start,
|
||||
offset,
|
||||
size,
|
||||
)
|
||||
} else {
|
||||
file_io::assemble_flac_read(
|
||||
&reader,
|
||||
&header,
|
||||
&pic_hdrs,
|
||||
&pic_ranges,
|
||||
real_audio_start,
|
||||
offset,
|
||||
size,
|
||||
)
|
||||
};
|
||||
match result {
|
||||
Ok(bytes) => reply.data(&bytes),
|
||||
Err(e) => {
|
||||
error!(%inode, offset, size, error = %e, "read: assembly failed; returning EIO");
|
||||
reply.error(Errno::EIO);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn lookup(
|
||||
&self,
|
||||
_req: &Request,
|
||||
parent: INodeNo,
|
||||
name: &std::ffi::OsStr,
|
||||
reply: fuser::ReplyEntry,
|
||||
) {
|
||||
trace!(%parent, name = %name.display(), "lookup");
|
||||
|
||||
match self
|
||||
.files
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|item| item.1.parent_inode == parent && item.1.name == name.to_str().unwrap())
|
||||
{
|
||||
Some(item) => {
|
||||
let ttl = Duration::new(1, 0);
|
||||
let attr = file_to_attr(item.1);
|
||||
|
||||
reply.entry(&ttl, &attr, Generation(0));
|
||||
}
|
||||
None => {
|
||||
debug!(%parent, name = %name.display(), "lookup: not found");
|
||||
reply.error(Errno::ENOENT);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::item::FileType;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
use fuser::INodeNo;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn file_to_attr_uses_real_size_for_non_flac() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
let attrs = FileAttrs::from(&metadata);
|
||||
let item = Item::new(
|
||||
INodeNo(42),
|
||||
INodeNo::ROOT,
|
||||
"test".to_string(),
|
||||
tmp.path().to_path_buf(),
|
||||
PathBuf::from("test"),
|
||||
FileType::File,
|
||||
attrs.clone(),
|
||||
None,
|
||||
);
|
||||
|
||||
let attr = file_to_attr(&item);
|
||||
assert_eq!(attr.size, metadata.size());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_to_attr_uses_virtual_size_for_flac() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let test_file = tmp.path().join("test.flac");
|
||||
std::fs::write(&test_file, vec![0u8; 1000]).unwrap();
|
||||
let metadata = std::fs::metadata(&test_file).unwrap();
|
||||
let attrs = FileAttrs::from(&metadata);
|
||||
let mm = MusicMetadata {
|
||||
real_audio_start: 500,
|
||||
header: vec![0u8; 100],
|
||||
picture_data_ranges: vec![(0, 50)],
|
||||
..MusicMetadata::default()
|
||||
};
|
||||
let item = Item::new(
|
||||
INodeNo(43),
|
||||
INodeNo::ROOT,
|
||||
"test_flac".to_string(),
|
||||
test_file.clone(),
|
||||
PathBuf::from("test_flac"),
|
||||
FileType::File,
|
||||
attrs.clone(),
|
||||
Some(mm.clone()),
|
||||
);
|
||||
|
||||
let attr = file_to_attr(&item);
|
||||
let expected_virtual_size = mm.virtual_size(attrs.size);
|
||||
assert_eq!(attr.size, expected_virtual_size);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_to_attr_kind_directory() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
let item = Item::new(
|
||||
INodeNo(44),
|
||||
INodeNo::ROOT,
|
||||
"test_dir".to_string(),
|
||||
tmp.path().to_path_buf(),
|
||||
PathBuf::from("test_dir"),
|
||||
FileType::Directory,
|
||||
FileAttrs::from(&metadata),
|
||||
None,
|
||||
);
|
||||
|
||||
let attr = file_to_attr(&item);
|
||||
assert_eq!(attr.kind, fuser::FileType::Directory);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_to_attr_kind_file() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = std::fs::metadata(tmp.path()).unwrap();
|
||||
let item = Item::new(
|
||||
INodeNo(45),
|
||||
INodeNo::ROOT,
|
||||
"test_file".to_string(),
|
||||
tmp.path().to_path_buf(),
|
||||
PathBuf::from("test_file"),
|
||||
FileType::File,
|
||||
FileAttrs::from(&metadata),
|
||||
None,
|
||||
);
|
||||
|
||||
let attr = file_to_attr(&item);
|
||||
assert_eq!(attr.kind, fuser::FileType::RegularFile);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
pub mod transport;
|
||||
pub mod watcher;
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
io,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, RwLock},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use sea_orm::EntityTrait;
|
||||
|
||||
use crate::db::cache::{delete_cached_bytes_for, get_cached_bytes, put_cached_bytes};
|
||||
use crate::db::entities as item_entities;
|
||||
use crate::db::sync::{run_db_blocking, sync_items_to_db};
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::music::db::restore_music_metadata_from_db;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
use crate::origins::{ByteSource, FileWatcher, Origin};
|
||||
use crate::proto::ManifestEntry as ProtoManifestEntry;
|
||||
use crate::virtual_dirs::{ensure_virtual_dirs, restore_virtual_paths};
|
||||
use tracing::{debug, error, info, trace};
|
||||
|
||||
use self::transport::NetworkTransport;
|
||||
|
||||
pub struct NetworkOrigin {
|
||||
pub(crate) endpoint: String,
|
||||
pub(crate) destination: PathBuf,
|
||||
handle: tokio::runtime::Handle,
|
||||
transport: NetworkTransport,
|
||||
client: sea_orm::DatabaseConnection,
|
||||
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
|
||||
server_status: crate::health::ServerStatus,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for NetworkOrigin {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return f
|
||||
.debug_struct("NetworkOrigin")
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("destination", &self.destination)
|
||||
.finish();
|
||||
}
|
||||
}
|
||||
|
||||
impl NetworkOrigin {
|
||||
pub fn new(
|
||||
endpoint: String,
|
||||
destination: String,
|
||||
client: sea_orm::DatabaseConnection,
|
||||
) -> io::Result<Self> {
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
let transport = NetworkTransport::new(endpoint.clone()).map_err(io_err)?;
|
||||
let server_status =
|
||||
crate::health::ServerStatus::new_network(destination.clone(), endpoint.clone());
|
||||
return Ok(NetworkOrigin {
|
||||
endpoint,
|
||||
destination: destination.into(),
|
||||
handle,
|
||||
transport,
|
||||
client,
|
||||
latest_manifest: Arc::new(RwLock::new(BTreeMap::new())),
|
||||
server_status,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn runtime_handle(&self) -> tokio::runtime::Handle {
|
||||
return self.handle.clone();
|
||||
}
|
||||
|
||||
pub fn server_status(&self) -> crate::health::ServerStatus {
|
||||
self.server_status.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Origin for NetworkOrigin {
|
||||
fn snapshot(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
|
||||
return run_db_blocking(self.snapshot_async()).map_err(io_err);
|
||||
}
|
||||
|
||||
fn byte_source(&self) -> Arc<dyn ByteSource> {
|
||||
return Arc::new(NetworkByteSource {
|
||||
transport: self.transport.clone(),
|
||||
runtime_handle: self.runtime_handle(),
|
||||
client: self.client.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
fn watcher(&self) -> Box<dyn FileWatcher> {
|
||||
return Box::new(watcher::NetworkOriginFileWatcher::new(
|
||||
self.transport.clone(),
|
||||
self.runtime_handle(),
|
||||
self.client.clone(),
|
||||
self.destination.clone(),
|
||||
self.latest_manifest.clone(),
|
||||
self.server_status.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
impl NetworkOrigin {
|
||||
/// Async snapshot driven directly on the caller's runtime. `main` (already
|
||||
/// `#[tokio::main]`) awaits this; the sync `Origin::snapshot()` wrapper is
|
||||
/// only for non-async callers and must not be invoked from within a runtime
|
||||
/// (it builds and `block_on`s a throwaway one).
|
||||
pub async fn snapshot_async(&self) -> io::Result<BTreeMap<INodeNo, Item>> {
|
||||
// 1. Pull client's current (inode, hash) pairs from the DB.
|
||||
let client_entries: Vec<(u64, u64)> = item_entities::Entity::find()
|
||||
.all(&self.client)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "network snapshot: client entries DB read failed");
|
||||
io_err(e)
|
||||
})?
|
||||
.into_iter()
|
||||
.filter(|m| m.file_type == "file")
|
||||
.map(|m| (m.inode as u64, m.hash as u64))
|
||||
.collect();
|
||||
|
||||
// 2. Ask server what changed.
|
||||
let response = self
|
||||
.transport
|
||||
.reconcile(client_entries)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "network snapshot: reconcile with server failed");
|
||||
io_err(e)
|
||||
})?;
|
||||
|
||||
// 3. Invalidate cached bytes for changed + deleted inodes — they are
|
||||
// stale by definition.
|
||||
let mut changed_or_deleted: Vec<i64> =
|
||||
response.changed.iter().map(|ih| ih.inode as i64).collect();
|
||||
changed_or_deleted.extend(response.deleted.iter().map(|i| *i as i64));
|
||||
delete_cached_bytes_for(&changed_or_deleted, &self.client).await;
|
||||
|
||||
// latest_manifest is in-memory; on restart it's empty so the reconcile
|
||||
// delta misses unchanged files. Fetch full manifest then, delta otherwise.
|
||||
let mut current_manifest = self.latest_manifest.read().unwrap().clone();
|
||||
if current_manifest.is_empty() {
|
||||
let entries = self.transport.get_manifest().await.map_err(|e| {
|
||||
error!(error = %e, "network snapshot: get_manifest from server failed");
|
||||
io_err(e)
|
||||
})?;
|
||||
current_manifest = entries.into_iter().map(|e| (e.id, e)).collect();
|
||||
} else {
|
||||
let wanted: Vec<u64> = response.changed.iter().map(|ih| ih.inode).collect();
|
||||
if !wanted.is_empty() {
|
||||
let wanted_count = wanted.len();
|
||||
let entries = self.transport.get_metadata(wanted).await.map_err(|e| {
|
||||
error!(
|
||||
wanted = wanted_count,
|
||||
error = %e,
|
||||
"network snapshot: get_metadata from server failed"
|
||||
);
|
||||
io_err(e)
|
||||
})?;
|
||||
for entry in entries {
|
||||
current_manifest.insert(entry.id, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
for inode in &response.deleted {
|
||||
current_manifest.remove(inode);
|
||||
}
|
||||
*self.latest_manifest.write().unwrap() = current_manifest.clone();
|
||||
|
||||
let snapshot =
|
||||
build_snapshot_from_manifest(¤t_manifest, &self.destination, &self.client)
|
||||
.await?;
|
||||
|
||||
info!(
|
||||
changed = response.changed.len(),
|
||||
deleted = response.deleted.len(),
|
||||
files = snapshot.len(),
|
||||
"network snapshot complete"
|
||||
);
|
||||
return Ok(snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a complete Items snapshot from the server manifest, sync it to the
|
||||
/// DB, and restore music metadata + virtual paths from existing DB rows.
|
||||
///
|
||||
/// Shared between `snapshot_async` (initial mount) and the watcher's
|
||||
/// `reconcile_once` (runtime updates) so both paths produce identical
|
||||
/// snapshots and keep the DB in sync.
|
||||
pub(crate) async fn build_snapshot_from_manifest(
|
||||
manifest: &BTreeMap<u64, ProtoManifestEntry>,
|
||||
destination: &Path,
|
||||
client: &sea_orm::DatabaseConnection,
|
||||
) -> io::Result<BTreeMap<INodeNo, Item>> {
|
||||
let source_root = PathBuf::from("/");
|
||||
let mut snapshot = BTreeMap::new();
|
||||
let root_attrs = FileAttrs {
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
perm: 0o755,
|
||||
nlink: 2,
|
||||
uid: 0,
|
||||
gid: 0,
|
||||
rdev: 0,
|
||||
blksize: 4096,
|
||||
};
|
||||
snapshot.insert(
|
||||
INodeNo::ROOT,
|
||||
Item::new(
|
||||
INodeNo::ROOT,
|
||||
INodeNo::ROOT,
|
||||
"/".to_string(),
|
||||
destination.to_path_buf(),
|
||||
destination.to_path_buf(),
|
||||
FileType::Directory,
|
||||
root_attrs,
|
||||
None,
|
||||
),
|
||||
);
|
||||
|
||||
for (_id, entry) in manifest {
|
||||
let item = manifest_entry_to_item(entry, &source_root, &mut snapshot);
|
||||
snapshot.insert(item.inode, item);
|
||||
}
|
||||
|
||||
let db_items: std::collections::HashMap<i64, item_entities::Model> =
|
||||
item_entities::Entity::find()
|
||||
.all(client)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "build_snapshot_from_manifest: DB read failed");
|
||||
io_err(e)
|
||||
})?
|
||||
.into_iter()
|
||||
.map(|e| (e.inode, e))
|
||||
.collect();
|
||||
sync_items_to_db(&snapshot, &db_items, client).await;
|
||||
restore_music_metadata_from_db(&mut snapshot, &db_items, client).await;
|
||||
restore_virtual_paths(&mut snapshot, &db_items, &source_root);
|
||||
|
||||
return Ok(snapshot);
|
||||
}
|
||||
|
||||
fn manifest_entry_to_item(
|
||||
entry: &ProtoManifestEntry,
|
||||
source_root: &Path,
|
||||
snapshot: &mut BTreeMap<INodeNo, Item>,
|
||||
) -> Item {
|
||||
let inode = INodeNo(entry.id);
|
||||
let original_path = PathBuf::from(&entry.rel_path);
|
||||
let attrs = FileAttrs {
|
||||
size: entry.size,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.mtime),
|
||||
mtime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.mtime),
|
||||
ctime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.ctime),
|
||||
crtime: SystemTime::UNIX_EPOCH + Duration::from_secs(entry.crtime),
|
||||
perm: 0o644,
|
||||
nlink: 1,
|
||||
uid: 0,
|
||||
gid: 0,
|
||||
rdev: 0,
|
||||
blksize: 4096,
|
||||
};
|
||||
let music_metadata: Option<MusicMetadata> =
|
||||
entry.music_metadata.clone().map(music_metadata_from_proto);
|
||||
|
||||
let mut local_path = PathBuf::new();
|
||||
if let Some(mm) = &music_metadata {
|
||||
let joined;
|
||||
let artist_dir = match mm.album_artist.as_deref() {
|
||||
Some(a) => a,
|
||||
None => {
|
||||
joined = mm.artist.join("-");
|
||||
&joined
|
||||
}
|
||||
};
|
||||
local_path.push(artist_dir);
|
||||
local_path.push(&mm.album);
|
||||
}
|
||||
let name = Path::new(&entry.rel_path)
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| entry.rel_path.clone());
|
||||
local_path.push(&name);
|
||||
|
||||
let parent_inode = ensure_virtual_dirs(&local_path, source_root, snapshot);
|
||||
return Item::new(
|
||||
inode,
|
||||
parent_inode,
|
||||
name,
|
||||
original_path,
|
||||
local_path,
|
||||
FileType::File,
|
||||
attrs,
|
||||
music_metadata,
|
||||
);
|
||||
}
|
||||
|
||||
fn music_metadata_from_proto(mm: crate::proto::MusicMetadata) -> MusicMetadata {
|
||||
MusicMetadata {
|
||||
artist: mm.artist,
|
||||
album_artist: mm.album_artist,
|
||||
album: mm.album,
|
||||
track_number: mm.track_number,
|
||||
track_title: mm.track_title,
|
||||
other_tags: mm.other_tags,
|
||||
header: mm.header,
|
||||
picture_block_headers: mm.picture_block_headers,
|
||||
picture_data_ranges: mm
|
||||
.picture_data_ranges
|
||||
.into_iter()
|
||||
.map(|p| (p.offset, p.length))
|
||||
.collect(),
|
||||
real_audio_start: mm.real_audio_start,
|
||||
vorbis_comment_offset: mm.vorbis_comment_offset,
|
||||
vorbis_comment_length: mm.vorbis_comment_length,
|
||||
}
|
||||
}
|
||||
|
||||
fn io_err<E: std::fmt::Display>(e: E) -> io::Error {
|
||||
return io::Error::new(io::ErrorKind::Other, e.to_string());
|
||||
}
|
||||
|
||||
pub struct NetworkByteSource {
|
||||
transport: NetworkTransport,
|
||||
runtime_handle: tokio::runtime::Handle,
|
||||
client: sea_orm::DatabaseConnection,
|
||||
}
|
||||
|
||||
impl ByteSource for NetworkByteSource {
|
||||
fn read_at(
|
||||
&self,
|
||||
inode: INodeNo,
|
||||
_locator: &Path,
|
||||
offset: u64,
|
||||
len: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let inode_i64 = inode.0 as i64;
|
||||
trace!(%inode, offset, len, "network read_at");
|
||||
|
||||
let cached = self
|
||||
.runtime_handle
|
||||
.block_on(get_cached_bytes(inode_i64, &self.client));
|
||||
if let Some(data) = cached {
|
||||
debug!(%inode, "read_at cache hit");
|
||||
return slice_range(&data, offset, len);
|
||||
}
|
||||
|
||||
debug!(%inode, "read_at cache miss; fetching from server");
|
||||
let (data, _total_size) = self
|
||||
.runtime_handle
|
||||
.block_on(self.transport.fetch_file_range(inode.0, 0, 0))
|
||||
.map_err(|e| {
|
||||
error!(%inode, error = %e, "read_at: fetch_file_range failed");
|
||||
io_err(e)
|
||||
})?;
|
||||
self.runtime_handle
|
||||
.block_on(put_cached_bytes(inode_i64, data.clone(), &self.client));
|
||||
|
||||
return slice_range(&data, offset, len);
|
||||
}
|
||||
}
|
||||
|
||||
fn slice_range(data: &[u8], offset: u64, len: usize) -> io::Result<Vec<u8>> {
|
||||
let start = (offset as usize).min(data.len());
|
||||
let end = (start + len).min(data.len());
|
||||
return Ok(data[start..end].to_vec());
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use tokio_stream::StreamExt;
|
||||
use tonic::codec::Streaming;
|
||||
|
||||
use crate::proto::{
|
||||
ChangeEvent, GetFileRequest, GetManifestRequest, GetMetadataRequest, InodeHash, ManifestEntry,
|
||||
MusicFsClient, ReconcileRequest, ReconcileResponse, SubscribeEventsRequest,
|
||||
};
|
||||
|
||||
/// Thin wrapper over the generated tonic client. Owns the connection and
|
||||
/// exposes four operations matching the four RPCs NetworkOrigin needs.
|
||||
///
|
||||
/// All methods are async and must be polled on the runtime whose Handle was
|
||||
/// passed in at construction (or a child of it). The sync FUSE path uses
|
||||
/// `runtime_handle.block_on(...)` to enter this runtime.
|
||||
#[derive(Clone)]
|
||||
pub struct NetworkTransport {
|
||||
client: Arc<tokio::sync::Mutex<MusicFsClient<tonic::transport::Channel>>>,
|
||||
endpoint: tonic::transport::Endpoint,
|
||||
}
|
||||
|
||||
impl NetworkTransport {
|
||||
/// Build a transport that connects lazily. The first RPC establishes the
|
||||
/// connection; reconnects happen automatically if the channel drops.
|
||||
pub fn new(url: String) -> Result<Self> {
|
||||
let endpoint: tonic::transport::Endpoint = url
|
||||
.try_into()
|
||||
.map_err(|e| anyhow!("invalid server URL: {e}"))?;
|
||||
let endpoint = endpoint
|
||||
.timeout(Duration::from_secs(3))
|
||||
.connect_timeout(Duration::from_secs(5))
|
||||
.http2_keep_alive_interval(Duration::from_secs(10))
|
||||
.keep_alive_timeout(Duration::from_secs(5));
|
||||
let channel = endpoint.connect_lazy();
|
||||
let client = Arc::new(tokio::sync::Mutex::new(MusicFsClient::new(channel)));
|
||||
return Ok(NetworkTransport { client, endpoint });
|
||||
}
|
||||
|
||||
pub async fn reconcile(&self, entries: Vec<(u64, u64)>) -> Result<ReconcileResponse> {
|
||||
let request = ReconcileRequest {
|
||||
entries: entries
|
||||
.into_iter()
|
||||
.map(|(inode, hash)| InodeHash { inode, hash })
|
||||
.collect(),
|
||||
};
|
||||
let mut client = self.client.lock().await;
|
||||
let response = client
|
||||
.reconcile(request)
|
||||
.await
|
||||
.context("Reconcile RPC failed")?;
|
||||
return Ok(response.into_inner());
|
||||
}
|
||||
|
||||
pub async fn get_metadata(&self, inodes: Vec<u64>) -> Result<Vec<ManifestEntry>> {
|
||||
let request = GetMetadataRequest { inodes };
|
||||
let mut client = self.client.lock().await;
|
||||
let mut stream: Streaming<ManifestEntry> = client
|
||||
.get_metadata(request)
|
||||
.await
|
||||
.context("GetMetadata RPC failed")?
|
||||
.into_inner();
|
||||
let mut out = Vec::new();
|
||||
while let Some(entry) = stream.next().await {
|
||||
out.push(entry.context("GetMetadata stream error")?);
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
pub async fn get_manifest(&self) -> Result<Vec<ManifestEntry>> {
|
||||
let mut client = self.client.lock().await;
|
||||
let mut stream: Streaming<ManifestEntry> = client
|
||||
.get_manifest(GetManifestRequest {})
|
||||
.await
|
||||
.context("GetManifest RPC failed")?
|
||||
.into_inner();
|
||||
let mut out = Vec::new();
|
||||
while let Some(entry) = stream.next().await {
|
||||
out.push(entry.context("GetManifest stream error")?);
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
pub async fn fetch_file_range(
|
||||
&self,
|
||||
id: u64,
|
||||
start: u64,
|
||||
length: u64,
|
||||
) -> Result<(Vec<u8>, u64)> {
|
||||
let request = GetFileRequest { id, start, length };
|
||||
let mut client = self.client.lock().await;
|
||||
let mut stream = client
|
||||
.get_file(request)
|
||||
.await
|
||||
.context("GetFile RPC failed")?
|
||||
.into_inner();
|
||||
let first = stream
|
||||
.next()
|
||||
.await
|
||||
.ok_or_else(|| anyhow!("GetFile returned empty stream for id {id}"))?
|
||||
.context("GetFile stream error")?;
|
||||
let total_size = first.total_size;
|
||||
let mut data = first.data;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("GetFile stream error")?;
|
||||
data.extend_from_slice(&chunk.data);
|
||||
}
|
||||
return Ok((data, total_size));
|
||||
}
|
||||
|
||||
/// Take a fresh receiver on the SubscribeEvents stream. Each call opens a
|
||||
/// new server-streaming RPC; the caller owns the lifetime.
|
||||
pub async fn subscribe_events(&self) -> Result<Streaming<ChangeEvent>> {
|
||||
let mut client = self.client.lock().await;
|
||||
let stream = client
|
||||
.subscribe_events(SubscribeEventsRequest {})
|
||||
.await
|
||||
.context("SubscribeEvents RPC failed")?
|
||||
.into_inner();
|
||||
return Ok(stream);
|
||||
}
|
||||
|
||||
/// Reconnect — used by the watcher when the channel has gone bad.
|
||||
#[allow(dead_code)]
|
||||
pub async fn reconnect(&self) -> Result<()> {
|
||||
let channel = self.endpoint.connect().await.context("reconnect failed")?;
|
||||
let new_client = MusicFsClient::new(channel);
|
||||
let mut guard = self.client.lock().await;
|
||||
*guard = new_client;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
path::PathBuf,
|
||||
sync::{Arc, RwLock},
|
||||
thread,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use sea_orm::{DatabaseConnection, EntityTrait};
|
||||
use tokio::sync::Notify;
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
use crate::item::Item;
|
||||
use crate::origins::network::transport::NetworkTransport;
|
||||
use crate::origins::{FileWatcher, WatcherHandle};
|
||||
use crate::proto::ManifestEntry as ProtoManifestEntry;
|
||||
use tracing::{info, warn};
|
||||
|
||||
const INITIAL_RETRY_DELAY: Duration = Duration::from_secs(1);
|
||||
const MAX_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
pub struct NetworkOriginFileWatcher {
|
||||
transport: NetworkTransport,
|
||||
runtime_handle: tokio::runtime::Handle,
|
||||
client: DatabaseConnection,
|
||||
destination: PathBuf,
|
||||
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
|
||||
server_status: crate::health::ServerStatus,
|
||||
}
|
||||
|
||||
impl NetworkOriginFileWatcher {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
transport: NetworkTransport,
|
||||
runtime_handle: tokio::runtime::Handle,
|
||||
client: DatabaseConnection,
|
||||
destination: PathBuf,
|
||||
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
|
||||
server_status: crate::health::ServerStatus,
|
||||
) -> Self {
|
||||
return NetworkOriginFileWatcher {
|
||||
transport,
|
||||
runtime_handle,
|
||||
client,
|
||||
destination,
|
||||
latest_manifest,
|
||||
server_status,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
impl FileWatcher for NetworkOriginFileWatcher {
|
||||
fn watch(&self, files: Arc<std::sync::Mutex<BTreeMap<INodeNo, Item>>>) -> WatcherHandle {
|
||||
let runtime_handle = self.runtime_handle.clone();
|
||||
let shutdown = Arc::new(Notify::new());
|
||||
let state = WatcherState {
|
||||
transport: self.transport.clone(),
|
||||
client: self.client.clone(),
|
||||
destination: self.destination.clone(),
|
||||
latest_manifest: self.latest_manifest.clone(),
|
||||
server_status: self.server_status.clone(),
|
||||
files,
|
||||
};
|
||||
let loop_shutdown = shutdown.clone();
|
||||
let join = thread::spawn(move || {
|
||||
runtime_handle.block_on(state.run_loop(loop_shutdown));
|
||||
});
|
||||
return WatcherHandle::new(move || {
|
||||
shutdown.notify_one();
|
||||
let _ = join.join();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
struct WatcherState {
|
||||
transport: NetworkTransport,
|
||||
client: DatabaseConnection,
|
||||
destination: PathBuf,
|
||||
latest_manifest: Arc<RwLock<BTreeMap<u64, ProtoManifestEntry>>>,
|
||||
server_status: crate::health::ServerStatus,
|
||||
files: Arc<std::sync::Mutex<BTreeMap<INodeNo, Item>>>,
|
||||
}
|
||||
|
||||
impl WatcherState {
|
||||
async fn run_loop(self, shutdown: Arc<Notify>) {
|
||||
let mut retry_delay = INITIAL_RETRY_DELAY;
|
||||
loop {
|
||||
let subscribed = tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.notified() => return,
|
||||
s = self.transport.subscribe_events() => s,
|
||||
};
|
||||
match subscribed {
|
||||
Ok(mut stream) => {
|
||||
info!("network watcher: subscribed to /events");
|
||||
self.server_status.set_connected(true);
|
||||
retry_delay = INITIAL_RETRY_DELAY;
|
||||
loop {
|
||||
let item = tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.notified() => return,
|
||||
item = stream.next() => item,
|
||||
};
|
||||
match item {
|
||||
Some(Ok(_event)) => {
|
||||
if let Err(e) = self.reconcile_once().await {
|
||||
warn!(error = %e, "network watcher: reconcile after event failed");
|
||||
}
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
warn!(error = %e, "network watcher: stream error; reconnecting");
|
||||
self.server_status.set_connected(false);
|
||||
break;
|
||||
}
|
||||
None => {
|
||||
self.server_status.set_connected(false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
self.server_status.set_connected(false);
|
||||
warn!(error = %e, "network watcher: subscribe failed; will retry");
|
||||
}
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.notified() => return,
|
||||
_ = tokio::time::sleep(retry_delay) => {}
|
||||
}
|
||||
retry_delay = (retry_delay * 2).min(MAX_RETRY_DELAY);
|
||||
if let Err(e) = self.reconcile_once().await {
|
||||
warn!(error = %e, "network watcher: poll reconcile failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn reconcile_once(&self) -> anyhow::Result<()> {
|
||||
let client_entries: Vec<(u64, u64)> = crate::db::entities::Entity::find()
|
||||
.all(&self.client)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|m| m.file_type == "file")
|
||||
.map(|m| (m.inode as u64, m.hash as u64))
|
||||
.collect();
|
||||
|
||||
let response = self.transport.reconcile(client_entries).await?;
|
||||
|
||||
let mut changed_or_deleted: Vec<i64> =
|
||||
response.changed.iter().map(|ih| ih.inode as i64).collect();
|
||||
changed_or_deleted.extend(response.deleted.iter().map(|i| *i as i64));
|
||||
crate::db::cache::delete_cached_bytes_for(&changed_or_deleted, &self.client).await;
|
||||
|
||||
let wanted: Vec<u64> = response.changed.iter().map(|ih| ih.inode).collect();
|
||||
let mut current_manifest = self.latest_manifest.read().unwrap().clone();
|
||||
if !wanted.is_empty() {
|
||||
let entries = self.transport.get_metadata(wanted).await?;
|
||||
for entry in entries {
|
||||
current_manifest.insert(entry.id, entry);
|
||||
}
|
||||
}
|
||||
for inode in &response.deleted {
|
||||
current_manifest.remove(inode);
|
||||
}
|
||||
*self.latest_manifest.write().unwrap() = current_manifest.clone();
|
||||
|
||||
let new_snapshot =
|
||||
super::build_snapshot_from_manifest(¤t_manifest, &self.destination, &self.client)
|
||||
.await?;
|
||||
|
||||
{
|
||||
let mut files = self.files.lock().unwrap();
|
||||
files.retain(|ino, _| new_snapshot.contains_key(ino));
|
||||
for (ino, new_item) in &new_snapshot {
|
||||
match files.get(ino) {
|
||||
Some(existing) if existing.hash == new_item.hash => {}
|
||||
_ => {
|
||||
files.insert(*ino, new_item.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.server_status.mark_reconciled();
|
||||
|
||||
info!(
|
||||
changed = response.changed.len(),
|
||||
deleted = response.deleted.len(),
|
||||
total = current_manifest.len(),
|
||||
"network watcher: reconcile applied"
|
||||
);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,531 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs,
|
||||
hash::Hasher,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use twox_hash::XxHash64;
|
||||
|
||||
use crate::db::entities::Model;
|
||||
use crate::item::{FileType, Item};
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::origins::attrs::FileAttrs;
|
||||
|
||||
pub fn virtual_inode(path: &str) -> INodeNo {
|
||||
let mut hasher = XxHash64::with_seed(5678);
|
||||
hasher.write(path.as_bytes());
|
||||
|
||||
return INodeNo(hasher.finish() | (1u64 << 63));
|
||||
}
|
||||
|
||||
pub fn parent_inode_from_path(local_path: &Path) -> INodeNo {
|
||||
let components: Vec<_> = local_path
|
||||
.components()
|
||||
.filter(|c| matches!(c, std::path::Component::Normal(_)))
|
||||
.collect();
|
||||
if components.len() <= 1 {
|
||||
return INodeNo::ROOT;
|
||||
}
|
||||
let mut parent_path = String::new();
|
||||
for (i, comp) in components[..components.len() - 1].iter().enumerate() {
|
||||
if i > 0 {
|
||||
parent_path.push('/');
|
||||
}
|
||||
parent_path.push_str(comp.as_os_str().to_str().unwrap_or_default());
|
||||
}
|
||||
|
||||
return virtual_inode(&parent_path);
|
||||
}
|
||||
|
||||
pub fn ensure_virtual_dirs(
|
||||
local_path: &Path,
|
||||
source: &Path,
|
||||
map: &mut BTreeMap<INodeNo, Item>,
|
||||
) -> INodeNo {
|
||||
let components: Vec<_> = local_path
|
||||
.components()
|
||||
.filter(|c| matches!(c, std::path::Component::Normal(_)))
|
||||
.collect();
|
||||
|
||||
if components.len() <= 1 {
|
||||
return INodeNo::ROOT;
|
||||
}
|
||||
|
||||
let mut current_parent = INodeNo::ROOT;
|
||||
let mut current_path = String::new();
|
||||
|
||||
for component in &components[..components.len() - 1] {
|
||||
let comp_str = component.as_os_str().to_str().unwrap_or_default();
|
||||
if !current_path.is_empty() {
|
||||
current_path.push('/');
|
||||
}
|
||||
current_path.push_str(comp_str);
|
||||
|
||||
let virt_ino = virtual_inode(¤t_path);
|
||||
|
||||
if !map.contains_key(&virt_ino) {
|
||||
let virt_item = Item::new(
|
||||
virt_ino,
|
||||
current_parent,
|
||||
comp_str.to_string(),
|
||||
source.to_path_buf(),
|
||||
PathBuf::from(¤t_path),
|
||||
FileType::Directory,
|
||||
FileAttrs::from(&fs::metadata(source).unwrap()),
|
||||
None,
|
||||
);
|
||||
map.insert(virt_ino, virt_item);
|
||||
}
|
||||
|
||||
current_parent = virt_ino;
|
||||
}
|
||||
|
||||
return current_parent;
|
||||
}
|
||||
|
||||
pub fn restore_virtual_paths(
|
||||
snapshot: &mut BTreeMap<INodeNo, Item>,
|
||||
db_items: &std::collections::HashMap<i64, Model>,
|
||||
source: &Path,
|
||||
) {
|
||||
let restorations: Vec<(INodeNo, String, PathBuf)> = db_items
|
||||
.values()
|
||||
.filter_map(|db_item| {
|
||||
let ino = INodeNo(db_item.inode as u64);
|
||||
if ino == INodeNo::ROOT {
|
||||
return None;
|
||||
}
|
||||
snapshot
|
||||
.get(&ino)
|
||||
.filter(|item| item.hash as i64 == db_item.hash)
|
||||
.map(|_| {
|
||||
(
|
||||
ino,
|
||||
db_item.name.clone(),
|
||||
PathBuf::from(&db_item.local_path),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (_, _, local_path) in &restorations {
|
||||
ensure_virtual_dirs(local_path, source, snapshot);
|
||||
}
|
||||
for (ino, name, local_path) in restorations {
|
||||
if let Some(item) = snapshot.get_mut(&ino) {
|
||||
item.name = name;
|
||||
item.parent_inode = parent_inode_from_path(&local_path);
|
||||
item.local_path = local_path;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes the new (name, local_path) for an Item based on its updated
|
||||
/// `MusicMetadata`. Mirrors the layout logic in
|
||||
/// `origins::local::snapshot::read_into_map`:
|
||||
/// - artist dir = `album_artist` if set, else `artists.join("-")`
|
||||
/// - if `track_title` is non-empty, the filename becomes
|
||||
/// `{track_number:02} - {track_title}.{ext}` (zero-padded track number
|
||||
/// when > 0, otherwise just `{track_title}.{ext}`), ext preserved from
|
||||
/// `current_name`
|
||||
/// - otherwise the filename is unchanged
|
||||
/// Returns `(new_name, new_local_path)`.
|
||||
pub fn compute_new_layout(current_name: &str, mm: &MusicMetadata) -> (String, PathBuf) {
|
||||
let artist_dir = artist_dir(mm);
|
||||
let filename = match &mm.track_title {
|
||||
title if !title.is_empty() => {
|
||||
let stem = if mm.track_number > 0 {
|
||||
format!("{:02} - {}", mm.track_number, title)
|
||||
} else {
|
||||
title.clone()
|
||||
};
|
||||
rename_with_extension(current_name, &stem)
|
||||
}
|
||||
_ => current_name.to_string(),
|
||||
};
|
||||
|
||||
let mut local_path = PathBuf::new();
|
||||
local_path.push(&artist_dir);
|
||||
local_path.push(&mm.album);
|
||||
local_path.push(&filename);
|
||||
|
||||
(filename, local_path)
|
||||
}
|
||||
|
||||
fn artist_dir(mm: &MusicMetadata) -> String {
|
||||
match &mm.album_artist {
|
||||
Some(a) if !a.is_empty() => a.clone(),
|
||||
_ => mm.artist.join("-"),
|
||||
}
|
||||
}
|
||||
|
||||
fn rename_with_extension(current_name: &str, new_stem: &str) -> String {
|
||||
let current_path = Path::new(current_name);
|
||||
match current_path.extension() {
|
||||
Some(ext) => format!("{new_stem}.{}", ext.to_string_lossy()),
|
||||
None => new_stem.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the virtual inodes of directories in the `old_local_path`'s
|
||||
/// parent chain that will have no remaining files after this item moves
|
||||
/// away. Walks the chain from the deepest directory upward; stops at the
|
||||
/// first directory that still has other files living in it.
|
||||
///
|
||||
/// `moving_inode` is the actual inode of the file being relocated (the
|
||||
/// request's `inode`), used to exclude it from the "any other files here?"
|
||||
/// check. `old_local_path` is the LOCAL (virtual FUSE) path of the file
|
||||
/// before its metadata changed.
|
||||
pub fn find_orphaned_dirs(
|
||||
files: &BTreeMap<INodeNo, Item>,
|
||||
moving_inode: INodeNo,
|
||||
old_local_path: &Path,
|
||||
) -> Vec<INodeNo> {
|
||||
let components: Vec<_> = old_local_path
|
||||
.components()
|
||||
.filter(|c| matches!(c, std::path::Component::Normal(_)))
|
||||
.collect();
|
||||
if components.len() <= 1 {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
let mut orphaned = vec![];
|
||||
for depth in (1..components.len()).rev() {
|
||||
let dir_path: PathBuf = components[..depth].iter().map(|c| c.as_os_str()).collect();
|
||||
let dir_path_str = dir_path.to_string_lossy().into_owned();
|
||||
let dir_inode = virtual_inode(&dir_path_str);
|
||||
|
||||
// Path-prefix match: a file at "Artist/Album2/b.flac" still keeps
|
||||
// "Artist" non-orphan even though its direct parent is "Artist/Album2".
|
||||
let prefix = format!("{dir_path_str}/");
|
||||
let has_other_files = files.values().any(|item| {
|
||||
item.inode != moving_inode
|
||||
&& item.file_type == FileType::File
|
||||
&& item.local_path.starts_with(&prefix)
|
||||
});
|
||||
if has_other_files {
|
||||
break;
|
||||
}
|
||||
orphaned.push(dir_inode);
|
||||
}
|
||||
orphaned
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
// ---- compute_new_layout ----
|
||||
|
||||
fn mm(artist: &str, album: &str, title: &str) -> MusicMetadata {
|
||||
MusicMetadata {
|
||||
artist: vec![artist.to_string()],
|
||||
album_artist: None,
|
||||
album: album.to_string(),
|
||||
track_number: 1,
|
||||
track_title: title.to_string(),
|
||||
other_tags: vec![],
|
||||
header: vec![],
|
||||
picture_block_headers: vec![],
|
||||
picture_data_ranges: vec![],
|
||||
real_audio_start: 0,
|
||||
vorbis_comment_offset: 0,
|
||||
vorbis_comment_length: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn file_item(inode: u64, name: &str, local_path: &str) -> Item {
|
||||
Item {
|
||||
inode: INodeNo(inode),
|
||||
parent_inode: parent_inode_from_path(Path::new(local_path)),
|
||||
name: name.to_string(),
|
||||
original_path: format!("/torrent/{name}").into(),
|
||||
local_path: local_path.into(),
|
||||
file_type: FileType::File,
|
||||
attrs: FileAttrs {
|
||||
size: 1,
|
||||
blocks: 1,
|
||||
atime: std::time::SystemTime::UNIX_EPOCH,
|
||||
mtime: std::time::SystemTime::UNIX_EPOCH,
|
||||
ctime: std::time::SystemTime::UNIX_EPOCH,
|
||||
crtime: std::time::SystemTime::UNIX_EPOCH,
|
||||
perm: 0o644,
|
||||
nlink: 1,
|
||||
uid: 0,
|
||||
gid: 0,
|
||||
rdev: 0,
|
||||
blksize: 4096,
|
||||
},
|
||||
music_metadata: None,
|
||||
hash: inode,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_new_layout_renames_file_when_track_title_provided() {
|
||||
let mm = mm(
|
||||
"Pink Floyd",
|
||||
"Wish You Were Here",
|
||||
"Shine On You Crazy Diamond",
|
||||
);
|
||||
let (name, path) = compute_new_layout("01-track.flac", &mm);
|
||||
assert_eq!(name, "01 - Shine On You Crazy Diamond.flac");
|
||||
assert_eq!(
|
||||
path,
|
||||
PathBuf::from("Pink Floyd/Wish You Were Here/01 - Shine On You Crazy Diamond.flac")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_new_layout_preserves_extension_through_rename() {
|
||||
let mm = mm("A", "B", "New Title");
|
||||
let (name, _) = compute_new_layout("old.mp3", &mm);
|
||||
assert_eq!(name, "01 - New Title.mp3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_new_layout_handles_multi_dot_extensions() {
|
||||
let mm = mm("A", "B", "New");
|
||||
let (name, _) = compute_new_layout("old.tar.gz", &mm);
|
||||
assert_eq!(name, "01 - New.gz", "only the final extension is preserved");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_new_layout_keeps_original_name_when_track_title_empty() {
|
||||
let mut metadata = mm("A", "B", "ignored");
|
||||
metadata.track_title = String::new();
|
||||
let (name, path) = compute_new_layout("original-name.flac", &metadata);
|
||||
assert_eq!(name, "original-name.flac");
|
||||
assert_eq!(path, PathBuf::from("A/B/original-name.flac"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_new_layout_uses_album_artist_when_present() {
|
||||
let mut metadata = mm("Secondary", "Album", "Title");
|
||||
metadata.album_artist = Some("Primary Artist".to_string());
|
||||
let (_, path) = compute_new_layout("track.flac", &metadata);
|
||||
assert_eq!(path, PathBuf::from("Primary Artist/Album/01 - Title.flac"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_new_layout_joins_multiple_artists_with_dash_when_no_album_artist() {
|
||||
let mut metadata = mm("Foo", "Album", "Title");
|
||||
metadata.artist = vec!["Foo".to_string(), "Bar".to_string()];
|
||||
let (_, path) = compute_new_layout("track.flac", &metadata);
|
||||
assert_eq!(path, PathBuf::from("Foo-Bar/Album/01 - Title.flac"));
|
||||
}
|
||||
|
||||
// ---- find_orphaned_dirs ----
|
||||
|
||||
#[test]
|
||||
fn find_orphaned_dirs_returns_both_album_and_artist_when_last_file_moves() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
file_item(100, "track.flac", "Artist/Album/track.flac"),
|
||||
);
|
||||
|
||||
let orphaned =
|
||||
find_orphaned_dirs(&files, INodeNo(100), Path::new("Artist/Album/track.flac"));
|
||||
assert_eq!(orphaned.len(), 2);
|
||||
assert!(orphaned.contains(&virtual_inode("Artist/Album")));
|
||||
assert!(orphaned.contains(&virtual_inode("Artist")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_orphaned_dirs_returns_album_only_when_other_album_keeps_artist() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
file_item(100, "a.flac", "Artist/Album1/a.flac"),
|
||||
);
|
||||
files.insert(
|
||||
INodeNo(101),
|
||||
file_item(101, "b.flac", "Artist/Album2/b.flac"),
|
||||
);
|
||||
|
||||
let orphaned = find_orphaned_dirs(&files, INodeNo(100), Path::new("Artist/Album1/a.flac"));
|
||||
assert_eq!(orphaned, vec![virtual_inode("Artist/Album1")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_orphaned_dirs_returns_nothing_when_other_files_remain_in_same_album() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
file_item(100, "a.flac", "Artist/Album/a.flac"),
|
||||
);
|
||||
files.insert(
|
||||
INodeNo(101),
|
||||
file_item(101, "b.flac", "Artist/Album/b.flac"),
|
||||
);
|
||||
|
||||
let orphaned = find_orphaned_dirs(&files, INodeNo(100), Path::new("Artist/Album/a.flac"));
|
||||
assert!(orphaned.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_orphaned_dirs_returns_nothing_for_file_at_root() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(INodeNo(100), file_item(100, "track.flac", "track.flac"));
|
||||
|
||||
let orphaned = find_orphaned_dirs(&files, INodeNo(100), Path::new("track.flac"));
|
||||
assert!(orphaned.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_orphaned_dirs_excludes_moving_file_from_other_file_check() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
file_item(100, "only.flac", "Artist/Album/only.flac"),
|
||||
);
|
||||
// If we forgot to exclude the moving inode, this test would return
|
||||
// empty (thinking the dir is still populated). Must return both dirs.
|
||||
let orphaned =
|
||||
find_orphaned_dirs(&files, INodeNo(100), Path::new("Artist/Album/only.flac"));
|
||||
assert_eq!(orphaned.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn virtual_inode_deterministic() {
|
||||
let ino1 = virtual_inode("foo");
|
||||
let ino2 = virtual_inode("foo");
|
||||
assert_eq!(ino1, ino2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn virtual_inode_high_bit_set() {
|
||||
let ino = virtual_inode("test");
|
||||
assert_eq!(ino.0 & (1u64 << 63), 1u64 << 63);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn virtual_inode_different_inputs() {
|
||||
let ino1 = virtual_inode("foo");
|
||||
let ino2 = virtual_inode("bar");
|
||||
assert_ne!(ino1, ino2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parent_inode_single_component() {
|
||||
let ino = parent_inode_from_path(Path::new("file.txt"));
|
||||
assert_eq!(ino, INodeNo::ROOT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parent_inode_multi_component() {
|
||||
let ino = parent_inode_from_path(Path::new("a/b/c"));
|
||||
let expected = virtual_inode("a/b");
|
||||
assert_eq!(ino, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parent_inode_absolute_path_filtered() {
|
||||
let ino = parent_inode_from_path(Path::new("/file"));
|
||||
assert_eq!(ino, INodeNo::ROOT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_virtual_dirs_single_component() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let source = tmp.path();
|
||||
let mut map = BTreeMap::new();
|
||||
|
||||
let ino = ensure_virtual_dirs(Path::new("file.txt"), source, &mut map);
|
||||
assert_eq!(ino, INodeNo::ROOT);
|
||||
assert!(map.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_virtual_dirs_creates_hierarchy() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let source = tmp.path();
|
||||
let mut map = BTreeMap::new();
|
||||
|
||||
let ino = ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
|
||||
|
||||
let a_ino = virtual_inode("a");
|
||||
let ab_ino = virtual_inode("a/b");
|
||||
|
||||
assert_eq!(ino, ab_ino);
|
||||
assert_eq!(map.len(), 2);
|
||||
assert!(map.contains_key(&a_ino));
|
||||
assert!(map.contains_key(&ab_ino));
|
||||
|
||||
let a_item = &map[&a_ino];
|
||||
assert_eq!(a_item.parent_inode, INodeNo::ROOT);
|
||||
|
||||
let ab_item = &map[&ab_ino];
|
||||
assert_eq!(ab_item.parent_inode, a_ino);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_virtual_dirs_idempotent() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let source = tmp.path();
|
||||
let mut map = BTreeMap::new();
|
||||
|
||||
ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
|
||||
let size_after_first = map.len();
|
||||
|
||||
ensure_virtual_dirs(Path::new("a/b/file"), source, &mut map);
|
||||
let size_after_second = map.len();
|
||||
|
||||
assert_eq!(size_after_first, size_after_second);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_virtual_paths_skips_root() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let source = tmp.path();
|
||||
let mut snapshot = BTreeMap::new();
|
||||
|
||||
let real_ino = INodeNo(42);
|
||||
let real_item = Item::new(
|
||||
real_ino,
|
||||
INodeNo::ROOT,
|
||||
"real_file".to_string(),
|
||||
source.to_path_buf(),
|
||||
PathBuf::from("real_file"),
|
||||
FileType::File,
|
||||
FileAttrs::from(&fs::metadata(source).unwrap()),
|
||||
None,
|
||||
);
|
||||
snapshot.insert(real_ino, real_item);
|
||||
|
||||
let mut db_items = HashMap::new();
|
||||
db_items.insert(
|
||||
1i64,
|
||||
Model {
|
||||
inode: 1,
|
||||
hash: 0,
|
||||
name: "root".to_string(),
|
||||
original_path: "/tmp/test".to_string(),
|
||||
local_path: "/tmp/test".to_string(),
|
||||
file_type: "directory".to_string(),
|
||||
},
|
||||
);
|
||||
db_items.insert(
|
||||
42i64,
|
||||
Model {
|
||||
inode: 42,
|
||||
hash: 0,
|
||||
name: "real_file".to_string(),
|
||||
original_path: "real_file".to_string(),
|
||||
local_path: "real_file".to_string(),
|
||||
file_type: "file".to_string(),
|
||||
},
|
||||
);
|
||||
|
||||
restore_virtual_paths(&mut snapshot, &db_items, source);
|
||||
|
||||
for item in snapshot.values() {
|
||||
assert_ne!(item.name, "/");
|
||||
assert_ne!(item.name, "tmp");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,562 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use fuser::INodeNo;
|
||||
use musicfs::control::ClientControlServiceImpl;
|
||||
use musicfs::db::entities;
|
||||
use musicfs::item::{FileType, Item};
|
||||
use musicfs::music::db::entities::{artists, music_metadata as mm_entity};
|
||||
use musicfs::music::metadata::MusicMetadata;
|
||||
use musicfs::origins::attrs::FileAttrs;
|
||||
use musicfs_proto::{
|
||||
ClientControl, GetMusicMetadataRequest, ListFilesRequest, UpdateMusicMetadataRequest,
|
||||
};
|
||||
use sea_orm::{DatabaseBackend, MockDatabase, MockExecResult};
|
||||
use tonic::{Code, Request};
|
||||
|
||||
// ── Test helpers ───────────────────────────────────────────────────────
|
||||
|
||||
fn make_attrs() -> FileAttrs {
|
||||
FileAttrs {
|
||||
size: 4096,
|
||||
blocks: 8,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
perm: 0o644,
|
||||
nlink: 1,
|
||||
uid: 0,
|
||||
gid: 0,
|
||||
rdev: 0,
|
||||
blksize: 4096,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_flac_item(inode: u64, name: &str, metadata: Option<MusicMetadata>) -> Item {
|
||||
Item {
|
||||
inode: INodeNo(inode),
|
||||
parent_inode: INodeNo(1),
|
||||
name: name.to_string(),
|
||||
original_path: format!("/{name}").into(),
|
||||
local_path: format!("Artist/Album/{name}").into(),
|
||||
file_type: FileType::File,
|
||||
attrs: make_attrs(),
|
||||
music_metadata: metadata,
|
||||
hash: inode,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_flac_metadata() -> MusicMetadata {
|
||||
MusicMetadata {
|
||||
artist: vec!["Test Artist".to_string()],
|
||||
album_artist: Some("Test Artist".to_string()),
|
||||
album: "Test Album".to_string(),
|
||||
track_number: 1,
|
||||
track_title: "Test Track".to_string(),
|
||||
other_tags: vec![],
|
||||
header: vec![],
|
||||
picture_block_headers: vec![],
|
||||
picture_data_ranges: vec![],
|
||||
real_audio_start: 0,
|
||||
vorbis_comment_offset: 0,
|
||||
vorbis_comment_length: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Mock DB for get tests (no INSERT operations expected).
|
||||
fn mock_db_readonly() -> sea_orm::DatabaseConnection {
|
||||
MockDatabase::new(DatabaseBackend::Postgres).into_connection()
|
||||
}
|
||||
|
||||
/// Mock DB for update with one artist.
|
||||
/// save_music_metadata does: DELETE, INSERT music_metadata, INSERT artists.
|
||||
/// persist_layout_changes does: UPDATE items (RETURNING), then one DELETE
|
||||
/// per orphaned virtual directory (worst case = 2: artist + album dirs).
|
||||
fn mock_db_update_with_artist(inode: i64) -> sea_orm::DatabaseConnection {
|
||||
MockDatabase::new(DatabaseBackend::Postgres)
|
||||
.append_exec_results([MockExecResult {
|
||||
rows_affected: 1,
|
||||
..Default::default()
|
||||
}])
|
||||
.append_query_results([vec![mm_entity::Model {
|
||||
inode,
|
||||
track_title: String::new(),
|
||||
album: String::new(),
|
||||
track_number: 0,
|
||||
header: vec![],
|
||||
real_audio_start: 0,
|
||||
}]])
|
||||
.append_query_results([vec![artists::Model {
|
||||
inode,
|
||||
artist: String::new(),
|
||||
}]])
|
||||
.append_query_results([vec![entities::Model {
|
||||
inode,
|
||||
name: String::new(),
|
||||
original_path: String::new(),
|
||||
local_path: String::new(),
|
||||
file_type: "file".to_string(),
|
||||
hash: 0,
|
||||
}]])
|
||||
.append_exec_results([MockExecResult {
|
||||
rows_affected: 1,
|
||||
..Default::default()
|
||||
}])
|
||||
.append_exec_results([MockExecResult {
|
||||
rows_affected: 1,
|
||||
..Default::default()
|
||||
}])
|
||||
.into_connection()
|
||||
}
|
||||
|
||||
/// Mock DB for update with no artists.
|
||||
/// save_music_metadata does: DELETE, INSERT music_metadata only.
|
||||
/// persist_layout_changes does: UPDATE items (RETURNING), then up to two
|
||||
/// DELETEs for orphaned virtual directories.
|
||||
fn mock_db_update_no_artist(inode: i64) -> sea_orm::DatabaseConnection {
|
||||
MockDatabase::new(DatabaseBackend::Postgres)
|
||||
.append_exec_results([MockExecResult {
|
||||
rows_affected: 1,
|
||||
..Default::default()
|
||||
}])
|
||||
.append_query_results([vec![mm_entity::Model {
|
||||
inode,
|
||||
track_title: String::new(),
|
||||
album: String::new(),
|
||||
track_number: 0,
|
||||
header: vec![],
|
||||
real_audio_start: 0,
|
||||
}]])
|
||||
.append_query_results([vec![entities::Model {
|
||||
inode,
|
||||
name: String::new(),
|
||||
original_path: String::new(),
|
||||
local_path: String::new(),
|
||||
file_type: "file".to_string(),
|
||||
hash: 0,
|
||||
}]])
|
||||
.append_exec_results([MockExecResult {
|
||||
rows_affected: 1,
|
||||
..Default::default()
|
||||
}])
|
||||
.append_exec_results([MockExecResult {
|
||||
rows_affected: 1,
|
||||
..Default::default()
|
||||
}])
|
||||
.into_connection()
|
||||
}
|
||||
|
||||
fn make_dir_item(inode: u64, name: &str) -> Item {
|
||||
Item {
|
||||
inode: INodeNo(inode),
|
||||
parent_inode: INodeNo(1),
|
||||
name: name.to_string(),
|
||||
original_path: format!("/{name}").into(),
|
||||
local_path: name.into(),
|
||||
file_type: FileType::Directory,
|
||||
attrs: make_attrs(),
|
||||
music_metadata: None,
|
||||
hash: inode,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_removes_orphaned_virtual_dirs_when_last_file_moves_out() {
|
||||
let source_dir = tempfile::tempdir().unwrap();
|
||||
let source = source_dir.path();
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(0),
|
||||
Item {
|
||||
inode: INodeNo(0),
|
||||
parent_inode: INodeNo::ROOT,
|
||||
name: "/".to_string(),
|
||||
original_path: source.to_path_buf(),
|
||||
local_path: "/".into(),
|
||||
file_type: FileType::Directory,
|
||||
attrs: make_attrs(),
|
||||
music_metadata: None,
|
||||
hash: 0,
|
||||
},
|
||||
);
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
make_flac_item(100, "song.flac", Some(make_flac_metadata())),
|
||||
);
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files.clone(), mock_db_update_with_artist(100));
|
||||
|
||||
// Pre-create the virtual dirs for the file's current layout so we can
|
||||
// assert they get pruned after the metadata update moves the file.
|
||||
{
|
||||
use musicfs::virtual_dirs::{ensure_virtual_dirs, virtual_inode};
|
||||
let mut guard = files.lock().unwrap();
|
||||
ensure_virtual_dirs(
|
||||
std::path::Path::new("Artist/Album/song.flac"),
|
||||
source,
|
||||
&mut guard,
|
||||
);
|
||||
assert!(guard.contains_key(&virtual_inode("Artist")));
|
||||
assert!(guard.contains_key(&virtual_inode("Artist/Album")));
|
||||
}
|
||||
|
||||
svc.update_music_metadata(Request::new(UpdateMusicMetadataRequest {
|
||||
inode: 100,
|
||||
artist: vec!["New Artist".to_string()],
|
||||
album_artist: Some("New Artist".to_string()),
|
||||
album: "New Album".to_string(),
|
||||
track_number: 1,
|
||||
track_title: "New Title".to_string(),
|
||||
other_tags: vec![],
|
||||
}))
|
||||
.await
|
||||
.expect("update should succeed");
|
||||
|
||||
let guard = files.lock().unwrap();
|
||||
use musicfs::virtual_dirs::virtual_inode;
|
||||
assert!(
|
||||
!guard.contains_key(&virtual_inode("Artist")),
|
||||
"old Artist dir must be removed when no files remain under it"
|
||||
);
|
||||
assert!(
|
||||
!guard.contains_key(&virtual_inode("Artist/Album")),
|
||||
"old Album dir must be removed"
|
||||
);
|
||||
assert!(guard.contains_key(&virtual_inode("New Artist")));
|
||||
assert!(guard.contains_key(&virtual_inode("New Artist/New Album")));
|
||||
|
||||
let item = guard.get(&INodeNo(100)).expect("file still present");
|
||||
assert_eq!(
|
||||
item.local_path,
|
||||
std::path::PathBuf::from("New Artist/New Album/01 - New Title.flac")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_keeps_shared_artist_dir_when_other_album_remains() {
|
||||
let source_dir = tempfile::tempdir().unwrap();
|
||||
let source = source_dir.path();
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(0),
|
||||
Item {
|
||||
inode: INodeNo(0),
|
||||
parent_inode: INodeNo::ROOT,
|
||||
name: "/".to_string(),
|
||||
original_path: source.to_path_buf(),
|
||||
local_path: "/".into(),
|
||||
file_type: FileType::Directory,
|
||||
attrs: make_attrs(),
|
||||
music_metadata: None,
|
||||
hash: 0,
|
||||
},
|
||||
);
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
make_flac_item(100, "a.flac", Some(make_flac_metadata())),
|
||||
);
|
||||
files.insert(
|
||||
INodeNo(101),
|
||||
make_flac_item(101, "b.flac", Some(make_flac_metadata())),
|
||||
);
|
||||
files.get_mut(&INodeNo(101)).unwrap().local_path = "Artist/Other Album/b.flac".into();
|
||||
|
||||
// Pre-create virtual dirs for both files' layouts: pruning after the
|
||||
// update should remove Album/ (file 100's old parent) but keep Artist/
|
||||
// (still has Other Album/ with file 101 in it).
|
||||
{
|
||||
use musicfs::virtual_dirs::ensure_virtual_dirs;
|
||||
ensure_virtual_dirs(
|
||||
std::path::Path::new("Artist/Album/a.flac"),
|
||||
source,
|
||||
&mut files,
|
||||
);
|
||||
ensure_virtual_dirs(
|
||||
std::path::Path::new("Artist/Other Album/b.flac"),
|
||||
source,
|
||||
&mut files,
|
||||
);
|
||||
}
|
||||
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files.clone(), mock_db_update_with_artist(100));
|
||||
|
||||
svc.update_music_metadata(Request::new(UpdateMusicMetadataRequest {
|
||||
inode: 100,
|
||||
artist: vec!["New Artist".to_string()],
|
||||
album_artist: Some("New Artist".to_string()),
|
||||
album: "New Album".to_string(),
|
||||
track_number: 1,
|
||||
track_title: "New Title".to_string(),
|
||||
other_tags: vec![],
|
||||
}))
|
||||
.await
|
||||
.expect("update should succeed");
|
||||
|
||||
let guard = files.lock().unwrap();
|
||||
use musicfs::virtual_dirs::virtual_inode;
|
||||
assert!(
|
||||
guard.contains_key(&virtual_inode("Artist")),
|
||||
"Artist dir must remain while Other Album still has files"
|
||||
);
|
||||
assert!(
|
||||
!guard.contains_key(&virtual_inode("Artist/Album")),
|
||||
"Artist/Album dir must be removed"
|
||||
);
|
||||
}
|
||||
|
||||
// ── ListFiles ─────────────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_files_returns_all_files_with_full_field_projection() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
make_flac_item(100, "a.flac", Some(make_flac_metadata())),
|
||||
);
|
||||
files.insert(INodeNo(101), make_flac_item(101, "b.flac", None));
|
||||
// Directory entries must be filtered out.
|
||||
files.insert(INodeNo(2), make_dir_item(2, "Artist"));
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files, mock_db_readonly());
|
||||
|
||||
let resp = svc
|
||||
.list_files(Request::new(ListFilesRequest {}))
|
||||
.await
|
||||
.expect("list should succeed");
|
||||
|
||||
let mut entries = resp.into_inner().files;
|
||||
assert_eq!(entries.len(), 2, "directories must be filtered out");
|
||||
entries.sort_by_key(|e| e.inode);
|
||||
|
||||
assert_eq!(entries[0].inode, 100);
|
||||
assert_eq!(entries[0].name, "a.flac");
|
||||
assert_eq!(entries[0].original_path, "/a.flac");
|
||||
assert_eq!(entries[0].local_path, "Artist/Album/a.flac");
|
||||
assert!(entries[0].metadata.is_some());
|
||||
|
||||
assert_eq!(entries[1].inode, 101);
|
||||
assert_eq!(entries[1].name, "b.flac");
|
||||
assert!(entries[1].metadata.is_none(), "no metadata on this file");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_files_empty_map_returns_empty_list() {
|
||||
let files = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let svc = ClientControlServiceImpl::new(files, mock_db_readonly());
|
||||
|
||||
let resp = svc
|
||||
.list_files(Request::new(ListFilesRequest {}))
|
||||
.await
|
||||
.expect("list should succeed");
|
||||
|
||||
assert!(resp.into_inner().files.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_files_skips_directories_and_root() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(INodeNo(0), make_dir_item(0, "/")); // root
|
||||
files.insert(INodeNo(1), make_dir_item(1, "Artist"));
|
||||
files.insert(INodeNo(2), make_dir_item(2, "Album"));
|
||||
files.insert(INodeNo(100), make_flac_item(100, "track.flac", None));
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files, mock_db_readonly());
|
||||
|
||||
let resp = svc
|
||||
.list_files(Request::new(ListFilesRequest {}))
|
||||
.await
|
||||
.expect("list should succeed");
|
||||
|
||||
let inodes: Vec<u64> = resp
|
||||
.into_inner()
|
||||
.files
|
||||
.into_iter()
|
||||
.map(|e| e.inode)
|
||||
.collect();
|
||||
assert_eq!(inodes, vec![100]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_returns_metadata_for_valid_inode() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
make_flac_item(100, "song.flac", Some(make_flac_metadata())),
|
||||
);
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files, mock_db_readonly());
|
||||
|
||||
let resp = svc
|
||||
.get_music_metadata(Request::new(GetMusicMetadataRequest { inode: 100 }))
|
||||
.await
|
||||
.expect("get should succeed");
|
||||
|
||||
let md = resp.into_inner().metadata.expect("metadata present");
|
||||
assert_eq!(md.track_title, "Test Track");
|
||||
assert_eq!(md.album, "Test Album");
|
||||
assert_eq!(md.artist, vec!["Test Artist".to_string()]);
|
||||
assert_eq!(md.track_number, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_returns_not_found_for_missing_inode() {
|
||||
let files = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let svc = ClientControlServiceImpl::new(files, mock_db_readonly());
|
||||
|
||||
let err = svc
|
||||
.get_music_metadata(Request::new(GetMusicMetadataRequest { inode: 999 }))
|
||||
.await
|
||||
.expect_err("should be NOT_FOUND");
|
||||
|
||||
assert_eq!(err.code(), Code::NotFound);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_returns_not_found_when_item_has_no_metadata() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(INodeNo(100), make_flac_item(100, "song.flac", None));
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files, mock_db_readonly());
|
||||
|
||||
let err = svc
|
||||
.get_music_metadata(Request::new(GetMusicMetadataRequest { inode: 100 }))
|
||||
.await
|
||||
.expect_err("should be NOT_FOUND");
|
||||
|
||||
assert_eq!(err.code(), Code::NotFound);
|
||||
}
|
||||
|
||||
// ── UpdateMusicMetadata ───────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_changes_tags_and_response_reflects_new_values() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
make_flac_item(100, "song.flac", Some(make_flac_metadata())),
|
||||
);
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files.clone(), mock_db_update_with_artist(100));
|
||||
|
||||
let resp = svc
|
||||
.update_music_metadata(Request::new(UpdateMusicMetadataRequest {
|
||||
inode: 100,
|
||||
artist: vec!["Updated Artist".to_string()],
|
||||
album_artist: Some("Updated Artist".to_string()),
|
||||
album: "Updated Album".to_string(),
|
||||
track_number: 7,
|
||||
track_title: "Updated Title".to_string(),
|
||||
other_tags: vec![],
|
||||
}))
|
||||
.await
|
||||
.expect("update should succeed");
|
||||
|
||||
let md = resp.into_inner().metadata.expect("metadata present");
|
||||
assert_eq!(md.track_title, "Updated Title");
|
||||
assert_eq!(md.album, "Updated Album");
|
||||
assert_eq!(md.artist, vec!["Updated Artist".to_string()]);
|
||||
assert_eq!(md.track_number, 7);
|
||||
|
||||
let guard = files.lock().unwrap();
|
||||
let item = guard.get(&INodeNo(100)).expect("item still in map");
|
||||
let mm = item.music_metadata.as_ref().expect("metadata present");
|
||||
assert_eq!(mm.track_title, "Updated Title");
|
||||
assert_eq!(mm.album, "Updated Album");
|
||||
assert_eq!(mm.artist, vec!["Updated Artist".to_string()]);
|
||||
assert_eq!(mm.track_number, 7);
|
||||
|
||||
// Layout was recomputed from the new metadata.
|
||||
assert_eq!(item.name, "07 - Updated Title.flac");
|
||||
assert_eq!(
|
||||
item.local_path,
|
||||
std::path::PathBuf::from("Updated Artist/Updated Album/07 - Updated Title.flac")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_returns_not_found_for_missing_inode() {
|
||||
let files = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let svc = ClientControlServiceImpl::new(files, mock_db_readonly());
|
||||
|
||||
let err = svc
|
||||
.update_music_metadata(Request::new(UpdateMusicMetadataRequest {
|
||||
inode: 999,
|
||||
artist: vec![],
|
||||
album_artist: None,
|
||||
album: String::new(),
|
||||
track_number: 0,
|
||||
track_title: String::new(),
|
||||
other_tags: vec![],
|
||||
}))
|
||||
.await
|
||||
.expect_err("should be NOT_FOUND");
|
||||
|
||||
assert_eq!(err.code(), Code::NotFound);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_returns_not_found_when_item_has_no_metadata() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(INodeNo(100), make_flac_item(100, "song.flac", None));
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files, mock_db_readonly());
|
||||
|
||||
let err = svc
|
||||
.update_music_metadata(Request::new(UpdateMusicMetadataRequest {
|
||||
inode: 100,
|
||||
artist: vec!["X".to_string()],
|
||||
album_artist: None,
|
||||
album: "X".to_string(),
|
||||
track_number: 1,
|
||||
track_title: "X".to_string(),
|
||||
other_tags: vec![],
|
||||
}))
|
||||
.await
|
||||
.expect_err("should be NOT_FOUND");
|
||||
|
||||
assert_eq!(err.code(), Code::NotFound);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_clears_tags_when_empty_values_sent() {
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert(
|
||||
INodeNo(100),
|
||||
make_flac_item(100, "song.flac", Some(make_flac_metadata())),
|
||||
);
|
||||
let files = Arc::new(Mutex::new(files));
|
||||
let svc = ClientControlServiceImpl::new(files.clone(), mock_db_update_no_artist(100));
|
||||
|
||||
let resp = svc
|
||||
.update_music_metadata(Request::new(UpdateMusicMetadataRequest {
|
||||
inode: 100,
|
||||
artist: vec![],
|
||||
album_artist: None,
|
||||
album: String::new(),
|
||||
track_number: 0,
|
||||
track_title: String::new(),
|
||||
other_tags: vec![],
|
||||
}))
|
||||
.await
|
||||
.expect("update should succeed");
|
||||
|
||||
let md = resp.into_inner().metadata.expect("metadata present");
|
||||
assert!(md.artist.is_empty());
|
||||
assert!(md.album.is_empty());
|
||||
assert!(md.track_title.is_empty());
|
||||
|
||||
let guard = files.lock().unwrap();
|
||||
let mm = guard
|
||||
.get(&INodeNo(100))
|
||||
.unwrap()
|
||||
.music_metadata
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
assert!(mm.artist.is_empty());
|
||||
assert!(mm.album.is_empty());
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use musicfs::origins::local::file_io::{assemble_flac_read, read_bytes_at};
|
||||
|
||||
fn file_reader(path: PathBuf) -> impl Fn(u64, usize) -> std::io::Result<Vec<u8>> {
|
||||
return move |offset, len| read_bytes_at(&path, offset, len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_picture_header_region() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let data: Vec<u8> = (0..100).collect();
|
||||
f.write_all(&data).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let header = b"ABCD";
|
||||
let pic_hdr = vec![0x86u8, 0x00, 0x00, 0x05];
|
||||
let pic_ranges = [(10u64, 5u64)];
|
||||
let real_audio_start = 50u64;
|
||||
|
||||
let reader = file_reader(path);
|
||||
let result = assemble_flac_read(
|
||||
&reader,
|
||||
header,
|
||||
&[pic_hdr.clone()],
|
||||
&pic_ranges,
|
||||
real_audio_start,
|
||||
4,
|
||||
4,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, pic_hdr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_picture_data_region() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let data: Vec<u8> = (0..100).collect();
|
||||
f.write_all(&data).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let header = b"ABCD";
|
||||
let pic_hdr = vec![0x86u8, 0x00, 0x00, 0x05];
|
||||
let pic_ranges = [(10u64, 5u64)];
|
||||
let real_audio_start = 50u64;
|
||||
|
||||
let reader = file_reader(path);
|
||||
let result = assemble_flac_read(
|
||||
&reader,
|
||||
header,
|
||||
&[pic_hdr.clone()],
|
||||
&pic_ranges,
|
||||
real_audio_start,
|
||||
8,
|
||||
5,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, vec![10, 11, 12, 13, 14]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assemble_flac_read_full_virtual_layout() {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
let data: Vec<u8> = (0..200).map(|i| i as u8).collect();
|
||||
f.write_all(&data).unwrap();
|
||||
f.flush().unwrap();
|
||||
let path = f.path().to_path_buf();
|
||||
|
||||
let header = vec![0xAA; 8];
|
||||
let real_audio_start = 100u64;
|
||||
|
||||
let reader = file_reader(path);
|
||||
let result = assemble_flac_read(&reader, &header, &[], &[], real_audio_start, 0, 18).unwrap();
|
||||
|
||||
let mut expected = vec![0xAA; 8];
|
||||
expected.extend_from_slice(&(100..110).map(|i| i as u8).collect::<Vec<u8>>());
|
||||
assert_eq!(result, expected);
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use fuser::INodeNo;
|
||||
|
||||
use musicfs::item::FileType;
|
||||
use musicfs::origins::local::snapshot::{build_snapshot, fill_fileset};
|
||||
|
||||
#[test]
|
||||
fn build_snapshot_empty_dir() {
|
||||
let source = tempfile::tempdir().unwrap();
|
||||
let dest = tempfile::tempdir().unwrap();
|
||||
|
||||
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
|
||||
|
||||
assert_eq!(snapshot.len(), 1);
|
||||
assert!(snapshot.contains_key(&INodeNo::ROOT));
|
||||
|
||||
let root = &snapshot[&INodeNo::ROOT];
|
||||
assert_eq!(root.inode, INodeNo::ROOT);
|
||||
assert_eq!(root.parent_inode, INodeNo::ROOT);
|
||||
assert_eq!(root.file_type, FileType::Directory);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_snapshot_flat_files() {
|
||||
let source = tempfile::tempdir().unwrap();
|
||||
let dest = tempfile::tempdir().unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("a.txt")).unwrap();
|
||||
f.write_all(b"content a").unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("b.txt")).unwrap();
|
||||
f.write_all(b"content b").unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("c.txt")).unwrap();
|
||||
f.write_all(b"content c").unwrap();
|
||||
|
||||
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
|
||||
|
||||
assert_eq!(snapshot.len(), 4);
|
||||
assert!(snapshot.contains_key(&INodeNo::ROOT));
|
||||
|
||||
let file_entries: Vec<_> = snapshot
|
||||
.values()
|
||||
.filter(|item| item.file_type == FileType::File)
|
||||
.collect();
|
||||
assert_eq!(file_entries.len(), 3);
|
||||
|
||||
for file_entry in file_entries {
|
||||
assert_eq!(file_entry.parent_inode, INodeNo::ROOT);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_snapshot_nested_dirs() {
|
||||
let source = tempfile::tempdir().unwrap();
|
||||
let dest = tempfile::tempdir().unwrap();
|
||||
|
||||
fs::create_dir(source.path().join("subdir")).unwrap();
|
||||
let nested_path = source.path().join("subdir").join("file.txt");
|
||||
let mut f = fs::File::create(&nested_path).unwrap();
|
||||
f.write_all(b"nested content").unwrap();
|
||||
|
||||
let snapshot = build_snapshot(source.path(), dest.path()).unwrap();
|
||||
|
||||
assert!(snapshot.contains_key(&INodeNo::ROOT));
|
||||
|
||||
let file_entry = snapshot
|
||||
.values()
|
||||
.find(|item| item.name == "file.txt" && item.file_type == FileType::File)
|
||||
.expect("file.txt not found");
|
||||
|
||||
assert_eq!(file_entry.parent_inode, INodeNo::ROOT);
|
||||
assert_eq!(file_entry.original_path, nested_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fill_fileset_removes_deleted() {
|
||||
let source = tempfile::tempdir().unwrap();
|
||||
let dest = tempfile::tempdir().unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("file1.txt")).unwrap();
|
||||
f.write_all(b"content 1").unwrap();
|
||||
|
||||
let mut f = fs::File::create(source.path().join("file2.txt")).unwrap();
|
||||
f.write_all(b"content 2").unwrap();
|
||||
|
||||
let initial_snapshot = build_snapshot(source.path(), dest.path()).unwrap();
|
||||
let map = Arc::new(Mutex::new(initial_snapshot));
|
||||
|
||||
assert_eq!(map.lock().unwrap().len(), 3);
|
||||
|
||||
fs::remove_file(source.path().join("file1.txt")).unwrap();
|
||||
|
||||
fill_fileset(&map, source.path(), dest.path());
|
||||
|
||||
let final_map = map.lock().unwrap();
|
||||
assert_eq!(final_map.len(), 2);
|
||||
assert!(final_map.contains_key(&INodeNo::ROOT));
|
||||
|
||||
let remaining_files: Vec<_> = final_map
|
||||
.values()
|
||||
.filter(|item| item.file_type == FileType::File)
|
||||
.collect();
|
||||
assert_eq!(remaining_files.len(), 1);
|
||||
assert_eq!(remaining_files[0].name, "file2.txt");
|
||||
}
|
||||
@@ -4,15 +4,11 @@ version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
toml.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
symphonia.workspace = true
|
||||
twox-hash.workspace = true
|
||||
tracing.workspace = true
|
||||
xxhash-rust.workspace = true
|
||||
hex.workspace = true
|
||||
parking_lot.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
tracing-appender.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
use std::{
|
||||
fs,
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use std::os::unix::fs::{MetadataExt, PermissionsExt};
|
||||
|
||||
/// Owned, `Clone`-able snapshot of the file attributes that musicfs needs to
|
||||
/// serve FUSE `getattr` and to compute the per-item identity hash.
|
||||
///
|
||||
/// Decoupled from `std::fs::Metadata` so that non-disk origins (e.g. a future
|
||||
/// `NetworkOrigin` reading a server manifest) can construct equivalent
|
||||
/// attributes without a real inode on disk. The on-disk origin builds this
|
||||
/// via `From<&fs::Metadata>`; other origins build it from their own metadata
|
||||
/// source using the same field types.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FileAttrs {
|
||||
pub size: u64,
|
||||
pub blocks: u64,
|
||||
pub atime: SystemTime,
|
||||
pub mtime: SystemTime,
|
||||
pub ctime: SystemTime,
|
||||
pub crtime: SystemTime,
|
||||
pub perm: u16,
|
||||
pub nlink: u32,
|
||||
pub uid: u32,
|
||||
pub gid: u32,
|
||||
pub rdev: u32,
|
||||
pub blksize: u32,
|
||||
}
|
||||
|
||||
impl From<&fs::Metadata> for FileAttrs {
|
||||
fn from(metadata: &fs::Metadata) -> Self {
|
||||
return FileAttrs {
|
||||
size: metadata.size(),
|
||||
blocks: metadata.blocks(),
|
||||
atime: metadata.accessed().unwrap_or(UNIX_EPOCH),
|
||||
mtime: metadata.modified().unwrap_or(UNIX_EPOCH),
|
||||
ctime: UNIX_EPOCH + Duration::from_secs(metadata.ctime() as u64),
|
||||
crtime: metadata.created().unwrap_or(UNIX_EPOCH),
|
||||
perm: metadata.permissions().mode() as u16,
|
||||
nlink: metadata.nlink() as u32,
|
||||
uid: metadata.uid(),
|
||||
gid: metadata.gid(),
|
||||
rdev: metadata.rdev() as u32,
|
||||
blksize: metadata.blksize() as u32,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn file_attrs_from_metadata_copies_size_and_block_fields() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = fs::metadata(tmp.path()).unwrap();
|
||||
|
||||
let attrs = FileAttrs::from(&metadata);
|
||||
|
||||
assert_eq!(attrs.size, metadata.size());
|
||||
assert_eq!(attrs.blocks, metadata.blocks());
|
||||
assert_eq!(attrs.blksize, metadata.blksize() as u32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_attrs_from_metadata_copies_unix_ownership() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = fs::metadata(tmp.path()).unwrap();
|
||||
|
||||
let attrs = FileAttrs::from(&metadata);
|
||||
|
||||
assert_eq!(attrs.uid, metadata.uid());
|
||||
assert_eq!(attrs.gid, metadata.gid());
|
||||
assert_eq!(attrs.perm, metadata.permissions().mode() as u16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_attrs_from_metadata_preserves_mtime() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let metadata = fs::metadata(tmp.path()).unwrap();
|
||||
|
||||
let attrs = FileAttrs::from(&metadata);
|
||||
|
||||
assert_eq!(attrs.mtime, metadata.modified().unwrap_or(UNIX_EPOCH));
|
||||
}
|
||||
}
|
||||
@@ -1,239 +0,0 @@
|
||||
use crate::OriginId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Config {
|
||||
pub mount_point: PathBuf,
|
||||
pub cache_dir: PathBuf,
|
||||
pub origins: Vec<OriginConfig>,
|
||||
|
||||
#[serde(default)]
|
||||
pub cache: CacheConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub health: HealthConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub logging: LoggingConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OriginConfig {
|
||||
pub id: String,
|
||||
pub origin_type: OriginType,
|
||||
pub priority: u8,
|
||||
|
||||
#[serde(default = "default_enabled")]
|
||||
pub enabled: bool,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub settings: HashMap<String, toml::Value>,
|
||||
}
|
||||
|
||||
fn default_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum OriginType {
|
||||
Local,
|
||||
Nfs,
|
||||
Smb,
|
||||
S3,
|
||||
Sftp,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CacheConfig {
|
||||
#[serde(default = "default_metadata_cache_mb")]
|
||||
pub metadata_cache_mb: u64,
|
||||
|
||||
#[serde(default = "default_content_cache_gb")]
|
||||
pub content_cache_gb: u64,
|
||||
}
|
||||
|
||||
impl Default for CacheConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
metadata_cache_mb: default_metadata_cache_mb(),
|
||||
content_cache_gb: default_content_cache_gb(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_metadata_cache_mb() -> u64 {
|
||||
100
|
||||
}
|
||||
fn default_content_cache_gb() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HealthConfig {
|
||||
#[serde(default = "default_check_interval_secs")]
|
||||
pub check_interval_secs: u64,
|
||||
|
||||
#[serde(default = "default_timeout_ms")]
|
||||
pub timeout_ms: u64,
|
||||
|
||||
#[serde(default = "default_unhealthy_threshold")]
|
||||
pub unhealthy_threshold: u32,
|
||||
|
||||
#[serde(default)]
|
||||
pub per_origin_thresholds: HashMap<OriginType, u32>,
|
||||
}
|
||||
|
||||
impl Default for HealthConfig {
|
||||
fn default() -> Self {
|
||||
let mut per_origin = HashMap::new();
|
||||
per_origin.insert(OriginType::Local, 1);
|
||||
per_origin.insert(OriginType::Nfs, 3);
|
||||
per_origin.insert(OriginType::Smb, 3);
|
||||
per_origin.insert(OriginType::S3, 3);
|
||||
per_origin.insert(OriginType::Sftp, 3);
|
||||
|
||||
Self {
|
||||
check_interval_secs: default_check_interval_secs(),
|
||||
timeout_ms: default_timeout_ms(),
|
||||
unhealthy_threshold: default_unhealthy_threshold(),
|
||||
per_origin_thresholds: per_origin,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthConfig {
|
||||
pub fn threshold_for(&self, origin_type: OriginType) -> u32 {
|
||||
self.per_origin_thresholds
|
||||
.get(&origin_type)
|
||||
.copied()
|
||||
.unwrap_or(self.unhealthy_threshold)
|
||||
}
|
||||
}
|
||||
|
||||
fn default_check_interval_secs() -> u64 {
|
||||
30
|
||||
}
|
||||
fn default_timeout_ms() -> u64 {
|
||||
5000
|
||||
}
|
||||
fn default_unhealthy_threshold() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LoggingConfig {
|
||||
#[serde(default = "default_log_dir")]
|
||||
pub log_dir: PathBuf,
|
||||
|
||||
#[serde(default)]
|
||||
pub json_output: bool,
|
||||
|
||||
#[serde(default = "default_true")]
|
||||
pub journald: bool,
|
||||
|
||||
#[serde(default = "default_log_level")]
|
||||
pub level: String,
|
||||
|
||||
#[serde(default = "default_sample_rate")]
|
||||
pub trace_sample_rate: f32,
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
log_dir: default_log_dir(),
|
||||
json_output: false,
|
||||
journald: true,
|
||||
level: default_log_level(),
|
||||
trace_sample_rate: default_sample_rate(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_log_dir() -> PathBuf {
|
||||
PathBuf::from("/var/log/musicfs")
|
||||
}
|
||||
|
||||
fn default_log_level() -> String {
|
||||
"musicfs=info,warn".to_string()
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn default_sample_rate() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn from_file(path: &std::path::Path) -> Result<Self, ConfigError> {
|
||||
let content =
|
||||
std::fs::read_to_string(path).map_err(|e| ConfigError::Read(e.to_string()))?;
|
||||
toml::from_str(&content).map_err(|e| ConfigError::Parse(e.to_string()))
|
||||
}
|
||||
|
||||
pub fn origin_id(&self, id: &str) -> Option<OriginId> {
|
||||
self.origins
|
||||
.iter()
|
||||
.find(|o| o.id == id)
|
||||
.map(|_| OriginId::from(id))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("Failed to read config: {0}")]
|
||||
Read(String),
|
||||
|
||||
#[error("Failed to parse config: {0}")]
|
||||
Parse(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_config() {
|
||||
let toml = r#"
|
||||
mount_point = "/mnt/music"
|
||||
cache_dir = "/home/user/.cache/musicfs"
|
||||
|
||||
[[origins]]
|
||||
id = "local"
|
||||
origin_type = "local"
|
||||
priority = 1
|
||||
path = "/mnt/nas/music"
|
||||
|
||||
[[origins]]
|
||||
id = "backup"
|
||||
origin_type = "s3"
|
||||
priority = 2
|
||||
bucket = "music-backup"
|
||||
region = "us-east-1"
|
||||
"#;
|
||||
|
||||
let config: Config = toml::from_str(toml).unwrap();
|
||||
assert_eq!(config.origins.len(), 2);
|
||||
assert_eq!(config.origins[0].priority, 1);
|
||||
assert_eq!(config.origins[1].origin_type, OriginType::S3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_health_thresholds() {
|
||||
let health = HealthConfig::default();
|
||||
assert_eq!(health.threshold_for(OriginType::Local), 1);
|
||||
assert_eq!(health.threshold_for(OriginType::Sftp), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_defaults() {
|
||||
let cache = CacheConfig::default();
|
||||
assert_eq!(cache.metadata_cache_mb, 100);
|
||||
assert_eq!(cache.content_cache_gb, 10);
|
||||
}
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use thiserror::Error;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CredentialStore {
|
||||
cache: HashMap<String, Credential>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for CredentialStore {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("CredentialStore")
|
||||
.field("cache_keys", &self.cache.keys().collect::<Vec<_>>())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum Credential {
|
||||
Basic {
|
||||
username: String,
|
||||
#[serde(skip_serializing)]
|
||||
password: String,
|
||||
},
|
||||
|
||||
AwsKey {
|
||||
access_key_id: String,
|
||||
#[serde(skip_serializing)]
|
||||
secret_access_key: String,
|
||||
session_token: Option<String>,
|
||||
region: String,
|
||||
},
|
||||
|
||||
SshKey {
|
||||
username: String,
|
||||
private_key_path: PathBuf,
|
||||
#[serde(skip_serializing)]
|
||||
passphrase: Option<String>,
|
||||
},
|
||||
|
||||
EnvVar {
|
||||
var_name: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Credential {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Basic { username, .. } => f
|
||||
.debug_struct("Basic")
|
||||
.field("username", username)
|
||||
.field("password", &"[REDACTED]")
|
||||
.finish(),
|
||||
Self::AwsKey {
|
||||
access_key_id,
|
||||
session_token,
|
||||
region,
|
||||
..
|
||||
} => {
|
||||
let key_preview = if access_key_id.len() > 4 {
|
||||
format!("{}...", &access_key_id[..4])
|
||||
} else {
|
||||
"****".to_string()
|
||||
};
|
||||
let token_display = if session_token.is_some() {
|
||||
"[REDACTED]"
|
||||
} else {
|
||||
"None"
|
||||
};
|
||||
f.debug_struct("AwsKey")
|
||||
.field("access_key_id", &key_preview)
|
||||
.field("secret_access_key", &"[REDACTED]")
|
||||
.field("session_token", &token_display)
|
||||
.field("region", region)
|
||||
.finish()
|
||||
}
|
||||
Self::SshKey {
|
||||
username,
|
||||
private_key_path,
|
||||
..
|
||||
} => f
|
||||
.debug_struct("SshKey")
|
||||
.field("username", username)
|
||||
.field("private_key_path", private_key_path)
|
||||
.field("passphrase", &"[REDACTED]")
|
||||
.finish(),
|
||||
Self::EnvVar { var_name } => f
|
||||
.debug_struct("EnvVar")
|
||||
.field("var_name", var_name)
|
||||
.finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CredentialStore {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
cache: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(
|
||||
&mut self,
|
||||
origin_id: &str,
|
||||
config: &CredentialConfig,
|
||||
) -> Result<Credential, CredentialError> {
|
||||
debug!(origin_id = %origin_id, "Loading credentials");
|
||||
|
||||
if let Some(cred) = self.cache.get(origin_id) {
|
||||
trace!(origin_id = %origin_id, "Credential cache hit");
|
||||
return Ok(cred.clone());
|
||||
}
|
||||
|
||||
let cred = match config {
|
||||
CredentialConfig::Environment { prefix } => {
|
||||
trace!(origin_id = %origin_id, prefix = %prefix, "Loading from environment");
|
||||
self.load_from_env(prefix)?
|
||||
}
|
||||
CredentialConfig::File { path } => {
|
||||
trace!(origin_id = %origin_id, path = ?path, "Loading from file");
|
||||
self.load_from_file(path)?
|
||||
}
|
||||
CredentialConfig::Inline(cred) => {
|
||||
trace!(origin_id = %origin_id, "Using inline credential");
|
||||
cred.clone()
|
||||
}
|
||||
};
|
||||
|
||||
let cred_type = match &cred {
|
||||
Credential::Basic { .. } => "Basic",
|
||||
Credential::AwsKey { .. } => "AwsKey",
|
||||
Credential::SshKey { .. } => "SshKey",
|
||||
Credential::EnvVar { .. } => "EnvVar",
|
||||
};
|
||||
info!(origin_id = %origin_id, cred_type = %cred_type, "Credential loaded");
|
||||
|
||||
self.cache.insert(origin_id.to_string(), cred.clone());
|
||||
Ok(cred)
|
||||
}
|
||||
|
||||
fn load_from_env(&self, prefix: &str) -> Result<Credential, CredentialError> {
|
||||
if let (Ok(key), Ok(secret)) = (
|
||||
std::env::var(format!("{}_ACCESS_KEY_ID", prefix)),
|
||||
std::env::var(format!("{}_SECRET_ACCESS_KEY", prefix)),
|
||||
) {
|
||||
return Ok(Credential::AwsKey {
|
||||
access_key_id: key,
|
||||
secret_access_key: secret,
|
||||
session_token: std::env::var(format!("{}_SESSION_TOKEN", prefix)).ok(),
|
||||
region: std::env::var(format!("{}_REGION", prefix))
|
||||
.unwrap_or_else(|_| "us-east-1".to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
if let (Ok(user), Ok(pass)) = (
|
||||
std::env::var(format!("{}_USERNAME", prefix)),
|
||||
std::env::var(format!("{}_PASSWORD", prefix)),
|
||||
) {
|
||||
return Ok(Credential::Basic {
|
||||
username: user,
|
||||
password: pass,
|
||||
});
|
||||
}
|
||||
|
||||
warn!(prefix = %prefix, "No credentials found in environment");
|
||||
Err(CredentialError::NotFound(format!(
|
||||
"No credentials found with prefix {}",
|
||||
prefix
|
||||
)))
|
||||
}
|
||||
|
||||
fn load_from_file(&self, path: &PathBuf) -> Result<Credential, CredentialError> {
|
||||
let content =
|
||||
std::fs::read_to_string(path).map_err(|e| CredentialError::FileRead(e.to_string()))?;
|
||||
|
||||
if path.extension().map(|e| e == "json").unwrap_or(false) {
|
||||
serde_json::from_str(&content).map_err(|e| CredentialError::Parse(e.to_string()))
|
||||
} else {
|
||||
toml::from_str(&content).map_err(|e| CredentialError::Parse(e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.cache.clear();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "source")]
|
||||
pub enum CredentialConfig {
|
||||
Environment { prefix: String },
|
||||
File { path: PathBuf },
|
||||
Inline(Credential),
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CredentialError {
|
||||
#[error("Credential not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
#[error("Failed to read credential file: {0}")]
|
||||
FileRead(String),
|
||||
|
||||
#[error("Failed to parse credential: {0}")]
|
||||
Parse(String),
|
||||
}
|
||||
|
||||
impl Default for CredentialStore {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_credential_debug_redacted() {
|
||||
let cred = Credential::Basic {
|
||||
username: "user".to_string(),
|
||||
password: "secret123".to_string(),
|
||||
};
|
||||
|
||||
let debug_output = format!("{:?}", cred);
|
||||
assert!(debug_output.contains("user"));
|
||||
assert!(!debug_output.contains("secret123"));
|
||||
assert!(debug_output.contains("[REDACTED]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_aws_credential_debug_redacted() {
|
||||
let cred = Credential::AwsKey {
|
||||
access_key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
|
||||
secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
|
||||
session_token: None,
|
||||
region: "us-east-1".to_string(),
|
||||
};
|
||||
|
||||
let debug_output = format!("{:?}", cred);
|
||||
assert!(debug_output.contains("AKIA..."));
|
||||
assert!(!debug_output.contains("wJalrXUtnFEMI"));
|
||||
assert!(debug_output.contains("[REDACTED]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_credential_store_debug() {
|
||||
let mut store = CredentialStore::new();
|
||||
store.cache.insert(
|
||||
"test".to_string(),
|
||||
Credential::Basic {
|
||||
username: "user".to_string(),
|
||||
password: "secret".to_string(),
|
||||
},
|
||||
);
|
||||
|
||||
let debug_output = format!("{:?}", store);
|
||||
assert!(debug_output.contains("test"));
|
||||
assert!(!debug_output.contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_from_env() {
|
||||
std::env::set_var("TEST_ORIGIN_USERNAME", "testuser");
|
||||
std::env::set_var("TEST_ORIGIN_PASSWORD", "testpass");
|
||||
|
||||
let store = CredentialStore::new();
|
||||
let cred = store.load_from_env("TEST_ORIGIN").unwrap();
|
||||
|
||||
match cred {
|
||||
Credential::Basic { username, password } => {
|
||||
assert_eq!(username, "testuser");
|
||||
assert_eq!(password, "testpass");
|
||||
}
|
||||
_ => panic!("Expected Basic credential"),
|
||||
}
|
||||
|
||||
std::env::remove_var("TEST_ORIGIN_USERNAME");
|
||||
std::env::remove_var("TEST_ORIGIN_PASSWORD");
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum Error {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("Origin not found: {0}")]
|
||||
OriginNotFound(String),
|
||||
|
||||
#[error("File not found: {0}")]
|
||||
FileNotFound(String),
|
||||
|
||||
#[error("Path resolution failed: {0}")]
|
||||
PathResolution(String),
|
||||
|
||||
#[error("Cache error: {0}")]
|
||||
Cache(String),
|
||||
|
||||
#[error("Metadata extraction error: {0}")]
|
||||
Metadata(String),
|
||||
|
||||
#[error("Database error: {0}")]
|
||||
Database(String),
|
||||
|
||||
#[error("Database corrupted: {0}")]
|
||||
DatabaseCorrupted(String),
|
||||
|
||||
#[error("NFS stale file handle")]
|
||||
NfsStaleHandle,
|
||||
|
||||
#[error("Operation not permitted (read-only filesystem)")]
|
||||
ReadOnly,
|
||||
|
||||
#[error("No origin available to serve request")]
|
||||
NoOriginAvailable,
|
||||
|
||||
#[error("Maximum retries exceeded")]
|
||||
MaxRetriesExceeded,
|
||||
|
||||
#[error("Origin error: {0}")]
|
||||
Origin(String),
|
||||
|
||||
#[error("S3 error: {0}")]
|
||||
S3(String),
|
||||
|
||||
#[error("SFTP error: {0}")]
|
||||
Sftp(String),
|
||||
|
||||
#[error("Operation timed out: {0}")]
|
||||
Timeout(String),
|
||||
|
||||
#[error("Credential error: {0}")]
|
||||
Credential(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl Error {
|
||||
pub fn is_not_found(&self) -> bool {
|
||||
matches!(self, Error::FileNotFound(_))
|
||||
}
|
||||
|
||||
pub fn downcast_io(&self) -> Option<&std::io::Error> {
|
||||
match self {
|
||||
Error::Io(e) => Some(e),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
use crate::types::{FileId, OriginId, VirtualPath};
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, trace};
|
||||
|
||||
pub struct EventBus {
|
||||
sender: broadcast::Sender<Event>,
|
||||
}
|
||||
|
||||
impl EventBus {
|
||||
pub fn new(capacity: usize) -> Self {
|
||||
let (sender, _) = broadcast::channel(capacity);
|
||||
Self { sender }
|
||||
}
|
||||
|
||||
pub fn publish(&self, event: Event) {
|
||||
trace!(event = ?event, "Publishing event");
|
||||
let receiver_count = self.sender.receiver_count();
|
||||
if self.sender.send(event).is_err() && receiver_count > 0 {
|
||||
debug!(
|
||||
receiver_count = receiver_count,
|
||||
"Event dropped, no active receivers"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<Event> {
|
||||
self.sender.subscribe()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EventBus {
|
||||
fn default() -> Self {
|
||||
Self::new(1024)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Event {
|
||||
FileAdded {
|
||||
path: VirtualPath,
|
||||
origin_id: OriginId,
|
||||
},
|
||||
FileRemoved {
|
||||
path: VirtualPath,
|
||||
file_id: Option<FileId>,
|
||||
},
|
||||
FileModified {
|
||||
path: VirtualPath,
|
||||
},
|
||||
FileAccessed {
|
||||
file_id: FileId,
|
||||
path: VirtualPath,
|
||||
origin_id: OriginId,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
},
|
||||
OriginConnected {
|
||||
origin_id: OriginId,
|
||||
},
|
||||
OriginDisconnected {
|
||||
origin_id: OriginId,
|
||||
},
|
||||
SyncStarted {
|
||||
origin_id: OriginId,
|
||||
},
|
||||
SyncCompleted {
|
||||
origin_id: OriginId,
|
||||
files_changed: u64,
|
||||
},
|
||||
CacheEviction {
|
||||
bytes_freed: u64,
|
||||
},
|
||||
AllOriginsUnhealthy {
|
||||
candidate_count: usize,
|
||||
},
|
||||
OriginHealthChanged {
|
||||
origin_id: OriginId,
|
||||
healthy: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_event_bus() {
|
||||
let bus = EventBus::new(16);
|
||||
let mut rx = bus.subscribe();
|
||||
|
||||
bus.publish(Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
});
|
||||
|
||||
let event = rx.recv().await.unwrap();
|
||||
assert!(matches!(event, Event::SyncStarted { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_event_bus_multiple_subscribers() {
|
||||
let bus = EventBus::new(16);
|
||||
let mut rx1 = bus.subscribe();
|
||||
let mut rx2 = bus.subscribe();
|
||||
|
||||
bus.publish(Event::CacheEviction { bytes_freed: 1024 });
|
||||
|
||||
let e1 = rx1.recv().await.unwrap();
|
||||
let e2 = rx2.recv().await.unwrap();
|
||||
|
||||
assert!(matches!(e1, Event::CacheEviction { bytes_freed: 1024 }));
|
||||
assert!(matches!(e2, Event::CacheEviction { bytes_freed: 1024 }));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
use std::hash::Hasher;
|
||||
|
||||
use twox_hash::XxHash64;
|
||||
|
||||
pub fn compute_item_hash(
|
||||
inode: u64,
|
||||
original_path: &[u8],
|
||||
ctime_secs: u64,
|
||||
mtime_secs: u64,
|
||||
crtime_secs: u64,
|
||||
) -> u64 {
|
||||
let seed = 1234;
|
||||
let mut hasher = XxHash64::with_seed(seed);
|
||||
hasher.write_u64(inode);
|
||||
hasher.write(original_path);
|
||||
hasher.write_u64(ctime_secs);
|
||||
hasher.write_u64(mtime_secs);
|
||||
hasher.write_u64(crtime_secs);
|
||||
hasher.finish()
|
||||
}
|
||||
@@ -1,60 +1,8 @@
|
||||
pub mod config;
|
||||
pub mod credentials;
|
||||
pub mod error;
|
||||
pub mod events;
|
||||
pub mod metrics;
|
||||
pub mod resolver;
|
||||
pub mod supervisor;
|
||||
pub mod types;
|
||||
pub mod attrs;
|
||||
pub mod hash;
|
||||
pub mod logging;
|
||||
pub mod music;
|
||||
|
||||
pub use config::{
|
||||
CacheConfig, Config, ConfigError, HealthConfig, LoggingConfig, OriginConfig, OriginType,
|
||||
};
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub fn sanitize_path(path: &Path) -> String {
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
path.to_string_lossy().replace(&home, "~")
|
||||
} else {
|
||||
path.to_string_lossy().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Install a custom panic hook that logs panics via tracing before the default behavior.
|
||||
/// This ensures panics are captured in log files and journald.
|
||||
pub fn install_panic_hook() {
|
||||
let default_hook = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(move |info| {
|
||||
let thread = std::thread::current();
|
||||
let thread_name = thread.name().unwrap_or("<unnamed>");
|
||||
|
||||
let message = if let Some(s) = info.payload().downcast_ref::<&str>() {
|
||||
(*s).to_string()
|
||||
} else if let Some(s) = info.payload().downcast_ref::<String>() {
|
||||
s.clone()
|
||||
} else {
|
||||
"unknown panic".to_string()
|
||||
};
|
||||
|
||||
let location = info
|
||||
.location()
|
||||
.map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
|
||||
.unwrap_or_else(|| "unknown location".to_string());
|
||||
|
||||
tracing::error!(
|
||||
thread = thread_name,
|
||||
location = %location,
|
||||
"PANIC: {}",
|
||||
message
|
||||
);
|
||||
|
||||
default_hook(info);
|
||||
}));
|
||||
}
|
||||
pub use credentials::{Credential, CredentialConfig, CredentialError, CredentialStore};
|
||||
pub use error::{Error, Result};
|
||||
pub use events::{Event, EventBus};
|
||||
pub use metrics::{CacheMetrics, FuseOpsMetrics, Metrics, OriginsMetrics};
|
||||
pub use resolver::{PathResolver, PathTemplate};
|
||||
pub use types::*;
|
||||
pub use attrs::FileAttrs;
|
||||
pub use hash::compute_item_hash;
|
||||
pub use music::metadata::MusicMetadata;
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Process-wide logging initialization.
|
||||
//!
|
||||
//! Wires three `fmt` layers under a single global [`EnvFilter`] (driven by
|
||||
//! `RUST_LOG`, defaulting to `info`):
|
||||
//!
|
||||
//! - **stdout** — `INFO`/`DEBUG`/`TRACE` (the verbose side)
|
||||
//! - **stderr** — `WARN`/`ERROR` (the severe side)
|
||||
//! - **file** — everything passing the global filter, daily-rotated with
|
||||
//! retention, written without ANSI colors so log files stay clean.
|
||||
//!
|
||||
//! [`init`] returns the [`WorkerGuard`] that owns the non-blocking file
|
||||
//! buffer; the caller binds it for the whole process lifetime so the buffer
|
||||
//! flushes on exit. Dropping it early would silently drop pending log lines.
|
||||
//!
|
||||
//! The FUSE/RPC hot paths carry `trace!`/`debug!` points that fire per syscall
|
||||
//! — thousands per second under load. They are inert unless
|
||||
//! `RUST_LOG=trace`/`debug` is set; do not enable those levels in production
|
||||
//! casually.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use tracing::Level;
|
||||
use tracing_subscriber::{EnvFilter, filter::filter_fn, fmt, prelude::*};
|
||||
|
||||
/// Configuration handed to [`init`] by each binary.
|
||||
pub struct LogConfig {
|
||||
/// Directory the daily-rotated log files are written into.
|
||||
pub log_dir: PathBuf,
|
||||
/// Filename prefix; conventionally the binary name (`musicfs` or
|
||||
/// `musicfs-server`).
|
||||
pub file_prefix: String,
|
||||
/// Retention: keep at most this many rotated log files.
|
||||
pub max_files: usize,
|
||||
}
|
||||
|
||||
/// Initialize the global tracing subscriber and return the file-writer
|
||||
/// [`WorkerGuard`]. Bind it for the process lifetime so pending file writes
|
||||
/// flush on exit.
|
||||
///
|
||||
/// Panics if the log directory cannot be created or the rolling file appender
|
||||
/// cannot be initialized — both are fatal startup conditions worth failing
|
||||
/// fast on, before any real work begins.
|
||||
pub fn init(cfg: LogConfig) -> tracing_appender::non_blocking::WorkerGuard {
|
||||
std::fs::create_dir_all(&cfg.log_dir).unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"logging: failed to create log dir {}: {e}",
|
||||
cfg.log_dir.display()
|
||||
)
|
||||
});
|
||||
|
||||
// One global gate for all three layers. RUST_LOG wins; default `info`.
|
||||
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
||||
|
||||
let file_appender = tracing_appender::rolling::RollingFileAppender::builder()
|
||||
.rotation(tracing_appender::rolling::Rotation::DAILY)
|
||||
.filename_prefix(&cfg.file_prefix)
|
||||
.filename_suffix("log")
|
||||
.max_log_files(cfg.max_files)
|
||||
.build(&cfg.log_dir)
|
||||
.unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"logging: failed to init rolling file appender in {}: {e}",
|
||||
cfg.log_dir.display()
|
||||
)
|
||||
});
|
||||
let (file_writer, guard) = tracing_appender::non_blocking(file_appender);
|
||||
|
||||
// tracing's Level ordering is ERROR < WARN < INFO < DEBUG < TRACE, so
|
||||
// `>= INFO` selects the verbose side (INFO/DEBUG/TRACE) routed to stdout,
|
||||
// and `<= WARN` selects the severe side (WARN/ERROR) routed to stderr.
|
||||
let stdout_layer = fmt::layer()
|
||||
.with_writer(std::io::stdout)
|
||||
.with_filter(filter_fn(|m| *m.level() >= Level::INFO));
|
||||
|
||||
let stderr_layer = fmt::layer()
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter(filter_fn(|m| *m.level() <= Level::WARN));
|
||||
|
||||
let file_layer = fmt::layer().with_ansi(false).with_writer(file_writer);
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(stdout_layer)
|
||||
.with(stderr_layer)
|
||||
.with(file_layer)
|
||||
.init();
|
||||
|
||||
guard
|
||||
}
|
||||
@@ -1,322 +0,0 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Metrics {
|
||||
pub fuse_ops: FuseOpsMetrics,
|
||||
pub fuse_latency: FuseLatencyMetrics,
|
||||
pub cache: CacheMetrics,
|
||||
pub origins: OriginsMetrics,
|
||||
pub origin_health: OriginHealthMetrics,
|
||||
start_time: Option<Instant>,
|
||||
}
|
||||
|
||||
impl Metrics {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
start_time: Some(Instant::now()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn uptime_secs(&self) -> u64 {
|
||||
self.start_time.map(|t| t.elapsed().as_secs()).unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn to_prometheus(&self) -> String {
|
||||
let mut output = String::new();
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_fuse_ops_total Total FUSE operations\n\
|
||||
# TYPE musicfs_fuse_ops_total counter\n\
|
||||
musicfs_fuse_ops_total{{op=\"lookup\"}} {}\n\
|
||||
musicfs_fuse_ops_total{{op=\"getattr\"}} {}\n\
|
||||
musicfs_fuse_ops_total{{op=\"read\"}} {}\n\
|
||||
musicfs_fuse_ops_total{{op=\"readdir\"}} {}\n\
|
||||
musicfs_fuse_ops_total{{op=\"open\"}} {}\n",
|
||||
self.fuse_ops.lookup.load(Ordering::Relaxed),
|
||||
self.fuse_ops.getattr.load(Ordering::Relaxed),
|
||||
self.fuse_ops.read.load(Ordering::Relaxed),
|
||||
self.fuse_ops.readdir.load(Ordering::Relaxed),
|
||||
self.fuse_ops.open.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
for (op, histogram) in self.fuse_latency.histograms.read().iter() {
|
||||
let quantiles = histogram.quantiles();
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_fuse_latency_seconds FUSE operation latency\n\
|
||||
# TYPE musicfs_fuse_latency_seconds summary\n\
|
||||
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.5\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.95\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds{{op=\"{}\",quantile=\"0.99\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds_sum{{op=\"{}\"}} {:.6}\n\
|
||||
musicfs_fuse_latency_seconds_count{{op=\"{}\"}} {}\n",
|
||||
op,
|
||||
quantiles.p50,
|
||||
op,
|
||||
quantiles.p95,
|
||||
op,
|
||||
quantiles.p99,
|
||||
op,
|
||||
histogram.sum_secs(),
|
||||
op,
|
||||
histogram.count(),
|
||||
));
|
||||
}
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_cache_hits_total Cache hits\n\
|
||||
# TYPE musicfs_cache_hits_total counter\n\
|
||||
musicfs_cache_hits_total {}\n",
|
||||
self.cache.hits.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_cache_misses_total Cache misses\n\
|
||||
# TYPE musicfs_cache_misses_total counter\n\
|
||||
musicfs_cache_misses_total {}\n",
|
||||
self.cache.misses.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_cache_size_bytes Current cache size in bytes\n\
|
||||
# TYPE musicfs_cache_size_bytes gauge\n\
|
||||
musicfs_cache_size_bytes {}\n",
|
||||
self.cache.size_bytes.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_cache_chunks_total Number of cached chunks\n\
|
||||
# TYPE musicfs_cache_chunks_total gauge\n\
|
||||
musicfs_cache_chunks_total {}\n",
|
||||
self.cache.chunk_count.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
output.push_str(
|
||||
"# HELP musicfs_origin_health Origin health status (1=healthy, 0=unhealthy)\n\
|
||||
# TYPE musicfs_origin_health gauge\n",
|
||||
);
|
||||
for (origin_id, healthy) in self.origin_health.status.read().iter() {
|
||||
output.push_str(&format!(
|
||||
"musicfs_origin_health{{origin=\"{}\"}} {}\n",
|
||||
origin_id,
|
||||
if *healthy { 1 } else { 0 }
|
||||
));
|
||||
}
|
||||
|
||||
output.push_str(&format!(
|
||||
"# HELP musicfs_uptime_seconds Daemon uptime in seconds\n\
|
||||
# TYPE musicfs_uptime_seconds gauge\n\
|
||||
musicfs_uptime_seconds {}\n",
|
||||
self.uptime_secs(),
|
||||
));
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
pub fn hit_ratio(&self) -> f64 {
|
||||
let hits = self.cache.hits.load(Ordering::Relaxed) as f64;
|
||||
let misses = self.cache.misses.load(Ordering::Relaxed) as f64;
|
||||
let total = hits + misses;
|
||||
|
||||
if total == 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
hits / total
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct FuseOpsMetrics {
|
||||
pub lookup: AtomicU64,
|
||||
pub getattr: AtomicU64,
|
||||
pub read: AtomicU64,
|
||||
pub readdir: AtomicU64,
|
||||
pub open: AtomicU64,
|
||||
}
|
||||
|
||||
impl FuseOpsMetrics {
|
||||
pub fn record_lookup(&self) {
|
||||
self.lookup.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_getattr(&self) {
|
||||
self.getattr.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_read(&self) {
|
||||
self.read.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_readdir(&self) {
|
||||
self.readdir.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_open(&self) {
|
||||
self.open.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CacheMetrics {
|
||||
pub hits: AtomicU64,
|
||||
pub misses: AtomicU64,
|
||||
pub size_bytes: AtomicU64,
|
||||
pub chunk_count: AtomicU64,
|
||||
}
|
||||
|
||||
impl CacheMetrics {
|
||||
pub fn record_hit(&self) {
|
||||
self.hits.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_miss(&self) {
|
||||
self.misses.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn update_size(&self, size: u64) {
|
||||
self.size_bytes.store(size, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn update_chunk_count(&self, count: u64) {
|
||||
self.chunk_count.store(count, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OriginsMetrics {
|
||||
pub healthy_count: AtomicU64,
|
||||
pub total_count: AtomicU64,
|
||||
}
|
||||
|
||||
impl OriginsMetrics {
|
||||
pub fn update(&self, healthy: u64, total: u64) {
|
||||
self.healthy_count.store(healthy, Ordering::Relaxed);
|
||||
self.total_count.store(total, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct FuseLatencyMetrics {
|
||||
pub histograms: RwLock<HashMap<String, LatencyHistogram>>,
|
||||
}
|
||||
|
||||
impl FuseLatencyMetrics {
|
||||
pub fn record(&self, op: &str, latency_secs: f64) {
|
||||
let mut histograms = self.histograms.write();
|
||||
histograms
|
||||
.entry(op.to_string())
|
||||
.or_default()
|
||||
.record(latency_secs);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct LatencyHistogram {
|
||||
samples: Vec<f64>,
|
||||
sum: f64,
|
||||
}
|
||||
|
||||
impl LatencyHistogram {
|
||||
pub fn record(&mut self, latency_secs: f64) {
|
||||
self.samples.push(latency_secs);
|
||||
self.sum += latency_secs;
|
||||
|
||||
if self.samples.len() > 10000 {
|
||||
self.samples.drain(..5000);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn quantiles(&self) -> Quantiles {
|
||||
if self.samples.is_empty() {
|
||||
return Quantiles::default();
|
||||
}
|
||||
|
||||
let mut sorted = self.samples.clone();
|
||||
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
let len = sorted.len();
|
||||
Quantiles {
|
||||
p50: sorted[len / 2],
|
||||
p95: sorted[(len as f64 * 0.95) as usize],
|
||||
p99: sorted[(len as f64 * 0.99) as usize],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sum_secs(&self) -> f64 {
|
||||
self.sum
|
||||
}
|
||||
|
||||
pub fn count(&self) -> u64 {
|
||||
self.samples.len() as u64
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Quantiles {
|
||||
pub p50: f64,
|
||||
pub p95: f64,
|
||||
pub p99: f64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OriginHealthMetrics {
|
||||
pub status: RwLock<HashMap<String, bool>>,
|
||||
}
|
||||
|
||||
impl OriginHealthMetrics {
|
||||
pub fn set_health(&self, origin_id: &str, healthy: bool) {
|
||||
self.status.write().insert(origin_id.to_string(), healthy);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_metrics_new() {
|
||||
let metrics = Metrics::new();
|
||||
assert!(metrics.uptime_secs() < 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fuse_ops_recording() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.fuse_ops.record_lookup();
|
||||
metrics.fuse_ops.record_lookup();
|
||||
metrics.fuse_ops.record_read();
|
||||
|
||||
assert_eq!(metrics.fuse_ops.lookup.load(Ordering::Relaxed), 2);
|
||||
assert_eq!(metrics.fuse_ops.read.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit_ratio() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.cache.hits.store(8, Ordering::Relaxed);
|
||||
metrics.cache.misses.store(2, Ordering::Relaxed);
|
||||
|
||||
assert!((metrics.hit_ratio() - 0.8).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit_ratio_zero() {
|
||||
let metrics = Metrics::new();
|
||||
assert_eq!(metrics.hit_ratio(), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_prometheus_format() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.fuse_ops.record_lookup();
|
||||
metrics.cache.record_hit();
|
||||
|
||||
let output = metrics.to_prometheus();
|
||||
assert!(output.contains("musicfs_fuse_ops_total"));
|
||||
assert!(output.contains("musicfs_cache_hits_total"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
use std::path::Path;
|
||||
|
||||
use crate::music::flac::FlacMusicMetadataEncoder;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::music::mp3::Mp3MusicMetadataEncoder;
|
||||
|
||||
/// Bakes the current (possibly overridden) tag fields of a [`MusicMetadata`]
|
||||
/// into its in-memory `header`. The original media file is never modified;
|
||||
/// the rebuilt header is served on the fly at read time, ahead of the
|
||||
/// externalized frames and the original audio.
|
||||
pub trait MusicMetadataEncoder {
|
||||
fn encode(&self, metadata: &mut MusicMetadata);
|
||||
}
|
||||
|
||||
/// Selects the right [`MusicMetadataEncoder`] for a given file.
|
||||
pub struct MusicMetadataEncoderFactory;
|
||||
|
||||
impl MusicMetadataEncoderFactory {
|
||||
pub fn for_path(path: &Path) -> Option<Box<dyn MusicMetadataEncoder>> {
|
||||
match path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.as_deref()
|
||||
{
|
||||
Some("flac") => Some(Box::new(FlacMusicMetadataEncoder)),
|
||||
Some("mp3") => Some(Box::new(Mp3MusicMetadataEncoder)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
use std::{
|
||||
fs,
|
||||
io::{Cursor, Read, Seek, SeekFrom},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
use symphonia::core::{
|
||||
formats::FormatOptions, io::MediaSourceStream, meta::MetadataOptions, probe::Hint,
|
||||
};
|
||||
|
||||
use crate::music::encoder::MusicMetadataEncoder;
|
||||
use crate::music::metadata::{MusicMetadata, extract_standard_tags};
|
||||
use crate::music::parser::MusicMetadataParser;
|
||||
use tracing::warn;
|
||||
|
||||
const BLOCK_PADDING: u8 = 1;
|
||||
const BLOCK_VORBIS_COMMENT: u8 = 4;
|
||||
const BLOCK_PICTURE: u8 = 6;
|
||||
const BLOCK_LAST_FLAG: u8 = 0x80;
|
||||
const BLOCK_TYPE_MASK: u8 = 0x7f;
|
||||
const PADDING_SIZE: usize = 8192;
|
||||
|
||||
/// FLAC parser. Owns all FLAC container parsing; returns `None` (never panics)
|
||||
/// on a file it can't read — a corrupt or mid-copy file is logged and served
|
||||
/// as plain passthrough rather than taking down the snapshot/watcher.
|
||||
pub struct FlacMusicMetadataParser;
|
||||
|
||||
impl MusicMetadataParser for FlacMusicMetadataParser {
|
||||
fn parse(&self, path: &Path) -> Option<MusicMetadata> {
|
||||
match parse_flac_metadata(path) {
|
||||
Some(mm) => Some(mm),
|
||||
None => {
|
||||
warn!(
|
||||
path = %path.display(),
|
||||
"failed to parse FLAC metadata; serving as passthrough"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// FLAC encoder. Rebuilds the FLAC metadata header from the current tag fields,
|
||||
/// re-injecting the (possibly overridden) Vorbis comment.
|
||||
pub struct FlacMusicMetadataEncoder;
|
||||
|
||||
impl MusicMetadataEncoder for FlacMusicMetadataEncoder {
|
||||
fn encode(&self, metadata: &mut MusicMetadata) {
|
||||
if metadata.header.is_empty() {
|
||||
return;
|
||||
}
|
||||
let blocks = extract_non_vorbis_blocks(&metadata.header);
|
||||
let (header, vorbis_comment_offset, vorbis_comment_length) =
|
||||
build_flac_header(blocks, metadata);
|
||||
metadata.header = header;
|
||||
metadata.vorbis_comment_offset = vorbis_comment_offset;
|
||||
metadata.vorbis_comment_length = vorbis_comment_length;
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_flac_metadata(path: &Path) -> Option<MusicMetadata> {
|
||||
let src = fs::File::open(path).ok()?;
|
||||
let mss = MediaSourceStream::new(Box::new(src), Default::default());
|
||||
let mut hint = Hint::new();
|
||||
hint.with_extension("flac");
|
||||
|
||||
let meta_opts: MetadataOptions = Default::default();
|
||||
let fmt_opts: FormatOptions = Default::default();
|
||||
|
||||
let mut format = symphonia::default::get_probe()
|
||||
.format(&hint, mss, &fmt_opts, &meta_opts)
|
||||
.ok()?;
|
||||
|
||||
let metadata = format.format.metadata();
|
||||
|
||||
let mut music_metadata = MusicMetadata::default();
|
||||
if let Some(revision) = metadata.current() {
|
||||
extract_standard_tags(revision, &mut music_metadata);
|
||||
}
|
||||
|
||||
if let Some(parsed) = parse_flac(path) {
|
||||
music_metadata.real_audio_start = parsed.audio_start;
|
||||
music_metadata.picture_block_headers = parsed.picture_block_headers;
|
||||
music_metadata.picture_data_ranges = parsed.picture_data_ranges;
|
||||
let (header, vorbis_comment_offset, vorbis_comment_length) =
|
||||
build_flac_header(parsed.other_blocks, &music_metadata);
|
||||
music_metadata.header = header;
|
||||
music_metadata.vorbis_comment_offset = vorbis_comment_offset;
|
||||
music_metadata.vorbis_comment_length = vorbis_comment_length;
|
||||
}
|
||||
|
||||
Some(music_metadata)
|
||||
}
|
||||
|
||||
impl MusicMetadata {
|
||||
/// Locate the Vorbis comment block within the rebuilt FLAC header so writes
|
||||
/// to it can be intercepted. No-op for non-FLAC (e.g. ID3) headers.
|
||||
pub fn find_vorbis_offsets(&mut self) {
|
||||
let mut cursor = Cursor::new(&self.header);
|
||||
let mut magic = [0u8; 4];
|
||||
if cursor.read_exact(&mut magic).is_err() {
|
||||
return;
|
||||
}
|
||||
if &magic != b"fLaC" {
|
||||
return;
|
||||
}
|
||||
loop {
|
||||
let mut hdr = [0u8; 4];
|
||||
if cursor.read_exact(&mut hdr).is_err() {
|
||||
break;
|
||||
}
|
||||
let (is_last, block_type, length) = read_block_header(&hdr);
|
||||
if block_type == BLOCK_VORBIS_COMMENT {
|
||||
self.vorbis_comment_offset = cursor.position();
|
||||
self.vorbis_comment_length = length;
|
||||
return;
|
||||
}
|
||||
if cursor.seek(SeekFrom::Current(length as i64)).is_err() {
|
||||
break;
|
||||
}
|
||||
if is_last {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a freshly written Vorbis comment block and rebuild the header.
|
||||
pub fn update_from_vorbis_comment_data(&mut self, data: &[u8]) {
|
||||
parse_vorbis_comment_block(data, self);
|
||||
let other_blocks = extract_non_vorbis_blocks(&self.header);
|
||||
let (header, vorbis_comment_offset, vorbis_comment_length) =
|
||||
build_flac_header(other_blocks, self);
|
||||
self.header = header;
|
||||
self.vorbis_comment_offset = vorbis_comment_offset;
|
||||
self.vorbis_comment_length = vorbis_comment_length;
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a 4-byte FLAC metadata block header into (is_last, block_type, data_length).
|
||||
fn read_block_header(hdr: &[u8; 4]) -> (bool, u8, u64) {
|
||||
let is_last = (hdr[0] & BLOCK_LAST_FLAG) != 0;
|
||||
let block_type = hdr[0] & BLOCK_TYPE_MASK;
|
||||
let length = u32::from_be_bytes([0, hdr[1], hdr[2], hdr[3]]) as u64;
|
||||
(is_last, block_type, length)
|
||||
}
|
||||
|
||||
struct FlacParsed {
|
||||
other_blocks: Vec<(u8, Vec<u8>)>,
|
||||
picture_block_headers: Vec<Vec<u8>>,
|
||||
picture_data_ranges: Vec<(u64, u64)>,
|
||||
audio_start: u64,
|
||||
}
|
||||
|
||||
fn parse_flac(path: &Path) -> Option<FlacParsed> {
|
||||
let mut f = fs::File::open(path).ok()?;
|
||||
|
||||
let mut magic = [0u8; 4];
|
||||
f.read_exact(&mut magic).ok()?;
|
||||
if &magic != b"fLaC" {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut other_blocks: Vec<(u8, Vec<u8>)> = vec![];
|
||||
let mut picture_block_headers: Vec<Vec<u8>> = vec![];
|
||||
let mut picture_data_ranges: Vec<(u64, u64)> = vec![];
|
||||
let mut pos = 4u64;
|
||||
|
||||
loop {
|
||||
let mut hdr = [0u8; 4];
|
||||
f.read_exact(&mut hdr).ok()?;
|
||||
let (is_last, block_type, length) = read_block_header(&hdr);
|
||||
pos += 4;
|
||||
|
||||
if block_type == BLOCK_PICTURE {
|
||||
// PICTURE: keep block header (we'll fix is_last later), record data range
|
||||
picture_block_headers.push(hdr.to_vec());
|
||||
picture_data_ranges.push((pos, length));
|
||||
f.seek(SeekFrom::Current(length as i64)).ok()?;
|
||||
} else {
|
||||
let mut data = vec![0u8; length as usize];
|
||||
f.read_exact(&mut data).ok()?;
|
||||
// Skip VORBIS_COMMENT — we rebuild it
|
||||
if block_type != BLOCK_VORBIS_COMMENT {
|
||||
other_blocks.push((block_type, data));
|
||||
}
|
||||
}
|
||||
|
||||
pos += length;
|
||||
if is_last {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Fix is_last on the last picture block header: it must be 1 when audio follows
|
||||
if let Some(last_hdr) = picture_block_headers.last_mut() {
|
||||
last_hdr[0] = BLOCK_LAST_FLAG | (last_hdr[0] & BLOCK_TYPE_MASK);
|
||||
}
|
||||
|
||||
Some(FlacParsed {
|
||||
other_blocks,
|
||||
picture_block_headers,
|
||||
picture_data_ranges,
|
||||
audio_start: pos,
|
||||
})
|
||||
}
|
||||
|
||||
fn build_flac_header(blocks: Vec<(u8, Vec<u8>)>, metadata: &MusicMetadata) -> (Vec<u8>, u64, u64) {
|
||||
let vorbis = build_vorbis_comment(metadata);
|
||||
let vorbis_len = vorbis.len() as u64;
|
||||
|
||||
let mut patched: Vec<(u8, Vec<u8>)> = blocks;
|
||||
patched.push((BLOCK_VORBIS_COMMENT, vorbis));
|
||||
patched.push((BLOCK_PADDING, vec![0u8; PADDING_SIZE])); // allows metaflac in-place writes
|
||||
|
||||
let vorbis_idx = patched.len() - 2;
|
||||
let has_pictures = !metadata.picture_data_ranges.is_empty();
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(b"fLaC");
|
||||
|
||||
let last = patched.len() - 1;
|
||||
let mut vorbis_offset = 0u64;
|
||||
for (i, (block_type, data)) in patched.iter().enumerate() {
|
||||
let is_last_block = i == last && !has_pictures;
|
||||
let flag: u8 = if is_last_block { BLOCK_LAST_FLAG } else { 0x00 };
|
||||
let length = data.len() as u32;
|
||||
if i == vorbis_idx {
|
||||
vorbis_offset = out.len() as u64 + 4; // data starts after 4-byte block header
|
||||
}
|
||||
out.push(flag | block_type);
|
||||
out.push((length >> 16) as u8);
|
||||
out.push((length >> 8) as u8);
|
||||
out.push(length as u8);
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
|
||||
(out, vorbis_offset, vorbis_len)
|
||||
}
|
||||
|
||||
fn parse_vorbis_comment_block(data: &[u8], out: &mut MusicMetadata) {
|
||||
let mut cursor = Cursor::new(data);
|
||||
let mut len_bytes = [0u8; 4];
|
||||
|
||||
if cursor.read_exact(&mut len_bytes).is_err() {
|
||||
return;
|
||||
}
|
||||
let vendor_len = u32::from_le_bytes(len_bytes) as i64;
|
||||
if cursor.seek(SeekFrom::Current(vendor_len)).is_err() {
|
||||
return;
|
||||
}
|
||||
if cursor.read_exact(&mut len_bytes).is_err() {
|
||||
return;
|
||||
}
|
||||
let count = u32::from_le_bytes(len_bytes);
|
||||
|
||||
out.artist.clear();
|
||||
out.other_tags.clear();
|
||||
|
||||
for _ in 0..count {
|
||||
if cursor.read_exact(&mut len_bytes).is_err() {
|
||||
break;
|
||||
}
|
||||
let comment_len = u32::from_le_bytes(len_bytes) as usize;
|
||||
let mut comment_bytes = vec![0u8; comment_len];
|
||||
if cursor.read_exact(&mut comment_bytes).is_err() {
|
||||
break;
|
||||
}
|
||||
let comment = String::from_utf8_lossy(&comment_bytes).into_owned();
|
||||
if let Some((key, value)) = comment.split_once('=') {
|
||||
match key.to_ascii_uppercase().as_str() {
|
||||
"TITLE" => out.track_title = value.to_string(),
|
||||
"ALBUM" => out.album = value.to_string(),
|
||||
"TRACKNUMBER" => out.track_number = value.parse().unwrap_or(0),
|
||||
"ARTIST" => out.artist.push(value.to_string()),
|
||||
_ => out.other_tags.push(comment),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_non_vorbis_blocks(header: &[u8]) -> Vec<(u8, Vec<u8>)> {
|
||||
let mut cursor = Cursor::new(header);
|
||||
let mut blocks = vec![];
|
||||
|
||||
let mut magic = [0u8; 4];
|
||||
if cursor.read_exact(&mut magic).is_err() {
|
||||
return blocks;
|
||||
}
|
||||
|
||||
loop {
|
||||
let mut hdr = [0u8; 4];
|
||||
if cursor.read_exact(&mut hdr).is_err() {
|
||||
break;
|
||||
}
|
||||
let (is_last, block_type, length) = read_block_header(&hdr);
|
||||
let mut data = vec![0u8; length as usize];
|
||||
if cursor.read_exact(&mut data).is_err() {
|
||||
break;
|
||||
}
|
||||
if block_type != BLOCK_VORBIS_COMMENT && block_type != BLOCK_PADDING {
|
||||
blocks.push((block_type, data));
|
||||
}
|
||||
if is_last {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
blocks
|
||||
}
|
||||
|
||||
fn build_vorbis_comment(metadata: &MusicMetadata) -> Vec<u8> {
|
||||
let vendor = b"musicfs";
|
||||
let mut out = Vec::new();
|
||||
|
||||
out.extend_from_slice(&(vendor.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(vendor);
|
||||
|
||||
let mut comments: Vec<String> = vec![
|
||||
format!("TITLE={}", metadata.track_title),
|
||||
format!("ALBUM={}", metadata.album),
|
||||
format!("TRACKNUMBER={}", metadata.track_number),
|
||||
];
|
||||
for artist in &metadata.artist {
|
||||
comments.push(format!("ARTIST={}", artist));
|
||||
}
|
||||
comments.extend(metadata.other_tags.iter().cloned());
|
||||
|
||||
out.extend_from_slice(&(comments.len() as u32).to_le_bytes());
|
||||
for comment in &comments {
|
||||
let bytes = comment.as_bytes();
|
||||
out.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(bytes);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn read_block_header_normal() {
|
||||
let hdr = [0x04, 0x00, 0x01, 0x00];
|
||||
let (is_last, block_type, length) = read_block_header(&hdr);
|
||||
assert!(!is_last);
|
||||
assert_eq!(block_type, 4);
|
||||
assert_eq!(length, 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_block_header_last() {
|
||||
let hdr = [0x84, 0x00, 0x00, 0x10];
|
||||
let (is_last, block_type, length) = read_block_header(&hdr);
|
||||
assert!(is_last);
|
||||
assert_eq!(block_type, 4);
|
||||
assert_eq!(length, 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_vorbis_comment_roundtrip() {
|
||||
let mut metadata = MusicMetadata::default();
|
||||
metadata.artist = vec!["Artist1".to_string(), "Artist2".to_string()];
|
||||
metadata.album = "Album".to_string();
|
||||
metadata.track_number = 3;
|
||||
metadata.track_title = "Title".to_string();
|
||||
metadata.other_tags = vec!["GENRE=Rock".to_string()];
|
||||
|
||||
let vorbis_bytes = build_vorbis_comment(&metadata);
|
||||
let mut parsed = MusicMetadata::default();
|
||||
parse_vorbis_comment_block(&vorbis_bytes, &mut parsed);
|
||||
|
||||
assert_eq!(parsed.artist, vec!["Artist1", "Artist2"]);
|
||||
assert_eq!(parsed.album, "Album");
|
||||
assert_eq!(parsed.track_number, 3);
|
||||
assert_eq!(parsed.track_title, "Title");
|
||||
assert_eq!(parsed.other_tags, vec!["GENRE=Rock"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_vorbis_comment_multi_artist() {
|
||||
let mut vorbis_bytes = Vec::new();
|
||||
let vendor = b"test";
|
||||
vorbis_bytes.extend_from_slice(&(vendor.len() as u32).to_le_bytes());
|
||||
vorbis_bytes.extend_from_slice(vendor);
|
||||
|
||||
let comments = vec!["ARTIST=Artist1", "ARTIST=Artist2"];
|
||||
vorbis_bytes.extend_from_slice(&(comments.len() as u32).to_le_bytes());
|
||||
for comment in &comments {
|
||||
let bytes = comment.as_bytes();
|
||||
vorbis_bytes.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
|
||||
vorbis_bytes.extend_from_slice(bytes);
|
||||
}
|
||||
|
||||
let mut metadata = MusicMetadata::default();
|
||||
parse_vorbis_comment_block(&vorbis_bytes, &mut metadata);
|
||||
|
||||
assert_eq!(metadata.artist.len(), 2);
|
||||
assert_eq!(metadata.artist[0], "Artist1");
|
||||
assert_eq!(metadata.artist[1], "Artist2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_non_vorbis_blocks_strips_vc_and_padding() {
|
||||
let mut header = Vec::new();
|
||||
header.extend_from_slice(b"fLaC");
|
||||
|
||||
let streaminfo_data = vec![0u8; 34];
|
||||
header.push(0x00);
|
||||
header.extend_from_slice(&[0x00, 0x00, 0x22]);
|
||||
header.extend_from_slice(&streaminfo_data);
|
||||
|
||||
let vorbis_data = vec![0u8; 50];
|
||||
header.push(0x04);
|
||||
header.extend_from_slice(&[0x00, 0x00, 0x32]);
|
||||
header.extend_from_slice(&vorbis_data);
|
||||
|
||||
let padding_data = vec![0u8; 100];
|
||||
header.push(0x81);
|
||||
header.extend_from_slice(&[0x00, 0x00, 0x64]);
|
||||
header.extend_from_slice(&padding_data);
|
||||
|
||||
let blocks = extract_non_vorbis_blocks(&header);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].0, 0);
|
||||
assert_eq!(blocks[0].1.len(), 34);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_vorbis_offsets_correct() {
|
||||
let mut metadata = MusicMetadata::default();
|
||||
metadata.artist = vec!["TestArtist".to_string()];
|
||||
metadata.album = "TestAlbum".to_string();
|
||||
metadata.track_number = 1;
|
||||
metadata.track_title = "TestTitle".to_string();
|
||||
|
||||
let other_blocks = vec![(0, vec![0u8; 34])];
|
||||
let (header, expected_offset, expected_length) = build_flac_header(other_blocks, &metadata);
|
||||
metadata.header = header;
|
||||
|
||||
metadata.find_vorbis_offsets();
|
||||
|
||||
assert_eq!(metadata.vorbis_comment_offset, expected_offset);
|
||||
assert_eq!(metadata.vorbis_comment_length, expected_length);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use symphonia::core::meta::{MetadataRevision, StandardTagKey};
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub struct MusicMetadata {
|
||||
pub artist: Vec<String>,
|
||||
pub album_artist: Option<String>,
|
||||
pub album: String,
|
||||
pub track_number: i32,
|
||||
pub track_title: String,
|
||||
pub other_tags: Vec<String>,
|
||||
pub header: Vec<u8>,
|
||||
pub picture_block_headers: Vec<Vec<u8>>,
|
||||
pub picture_data_ranges: Vec<(u64, u64)>,
|
||||
pub real_audio_start: u64,
|
||||
pub vorbis_comment_offset: u64,
|
||||
pub vorbis_comment_length: u64,
|
||||
}
|
||||
|
||||
impl MusicMetadata {
|
||||
pub fn virtual_size(&self, real_file_size: u64) -> u64 {
|
||||
let pictures_size: u64 = self
|
||||
.picture_block_headers
|
||||
.iter()
|
||||
.zip(self.picture_data_ranges.iter())
|
||||
.map(|(prefix, (_, len))| prefix.len() as u64 + len)
|
||||
.sum();
|
||||
self.header.len() as u64 + pictures_size + (real_file_size - self.real_audio_start)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn extract_standard_tags(revision: &MetadataRevision, out: &mut MusicMetadata) {
|
||||
for tag in revision.tags() {
|
||||
let value = tag.value.to_string();
|
||||
match tag.std_key {
|
||||
Some(StandardTagKey::Artist) => out.artist.push(value),
|
||||
Some(StandardTagKey::AlbumArtist) => out.album_artist = Some(value),
|
||||
Some(StandardTagKey::Album) => out.album = value,
|
||||
Some(StandardTagKey::TrackNumber) => {
|
||||
out.track_number = value.parse::<i32>().unwrap_or(0)
|
||||
}
|
||||
Some(StandardTagKey::TrackTitle) => out.track_title = value,
|
||||
_ => out.other_tags.push(format!("{}={}", tag.key, value)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn virtual_size_calculation() {
|
||||
let mut metadata = MusicMetadata::default();
|
||||
metadata.header = vec![0u8; 100];
|
||||
metadata.picture_block_headers = vec![vec![0u8; 4]];
|
||||
metadata.picture_data_ranges = vec![(0, 50)];
|
||||
metadata.real_audio_start = 200;
|
||||
let file_size = 1000u64;
|
||||
let virtual_size = metadata.virtual_size(file_size);
|
||||
assert_eq!(virtual_size, 954);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
pub mod encoder;
|
||||
pub mod flac;
|
||||
pub mod metadata;
|
||||
pub mod mp3;
|
||||
pub mod parse;
|
||||
pub mod parser;
|
||||
@@ -0,0 +1,507 @@
|
||||
use std::{
|
||||
fs,
|
||||
io::{Read, Seek, SeekFrom},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
use symphonia::core::{
|
||||
formats::FormatOptions, io::MediaSourceStream, meta::MetadataOptions, probe::Hint,
|
||||
};
|
||||
|
||||
use crate::music::encoder::MusicMetadataEncoder;
|
||||
use crate::music::metadata::{MusicMetadata, extract_standard_tags};
|
||||
use crate::music::parser::MusicMetadataParser;
|
||||
use tracing::warn;
|
||||
|
||||
/// Frames we replace from our own tag fields. Every other frame in the source
|
||||
/// ID3 tag is preserved verbatim via externalization.
|
||||
const OVERRIDE_FRAME_IDS: [[u8; 4]; 4] = [*b"TIT2", *b"TALB", *b"TPE1", *b"TRCK"];
|
||||
|
||||
/// MP3 parser. Reads ID3 tags (via symphonia) and locates where the audio
|
||||
/// frames begin (end of the ID3v2 tag).
|
||||
pub struct Mp3MusicMetadataParser;
|
||||
|
||||
impl MusicMetadataParser for Mp3MusicMetadataParser {
|
||||
fn parse(&self, path: &Path) -> Option<MusicMetadata> {
|
||||
match parse_mp3_metadata(path) {
|
||||
Some(mm) => Some(mm),
|
||||
None => {
|
||||
warn!(
|
||||
path = %path.display(),
|
||||
"failed to parse MP3 metadata; serving as passthrough"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode a 28-bit syncsafe integer (7 bits per byte) as used by ID3v2 sizes.
|
||||
fn syncsafe(b: [u8; 4]) -> u64 {
|
||||
((b[0] as u64) << 21) | ((b[1] as u64) << 14) | ((b[2] as u64) << 7) | (b[3] as u64)
|
||||
}
|
||||
|
||||
/// Byte offset where the MP3 audio frames begin: the end of the ID3v2 tag, or
|
||||
/// 0 when there is no tag. Reads only the 10-byte ID3 header.
|
||||
fn id3_audio_start(path: &Path) -> u64 {
|
||||
let mut f = match fs::File::open(path) {
|
||||
Ok(f) => f,
|
||||
Err(_) => return 0,
|
||||
};
|
||||
let mut hdr = [0u8; 10];
|
||||
if f.read_exact(&mut hdr).is_err() {
|
||||
return 0;
|
||||
}
|
||||
if &hdr[0..3] != b"ID3" {
|
||||
return 0;
|
||||
}
|
||||
let size = syncsafe([hdr[6], hdr[7], hdr[8], hdr[9]]);
|
||||
// ID3v2.4 footer flag adds another 10 bytes after the tag body.
|
||||
let footer = if hdr[5] & 0x10 != 0 { 10 } else { 0 };
|
||||
10 + size + footer
|
||||
}
|
||||
|
||||
fn parse_mp3_metadata(path: &Path) -> Option<MusicMetadata> {
|
||||
let src = fs::File::open(path).ok()?;
|
||||
let mss = MediaSourceStream::new(Box::new(src), Default::default());
|
||||
let mut hint = Hint::new();
|
||||
hint.with_extension("mp3");
|
||||
|
||||
let meta_opts: MetadataOptions = Default::default();
|
||||
let fmt_opts: FormatOptions = Default::default();
|
||||
|
||||
let mut probed = symphonia::default::get_probe()
|
||||
.format(&hint, mss, &fmt_opts, &meta_opts)
|
||||
.ok()?;
|
||||
|
||||
let mut music_metadata = MusicMetadata::default();
|
||||
|
||||
// ID3v2 tags at the start of the file surface in the probe-level metadata;
|
||||
// fall back to the in-stream metadata otherwise.
|
||||
let mut got_tags = false;
|
||||
if let Some(metadata) = probed.metadata.get() {
|
||||
if let Some(revision) = metadata.current() {
|
||||
extract_standard_tags(revision, &mut music_metadata);
|
||||
got_tags = true;
|
||||
}
|
||||
}
|
||||
if !got_tags {
|
||||
let metadata = probed.format.metadata();
|
||||
if let Some(revision) = metadata.current() {
|
||||
extract_standard_tags(revision, &mut music_metadata);
|
||||
}
|
||||
}
|
||||
|
||||
music_metadata.real_audio_start = id3_audio_start(path);
|
||||
|
||||
// Record the original frames we will preserve (cover art, lyrics, …) so the
|
||||
// encoder can stitch them back in from the original file at read time.
|
||||
let (prefixes, ranges) = parse_id3_preserved_frames(path);
|
||||
music_metadata.picture_block_headers = prefixes;
|
||||
music_metadata.picture_data_ranges = ranges;
|
||||
|
||||
// Untagged files would otherwise build a degenerate empty artist/album path.
|
||||
if music_metadata.artist.is_empty() {
|
||||
music_metadata.artist = vec!["Unknown Artist".to_string()];
|
||||
}
|
||||
if music_metadata.album.is_empty() {
|
||||
music_metadata.album = "Unknown Album".to_string();
|
||||
}
|
||||
|
||||
Some(music_metadata)
|
||||
}
|
||||
|
||||
impl MusicMetadata {
|
||||
/// Apply a freshly written ID3v2 tag and rebuild the in-memory header.
|
||||
pub fn update_from_id3_data(&mut self, data: &[u8]) {
|
||||
parse_id3_tag_frames(data, self);
|
||||
self.header = build_id3v2_header(self);
|
||||
}
|
||||
|
||||
/// Apply a freshly written ID3v1 tag (128-byte "TAG" block) and rebuild
|
||||
/// the in-memory ID3v2 header. ID3v1 is Latin-1, max 30 chars per field;
|
||||
/// non-ASCII bytes are replaced with '?' since Latin-1 ≠ UTF-8.
|
||||
pub fn update_from_id3v1_data(&mut self, data: &[u8]) {
|
||||
if data.len() < 128 || &data[0..3] != b"TAG" {
|
||||
return;
|
||||
}
|
||||
fn latin1_str(bytes: &[u8]) -> Option<String> {
|
||||
let end = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len());
|
||||
if end == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
bytes[..end]
|
||||
.iter()
|
||||
.map(|&b| if b < 0x80 { char::from(b) } else { '?' })
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
if let Some(t) = latin1_str(&data[3..33]) {
|
||||
self.track_title = t;
|
||||
}
|
||||
if let Some(a) = latin1_str(&data[33..63]) {
|
||||
self.artist = vec![a];
|
||||
}
|
||||
if let Some(a) = latin1_str(&data[63..93]) {
|
||||
self.album = a;
|
||||
}
|
||||
// ID3v1.1: byte 125 == 0 means byte 126 is the track number
|
||||
if data[125] == 0 && data[126] != 0 {
|
||||
self.track_number = data[126] as i32;
|
||||
}
|
||||
self.header = build_id3v2_header(self);
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_id3_text(data: &[u8]) -> String {
|
||||
if data.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
match data[0] {
|
||||
0x03 => String::from_utf8_lossy(&data[1..])
|
||||
.trim_end_matches('\0')
|
||||
.to_string(),
|
||||
0x00 => data[1..]
|
||||
.iter()
|
||||
.take_while(|&&b| b != 0)
|
||||
.map(|&b| char::from(b))
|
||||
.collect(),
|
||||
0x01 | 0x02 => {
|
||||
let raw = &data[1..];
|
||||
let (src, le) = if raw.len() >= 2 && raw[0] == 0xFF && raw[1] == 0xFE {
|
||||
(&raw[2..], true)
|
||||
} else if raw.len() >= 2 && raw[0] == 0xFE && raw[1] == 0xFF {
|
||||
(&raw[2..], false)
|
||||
} else {
|
||||
(raw, true)
|
||||
};
|
||||
let words: Vec<u16> = src
|
||||
.chunks_exact(2)
|
||||
.map(|c| {
|
||||
if le {
|
||||
u16::from_le_bytes([c[0], c[1]])
|
||||
} else {
|
||||
u16::from_be_bytes([c[0], c[1]])
|
||||
}
|
||||
})
|
||||
.take_while(|&w| w != 0)
|
||||
.collect();
|
||||
String::from_utf16_lossy(&words)
|
||||
}
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_id3_tag_frames(data: &[u8], out: &mut MusicMetadata) {
|
||||
if data.len() < 10 || &data[0..3] != b"ID3" {
|
||||
return;
|
||||
}
|
||||
let version = data[3];
|
||||
if data[5] & 0x80 != 0 {
|
||||
return; // unsynchronised — can't copy frames verbatim
|
||||
}
|
||||
let tag_end = (10 + syncsafe([data[6], data[7], data[8], data[9]]) as usize).min(data.len());
|
||||
|
||||
out.artist.clear();
|
||||
let mut offset = 10usize;
|
||||
while offset + 10 <= tag_end {
|
||||
if data[offset..offset + 4].iter().all(|&b| b == 0) {
|
||||
break; // padding
|
||||
}
|
||||
let size = if version >= 4 {
|
||||
syncsafe([
|
||||
data[offset + 4],
|
||||
data[offset + 5],
|
||||
data[offset + 6],
|
||||
data[offset + 7],
|
||||
]) as usize
|
||||
} else {
|
||||
u32::from_be_bytes([
|
||||
data[offset + 4],
|
||||
data[offset + 5],
|
||||
data[offset + 6],
|
||||
data[offset + 7],
|
||||
]) as usize
|
||||
};
|
||||
if size == 0 || offset + 10 + size > tag_end {
|
||||
break;
|
||||
}
|
||||
let body = &data[offset + 10..offset + 10 + size];
|
||||
match &data[offset..offset + 4] {
|
||||
b"TIT2" => out.track_title = decode_id3_text(body),
|
||||
b"TALB" => out.album = decode_id3_text(body),
|
||||
b"TPE1" => {
|
||||
let text = decode_id3_text(body);
|
||||
out.artist = text
|
||||
.split('\0')
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
}
|
||||
b"TRCK" => {
|
||||
let text = decode_id3_text(body);
|
||||
out.track_number = text
|
||||
.split('/')
|
||||
.next()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
offset += 10 + size;
|
||||
}
|
||||
}
|
||||
|
||||
/// MP3 encoder. Builds a fresh ID3v2.3 tag from the current (possibly
|
||||
/// overridden) tag fields. Preserved frames are not copied into the header;
|
||||
/// their lengths are counted so the tag's size field spans them.
|
||||
pub struct Mp3MusicMetadataEncoder;
|
||||
|
||||
impl MusicMetadataEncoder for Mp3MusicMetadataEncoder {
|
||||
fn encode(&self, metadata: &mut MusicMetadata) {
|
||||
metadata.header = build_id3v2_header(metadata);
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a 28-bit syncsafe integer (7 bits per byte) for ID3v2 size fields.
|
||||
fn syncsafe_encode(n: u64) -> [u8; 4] {
|
||||
[
|
||||
((n >> 21) & 0x7f) as u8,
|
||||
((n >> 14) & 0x7f) as u8,
|
||||
((n >> 7) & 0x7f) as u8,
|
||||
(n & 0x7f) as u8,
|
||||
]
|
||||
}
|
||||
|
||||
/// Build a single UTF-16 ID3v2.3 text frame.
|
||||
fn build_id3_text_frame(id: &[u8; 4], text: &str) -> Vec<u8> {
|
||||
let mut body = Vec::with_capacity(3 + text.len() * 2);
|
||||
body.push(0x01); // text encoding: UTF-16 with BOM
|
||||
body.extend_from_slice(&[0xFF, 0xFE]); // little-endian BOM
|
||||
for unit in text.encode_utf16() {
|
||||
body.extend_from_slice(&unit.to_le_bytes());
|
||||
}
|
||||
|
||||
let mut frame = Vec::with_capacity(10 + body.len());
|
||||
frame.extend_from_slice(id);
|
||||
frame.extend_from_slice(&(body.len() as u32).to_be_bytes());
|
||||
frame.extend_from_slice(&[0, 0]); // frame flags
|
||||
frame.extend_from_slice(&body);
|
||||
frame
|
||||
}
|
||||
|
||||
/// Build the in-memory portion of the virtual ID3v2.3 tag: the 10-byte tag
|
||||
/// header plus our four override frames. Preserved frames and audio are
|
||||
/// stitched in by the read path; the tag size field accounts for them.
|
||||
fn build_id3v2_header(m: &MusicMetadata) -> Vec<u8> {
|
||||
let mut frames = Vec::new();
|
||||
frames.extend(build_id3_text_frame(b"TIT2", &m.track_title));
|
||||
frames.extend(build_id3_text_frame(b"TALB", &m.album));
|
||||
frames.extend(build_id3_text_frame(b"TPE1", &m.artist.join("\0")));
|
||||
frames.extend(build_id3_text_frame(b"TRCK", &m.track_number.to_string()));
|
||||
|
||||
let preserved_len: u64 = m
|
||||
.picture_block_headers
|
||||
.iter()
|
||||
.zip(m.picture_data_ranges.iter())
|
||||
.map(|(prefix, (_, len))| prefix.len() as u64 + len)
|
||||
.sum();
|
||||
|
||||
let body_len = frames.len() as u64 + preserved_len;
|
||||
|
||||
let mut header = Vec::with_capacity(10 + frames.len());
|
||||
header.extend_from_slice(b"ID3");
|
||||
header.extend_from_slice(&[0x03, 0x00, 0x00]); // v2.3.0, no flags
|
||||
header.extend_from_slice(&syncsafe_encode(body_len));
|
||||
header.extend_from_slice(&frames);
|
||||
header
|
||||
}
|
||||
|
||||
/// Walk the source ID3v2 tag and record every frame we do NOT override
|
||||
/// (cover art, lyrics, other text frames) as a whole-frame range in the
|
||||
/// original file. Each gets an empty in-memory prefix — the frame is copied
|
||||
/// verbatim from disk at read time. Returns empty lists when there is no tag
|
||||
/// or the tag is unsynchronised (which can't be externalized verbatim).
|
||||
fn parse_id3_preserved_frames(path: &Path) -> (Vec<Vec<u8>>, Vec<(u64, u64)>) {
|
||||
let mut prefixes: Vec<Vec<u8>> = Vec::new();
|
||||
let mut ranges: Vec<(u64, u64)> = Vec::new();
|
||||
|
||||
let mut f = match fs::File::open(path) {
|
||||
Ok(f) => f,
|
||||
Err(_) => return (prefixes, ranges),
|
||||
};
|
||||
let mut hdr = [0u8; 10];
|
||||
if f.read_exact(&mut hdr).is_err() || &hdr[0..3] != b"ID3" {
|
||||
return (prefixes, ranges);
|
||||
}
|
||||
let version = hdr[3];
|
||||
// Unsynchronised tags store bytes we can't copy verbatim; fall back to
|
||||
// passthrough by preserving nothing (the whole original tag is dropped,
|
||||
// but that only loses art on a rare encoding — acceptable for v1).
|
||||
if hdr[5] & 0x80 != 0 {
|
||||
return (prefixes, ranges);
|
||||
}
|
||||
|
||||
let tag_end = id3_audio_start(path);
|
||||
let mut offset: u64 = 10;
|
||||
while offset + 10 <= tag_end {
|
||||
if f.seek(SeekFrom::Start(offset)).is_err() {
|
||||
break;
|
||||
}
|
||||
let mut fh = [0u8; 10];
|
||||
if f.read_exact(&mut fh).is_err() {
|
||||
break;
|
||||
}
|
||||
// A zero frame id marks the start of padding.
|
||||
if fh[0..4].iter().all(|&b| b == 0) {
|
||||
break;
|
||||
}
|
||||
let size = if version >= 4 {
|
||||
syncsafe([fh[4], fh[5], fh[6], fh[7]])
|
||||
} else {
|
||||
u32::from_be_bytes([fh[4], fh[5], fh[6], fh[7]]) as u64
|
||||
};
|
||||
let frame_total = 10 + size;
|
||||
if size == 0 || offset + frame_total > tag_end {
|
||||
break;
|
||||
}
|
||||
let id = &fh[0..4];
|
||||
let is_override = OVERRIDE_FRAME_IDS.iter().any(|o| &o[..] == id);
|
||||
if !is_override {
|
||||
prefixes.push(Vec::new());
|
||||
ranges.push((offset, frame_total));
|
||||
}
|
||||
offset += frame_total;
|
||||
}
|
||||
|
||||
(prefixes, ranges)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[test]
|
||||
fn syncsafe_decodes_seven_bit_groups() {
|
||||
assert_eq!(syncsafe([0, 0, 0, 0x23]), 35);
|
||||
assert_eq!(syncsafe([0, 0, 1, 0]), 128);
|
||||
}
|
||||
|
||||
fn write_temp(bytes: &[u8]) -> tempfile::NamedTempFile {
|
||||
let mut f = tempfile::NamedTempFile::new().unwrap();
|
||||
f.write_all(bytes).unwrap();
|
||||
f.flush().unwrap();
|
||||
f
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn id3_audio_start_after_tag() {
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(b"ID3");
|
||||
buf.extend_from_slice(&[0x04, 0x00, 0x00]); // version + flags
|
||||
buf.extend_from_slice(&[0x00, 0x00, 0x00, 0x23]); // syncsafe size = 35
|
||||
buf.extend_from_slice(&[0u8; 35]); // tag body
|
||||
let f = write_temp(&buf);
|
||||
assert_eq!(id3_audio_start(f.path()), 45);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn id3_audio_start_with_footer() {
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(b"ID3");
|
||||
buf.extend_from_slice(&[0x04, 0x00, 0x10]); // footer flag set
|
||||
buf.extend_from_slice(&[0x00, 0x00, 0x00, 0x23]); // size = 35
|
||||
buf.extend_from_slice(&[0u8; 45]);
|
||||
let f = write_temp(&buf);
|
||||
assert_eq!(id3_audio_start(f.path()), 55);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn id3_audio_start_no_tag() {
|
||||
// Raw MP3 frame sync, no ID3 tag.
|
||||
let f = write_temp(&[0xFF, 0xFB, 0x40, 0xC0, 0x00, 0x00]);
|
||||
assert_eq!(id3_audio_start(f.path()), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn syncsafe_encode_inverts_syncsafe() {
|
||||
for n in [0u64, 35, 128, 30696, 0x0FFF_FFFF] {
|
||||
assert_eq!(syncsafe(syncsafe_encode(n)), n);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_id3v2_header_structure() {
|
||||
let mm = MusicMetadata {
|
||||
track_title: "Title".to_string(),
|
||||
album: "Album".to_string(),
|
||||
artist: vec!["A".to_string(), "B".to_string()],
|
||||
track_number: 7,
|
||||
// one preserved frame of total length 20
|
||||
picture_block_headers: vec![Vec::new()],
|
||||
picture_data_ranges: vec![(100, 20)],
|
||||
..MusicMetadata::default()
|
||||
};
|
||||
let header = build_id3v2_header(&mm);
|
||||
|
||||
assert_eq!(&header[0..3], b"ID3");
|
||||
assert_eq!(&header[3..6], &[0x03, 0x00, 0x00]);
|
||||
|
||||
let frames_len = header.len() as u64 - 10;
|
||||
let declared = syncsafe([header[6], header[7], header[8], header[9]]);
|
||||
// size field spans our frames + the preserved frame's 20 bytes
|
||||
assert_eq!(declared, frames_len + 20);
|
||||
|
||||
assert!(header.windows(4).any(|w| w == b"TIT2"));
|
||||
assert!(header.windows(4).any(|w| w == b"TPE1"));
|
||||
}
|
||||
|
||||
fn id3_frame(id: &[u8; 4], data: &[u8], syncsafe_size: bool) -> Vec<u8> {
|
||||
let mut v = Vec::new();
|
||||
v.extend_from_slice(id);
|
||||
if syncsafe_size {
|
||||
v.extend_from_slice(&syncsafe_encode(data.len() as u64));
|
||||
} else {
|
||||
v.extend_from_slice(&(data.len() as u32).to_be_bytes());
|
||||
}
|
||||
v.extend_from_slice(&[0, 0]); // flags
|
||||
v.extend_from_slice(data);
|
||||
v
|
||||
}
|
||||
|
||||
fn build_tag(version: u8, frames: &[u8]) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(b"ID3");
|
||||
buf.extend_from_slice(&[version, 0x00, 0x00]);
|
||||
buf.extend_from_slice(&syncsafe_encode(frames.len() as u64));
|
||||
buf.extend_from_slice(frames);
|
||||
buf
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserved_frames_v23_keeps_apic_drops_overrides() {
|
||||
let mut frames = Vec::new();
|
||||
frames.extend(id3_frame(b"APIC", &[1, 2, 3, 4, 5], false));
|
||||
frames.extend(id3_frame(b"TIT2", &[0x03, b'h', b'i'], false));
|
||||
let f = write_temp(&build_tag(0x03, &frames));
|
||||
|
||||
let (prefixes, ranges) = parse_id3_preserved_frames(f.path());
|
||||
// APIC starts right after the 10-byte tag header; total = 10 + 5
|
||||
assert_eq!(ranges, vec![(10, 15)]);
|
||||
assert_eq!(prefixes, vec![Vec::<u8>::new()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserved_frames_v24_syncsafe_sizes() {
|
||||
let mut frames = Vec::new();
|
||||
frames.extend(id3_frame(b"USLT", &[9, 9, 9], true));
|
||||
frames.extend(id3_frame(b"TALB", &[0x03, b'x'], true));
|
||||
let f = write_temp(&build_tag(0x04, &frames));
|
||||
|
||||
let (prefixes, ranges) = parse_id3_preserved_frames(f.path());
|
||||
assert_eq!(ranges, vec![(10, 13)]); // USLT: 10 + 3
|
||||
assert_eq!(prefixes, vec![Vec::<u8>::new()]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
use std::path::Path;
|
||||
|
||||
use tracing::{debug, trace};
|
||||
|
||||
use crate::music::encoder::MusicMetadataEncoderFactory;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::music::parser::MusicMetadataParserFactory;
|
||||
|
||||
/// Parse a single media file into a fully-encoded [`MusicMetadata`].
|
||||
///
|
||||
/// `None` is returned for files musicfs does not treat as music (unknown
|
||||
/// extension or unparseable container). When metadata is produced, the
|
||||
/// matching [`MusicMetadataEncoder`] has already baked the tag fields into
|
||||
/// the in-memory `header`, so callers can serve virtualized bytes directly.
|
||||
///
|
||||
/// A music-extension file that fails to parse is logged at `warn!` by the
|
||||
/// selected parser (see [`crate::music::flac`] / [`crate::music::mp3`]); this
|
||||
/// function simply propagates the resulting `None` without re-logging.
|
||||
///
|
||||
/// This helper is the single source of truth for "parse + encode" — used by
|
||||
/// the local FUSE origin's snapshot builder, the network origin's manifest
|
||||
/// builder, and the server's manifest endpoint.
|
||||
pub fn parse_music_metadata_for_path(path: &Path) -> Option<MusicMetadata> {
|
||||
let parser = match MusicMetadataParserFactory::for_path(path) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
trace!(path = %path.display(), "non-music extension; skipping");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
debug!(path = %path.display(), "music parser selected");
|
||||
let mut metadata = parser.parse(path)?;
|
||||
if let Some(encoder) = MusicMetadataEncoderFactory::for_path(path) {
|
||||
encoder.encode(&mut metadata);
|
||||
}
|
||||
return Some(metadata);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::Path;
|
||||
|
||||
#[test]
|
||||
fn none_for_non_music_extension() {
|
||||
let path = Path::new("notes.txt");
|
||||
assert!(parse_music_metadata_for_path(path).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_for_missing_file_with_music_extension() {
|
||||
let path = Path::new("/nonexistent/track.flac");
|
||||
assert!(parse_music_metadata_for_path(path).is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
use std::path::Path;
|
||||
|
||||
use crate::music::flac::FlacMusicMetadataParser;
|
||||
use crate::music::metadata::MusicMetadata;
|
||||
use crate::music::mp3::Mp3MusicMetadataParser;
|
||||
|
||||
/// A parser that turns a single media file into our shared [`MusicMetadata`].
|
||||
/// Each supported container format provides one implementation.
|
||||
pub trait MusicMetadataParser {
|
||||
fn parse(&self, path: &Path) -> Option<MusicMetadata>;
|
||||
}
|
||||
|
||||
/// Selects the right [`MusicMetadataParser`] for a given file.
|
||||
pub struct MusicMetadataParserFactory;
|
||||
|
||||
impl MusicMetadataParserFactory {
|
||||
/// Returns a parser based on the file extension, or `None` for files we
|
||||
/// don't treat as music.
|
||||
pub fn for_path(path: &Path) -> Option<Box<dyn MusicMetadataParser>> {
|
||||
match path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.as_deref()
|
||||
{
|
||||
Some("flac") => Some(Box::new(FlacMusicMetadataParser)),
|
||||
Some("mp3") => Some(Box::new(Mp3MusicMetadataParser)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
use crate::{AudioMeta, VirtualPath};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PathTemplate {
|
||||
pub pattern: String,
|
||||
pub fallback_artist: String,
|
||||
pub fallback_album: String,
|
||||
pub fallback_title: String,
|
||||
pub fallback_year: String,
|
||||
}
|
||||
|
||||
impl Default for PathTemplate {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pattern: "$artist/$album ($year) [$format_upper]/$track - $title.$format".to_string(),
|
||||
fallback_artist: "Unknown Artist".to_string(),
|
||||
fallback_album: "Unknown Album".to_string(),
|
||||
fallback_title: "Unknown Track".to_string(),
|
||||
fallback_year: "Unknown".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PathResolver {
|
||||
template: PathTemplate,
|
||||
}
|
||||
|
||||
impl PathResolver {
|
||||
pub fn new(template: PathTemplate) -> Self {
|
||||
Self { template }
|
||||
}
|
||||
|
||||
pub fn resolve(&self, meta: &AudioMeta, extension: &str) -> VirtualPath {
|
||||
let artist = meta
|
||||
.artist
|
||||
.as_deref()
|
||||
.unwrap_or(&self.template.fallback_artist);
|
||||
let album = meta
|
||||
.album
|
||||
.as_deref()
|
||||
.unwrap_or(&self.template.fallback_album);
|
||||
let title = meta
|
||||
.title
|
||||
.as_deref()
|
||||
.unwrap_or(&self.template.fallback_title);
|
||||
let year = meta
|
||||
.year
|
||||
.map(|y| y.to_string())
|
||||
.unwrap_or_else(|| self.template.fallback_year.clone());
|
||||
let track = meta.track.unwrap_or(0);
|
||||
let disc = meta.disc.unwrap_or(1);
|
||||
let genre = meta.genre.as_deref().unwrap_or("Unknown");
|
||||
let format = extension.to_lowercase();
|
||||
let format_upper = extension.to_uppercase();
|
||||
|
||||
let artist = sanitize_path_component(artist);
|
||||
let album = sanitize_path_component(album);
|
||||
let title = sanitize_path_component(title);
|
||||
let genre = sanitize_path_component(genre);
|
||||
|
||||
let path = self
|
||||
.template
|
||||
.pattern
|
||||
.replace("$artist", &artist)
|
||||
.replace("$album", &album)
|
||||
.replace("$title", &title)
|
||||
.replace("$track", &format!("{:02}", track))
|
||||
.replace("$disc", &disc.to_string())
|
||||
.replace("$year", &year)
|
||||
.replace("$genre", &genre)
|
||||
.replace("$format_upper", &format_upper)
|
||||
.replace("$format", &format);
|
||||
|
||||
VirtualPath::new(path)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PathResolver {
|
||||
fn default() -> Self {
|
||||
Self::new(PathTemplate::default())
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_path_component(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| match c {
|
||||
'/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' | '\0' => '_',
|
||||
c => c,
|
||||
})
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::AudioFormat;
|
||||
|
||||
#[test]
|
||||
fn test_resolve_complete_metadata() {
|
||||
let resolver = PathResolver::default();
|
||||
let meta = AudioMeta {
|
||||
artist: Some("Metallica".to_string()),
|
||||
album: Some("Master of Puppets".to_string()),
|
||||
title: Some("Battery".to_string()),
|
||||
track: Some(1),
|
||||
year: Some(1986),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let path = resolver.resolve(&meta, "flac");
|
||||
assert_eq!(
|
||||
path.as_str(),
|
||||
"Metallica/Master of Puppets (1986) [FLAC]/01 - Battery.flac"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_missing_album() {
|
||||
let resolver = PathResolver::default();
|
||||
let meta = AudioMeta {
|
||||
artist: Some("Artist".to_string()),
|
||||
title: Some("Track".to_string()),
|
||||
track: Some(5),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let path = resolver.resolve(&meta, "mp3");
|
||||
assert_eq!(
|
||||
path.as_str(),
|
||||
"Artist/Unknown Album (Unknown) [MP3]/05 - Track.mp3"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_special_chars() {
|
||||
let resolver = PathResolver::default();
|
||||
let meta = AudioMeta {
|
||||
artist: Some("AC/DC".to_string()),
|
||||
album: Some("Who Made Who?".to_string()),
|
||||
title: Some("Test:Track".to_string()),
|
||||
track: Some(1),
|
||||
year: Some(1986),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let path = resolver.resolve(&meta, "flac");
|
||||
assert!(!path.as_str().contains(':'));
|
||||
assert!(!path.as_str().contains('?'));
|
||||
assert!(path.as_str().contains("AC_DC"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_template() {
|
||||
let template = PathTemplate {
|
||||
pattern: "$genre/$artist - $album/$track $title.$format".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let resolver = PathResolver::new(template);
|
||||
let meta = AudioMeta {
|
||||
artist: Some("Artist".to_string()),
|
||||
album: Some("Album".to_string()),
|
||||
title: Some("Song".to_string()),
|
||||
genre: Some("Rock".to_string()),
|
||||
track: Some(3),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let path = resolver.resolve(&meta, "flac");
|
||||
assert_eq!(path.as_str(), "Rock/Artist - Album/03 Song.flac");
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{error, warn};
|
||||
|
||||
pub struct TaskSupervisor {
|
||||
tasks: Arc<RwLock<HashMap<String, TaskEntry>>>,
|
||||
}
|
||||
|
||||
struct TaskEntry {
|
||||
handle: JoinHandle<()>,
|
||||
status: TaskStatus,
|
||||
restart_count: u32,
|
||||
last_restart: Option<Instant>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum TaskStatus {
|
||||
Running,
|
||||
Failed { error: String, at: Instant },
|
||||
Restarting { attempt: u32 },
|
||||
Stopped,
|
||||
}
|
||||
|
||||
impl Default for TaskSupervisor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl TaskSupervisor {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tasks: Arc::new(RwLock::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn spawn_supervised<F>(&self, name: &str, future: F)
|
||||
where
|
||||
F: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let name_owned = name.to_string();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
future.await;
|
||||
});
|
||||
|
||||
self.tasks.write().insert(
|
||||
name_owned,
|
||||
TaskEntry {
|
||||
handle,
|
||||
status: TaskStatus::Running,
|
||||
restart_count: 0,
|
||||
last_restart: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn spawn_critical<F, Fut>(&self, name: &str, factory: F)
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync + 'static,
|
||||
Fut: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let tasks = self.tasks.clone();
|
||||
let name_owned = name.to_string();
|
||||
|
||||
let monitor_handle = tokio::spawn(async move {
|
||||
let mut restart_count = 0u32;
|
||||
let max_restarts = 5u32;
|
||||
let backoff_durations = [
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(30),
|
||||
];
|
||||
|
||||
loop {
|
||||
let handle = tokio::spawn(factory());
|
||||
|
||||
{
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Running;
|
||||
}
|
||||
}
|
||||
|
||||
match handle.await {
|
||||
Ok(()) => {
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Stopped;
|
||||
}
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
restart_count += 1;
|
||||
|
||||
if restart_count > max_restarts {
|
||||
error!(task = %name_owned, "Task exceeded max restarts ({}), giving up", max_restarts);
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: format!("Exceeded max restarts: {}", e),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
let backoff_idx =
|
||||
(restart_count as usize - 1).min(backoff_durations.len() - 1);
|
||||
let backoff = backoff_durations[backoff_idx];
|
||||
|
||||
warn!(
|
||||
task = %name_owned,
|
||||
error = %e,
|
||||
attempt = restart_count,
|
||||
backoff_ms = backoff.as_millis() as u64,
|
||||
"Critical task failed, restarting with backoff"
|
||||
);
|
||||
|
||||
{
|
||||
let mut t = tasks.write();
|
||||
if let Some(entry) = t.get_mut(&name_owned) {
|
||||
entry.status = TaskStatus::Restarting {
|
||||
attempt: restart_count,
|
||||
};
|
||||
entry.restart_count = restart_count;
|
||||
entry.last_restart = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::sleep(backoff).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
self.tasks.write().insert(
|
||||
name.to_string(),
|
||||
TaskEntry {
|
||||
handle: monitor_handle,
|
||||
status: TaskStatus::Running,
|
||||
restart_count: 0,
|
||||
last_restart: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn task_status(&self, name: &str) -> TaskStatus {
|
||||
let mut tasks = self.tasks.write();
|
||||
if let Some(entry) = tasks.get_mut(name) {
|
||||
if entry.handle.is_finished() {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: "Task exited".into(),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
entry.status.clone()
|
||||
} else {
|
||||
TaskStatus::Stopped
|
||||
}
|
||||
}
|
||||
|
||||
pub fn check_all(&self) -> Vec<(String, TaskStatus)> {
|
||||
let mut tasks = self.tasks.write();
|
||||
tasks
|
||||
.iter_mut()
|
||||
.map(|(name, entry)| {
|
||||
if entry.handle.is_finished() {
|
||||
entry.status = TaskStatus::Failed {
|
||||
error: "Task exited".into(),
|
||||
at: Instant::now(),
|
||||
};
|
||||
}
|
||||
(name.clone(), entry.status.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::PathBuf;
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct OriginId(pub String);
|
||||
|
||||
impl From<&str> for OriginId {
|
||||
fn from(s: &str) -> Self {
|
||||
Self(s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for OriginId {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct FileId(pub i64);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct VirtualPath(pub PathBuf);
|
||||
|
||||
impl VirtualPath {
|
||||
pub fn new(path: impl Into<PathBuf>) -> Self {
|
||||
Self(path.into())
|
||||
}
|
||||
|
||||
pub fn as_path(&self) -> &std::path::Path {
|
||||
&self.0
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
self.0.to_str().unwrap_or("")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RealPath {
|
||||
pub origin_id: OriginId,
|
||||
pub path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ContentHash(pub [u8; 8]);
|
||||
|
||||
impl ContentHash {
|
||||
pub fn from_bytes(data: &[u8]) -> Self {
|
||||
use xxhash_rust::xxh64::xxh64;
|
||||
Self(xxh64(data, 0).to_le_bytes())
|
||||
}
|
||||
|
||||
pub fn to_hex(&self) -> String {
|
||||
hex::encode(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ChunkHash(pub [u8; 8]);
|
||||
|
||||
impl ChunkHash {
|
||||
pub fn from_bytes(data: &[u8]) -> Self {
|
||||
use xxhash_rust::xxh64::xxh64;
|
||||
Self(xxh64(data, 0).to_le_bytes())
|
||||
}
|
||||
|
||||
pub fn as_hex(&self) -> String {
|
||||
hex::encode(self.0)
|
||||
}
|
||||
|
||||
pub fn to_hex(&self) -> String {
|
||||
self.as_hex()
|
||||
}
|
||||
|
||||
pub fn from_hex(s: &str) -> Option<Self> {
|
||||
let bytes = hex::decode(s).ok()?;
|
||||
if bytes.len() != 8 {
|
||||
return None;
|
||||
}
|
||||
let mut arr = [0u8; 8];
|
||||
arr.copy_from_slice(&bytes);
|
||||
Some(Self(arr))
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ChunkHash {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.as_hex())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub enum AudioFormat {
|
||||
Flac,
|
||||
Mp3,
|
||||
Opus,
|
||||
Vorbis,
|
||||
Aac,
|
||||
Alac,
|
||||
Wav,
|
||||
#[default]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AudioFormat {
|
||||
pub fn from_extension(ext: &str) -> Self {
|
||||
match ext.to_lowercase().as_str() {
|
||||
"flac" => Self::Flac,
|
||||
"mp3" => Self::Mp3,
|
||||
"opus" => Self::Opus,
|
||||
"ogg" => Self::Vorbis,
|
||||
"m4a" | "aac" => Self::Aac,
|
||||
"wav" => Self::Wav,
|
||||
_ => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct AudioMeta {
|
||||
pub title: Option<String>,
|
||||
pub artist: Option<String>,
|
||||
pub album: Option<String>,
|
||||
pub album_artist: Option<String>,
|
||||
pub genre: Option<String>,
|
||||
pub year: Option<u32>,
|
||||
pub track: Option<u32>,
|
||||
pub disc: Option<u32>,
|
||||
pub duration_ms: Option<u64>,
|
||||
pub bitrate: Option<u32>,
|
||||
pub sample_rate: Option<u32>,
|
||||
pub format: AudioFormat,
|
||||
pub track_total: Option<u32>,
|
||||
pub disc_total: Option<u32>,
|
||||
pub date: Option<String>,
|
||||
pub composer: Option<String>,
|
||||
pub comment: Option<String>,
|
||||
pub lyrics: Option<String>,
|
||||
pub copyright: Option<String>,
|
||||
pub compilation: Option<bool>,
|
||||
pub artist_sort: Option<String>,
|
||||
pub album_artist_sort: Option<String>,
|
||||
pub album_sort: Option<String>,
|
||||
pub title_sort: Option<String>,
|
||||
pub mb_recording_id: Option<String>,
|
||||
pub mb_album_id: Option<String>,
|
||||
pub mb_artist_id: Option<String>,
|
||||
pub mb_album_artist_id: Option<String>,
|
||||
pub mb_release_group_id: Option<String>,
|
||||
pub replaygain_track_gain: Option<f32>,
|
||||
pub replaygain_track_peak: Option<f32>,
|
||||
pub replaygain_album_gain: Option<f32>,
|
||||
pub replaygain_album_peak: Option<f32>,
|
||||
pub channels: Option<u32>,
|
||||
pub bits_per_sample: Option<u32>,
|
||||
pub encoder: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FileMeta {
|
||||
pub id: FileId,
|
||||
pub virtual_path: VirtualPath,
|
||||
pub real_path: RealPath,
|
||||
pub size: u64,
|
||||
pub mtime: SystemTime,
|
||||
pub content_hash: Option<ContentHash>,
|
||||
pub audio: Option<AudioMeta>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DirEntry {
|
||||
pub name: String,
|
||||
pub is_dir: bool,
|
||||
pub size: u64,
|
||||
pub mtime: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileStat {
|
||||
pub size: u64,
|
||||
pub mtime: SystemTime,
|
||||
pub is_dir: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum HealthStatus {
|
||||
Healthy,
|
||||
Degraded,
|
||||
Unhealthy,
|
||||
#[default]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_content_hash() {
|
||||
let data = b"hello world";
|
||||
let hash1 = ContentHash::from_bytes(data);
|
||||
let hash2 = ContentHash::from_bytes(data);
|
||||
assert_eq!(hash1, hash2);
|
||||
|
||||
let hash3 = ContentHash::from_bytes(b"different");
|
||||
assert_ne!(hash1, hash3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_audio_format_from_extension() {
|
||||
assert_eq!(AudioFormat::from_extension("flac"), AudioFormat::Flac);
|
||||
assert_eq!(AudioFormat::from_extension("MP3"), AudioFormat::Mp3);
|
||||
assert_eq!(AudioFormat::from_extension("unknown"), AudioFormat::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_virtual_path() {
|
||||
let path = VirtualPath::new("/Artist/Album/Track.flac");
|
||||
assert_eq!(path.as_str(), "/Artist/Album/Track.flac");
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
[package]
|
||||
name = "musicfs-fuse"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
musicfs-cache = { path = "../musicfs-cache" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-search = { path = "../musicfs-search" }
|
||||
fuser.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
moka.workspace = true
|
||||
parking_lot.workspace = true
|
||||
libc = "0.2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -1,954 +0,0 @@
|
||||
use crate::ops::SearchOps;
|
||||
use fuser::{
|
||||
FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, ReplyOpen,
|
||||
Request,
|
||||
};
|
||||
use musicfs_cache::{
|
||||
Database, OverlayError, OverlayReader, RemoveError, RenameError, VirtualNode, VirtualTree,
|
||||
ROOT_INODE,
|
||||
};
|
||||
use musicfs_cas::FileReader;
|
||||
use musicfs_core::{Result, VirtualPath};
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::OsStr;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::runtime::Handle;
|
||||
use tracing::{debug, info, instrument, trace, warn};
|
||||
|
||||
const TTL: Duration = Duration::from_secs(1);
|
||||
const BLOCK_SIZE: u32 = 512;
|
||||
const SEARCH_QUERY_INODE_BASE: u64 = 0xFFFF_FFFF_0000_0100;
|
||||
|
||||
pub struct MusicFs {
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
reader: Option<Arc<FileReader>>,
|
||||
db: Option<Arc<Database>>,
|
||||
overlay_reader: Option<Arc<OverlayReader>>,
|
||||
runtime_handle: Handle,
|
||||
search_ops: Option<SearchOps>,
|
||||
query_inodes: RwLock<HashMap<String, u64>>,
|
||||
inode_queries: RwLock<HashMap<u64, String>>,
|
||||
next_query_inode: RwLock<u64>,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
}
|
||||
|
||||
impl MusicFs {
|
||||
pub fn new(tree: Arc<RwLock<VirtualTree>>, runtime_handle: Handle) -> Self {
|
||||
Self {
|
||||
tree,
|
||||
reader: None,
|
||||
db: None,
|
||||
overlay_reader: None,
|
||||
runtime_handle,
|
||||
search_ops: None,
|
||||
query_inodes: RwLock::new(HashMap::new()),
|
||||
inode_queries: RwLock::new(HashMap::new()),
|
||||
next_query_inode: RwLock::new(SEARCH_QUERY_INODE_BASE),
|
||||
uid: unsafe { libc::getuid() },
|
||||
gid: unsafe { libc::getgid() },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_reader(
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
reader: Arc<FileReader>,
|
||||
runtime_handle: Handle,
|
||||
) -> Self {
|
||||
Self {
|
||||
tree,
|
||||
reader: Some(reader),
|
||||
db: None,
|
||||
overlay_reader: None,
|
||||
runtime_handle,
|
||||
search_ops: None,
|
||||
query_inodes: RwLock::new(HashMap::new()),
|
||||
inode_queries: RwLock::new(HashMap::new()),
|
||||
next_query_inode: RwLock::new(SEARCH_QUERY_INODE_BASE),
|
||||
uid: unsafe { libc::getuid() },
|
||||
gid: unsafe { libc::getgid() },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_db(mut self, db: Arc<Database>) -> Self {
|
||||
self.db = Some(db);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_overlay(mut self, overlay: Arc<OverlayReader>) -> Self {
|
||||
self.overlay_reader = Some(overlay);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_search(mut self, search_ops: SearchOps) -> Self {
|
||||
self.search_ops = Some(search_ops);
|
||||
self
|
||||
}
|
||||
|
||||
fn resolve_path(&self, parent_inode: u64, name: &OsStr) -> Option<VirtualPath> {
|
||||
let tree = self.tree.read();
|
||||
let parent_path = self.inode_to_path_inner(&tree, parent_inode)?;
|
||||
let name_str = name.to_string_lossy();
|
||||
let full_path = if parent_path == "/" {
|
||||
format!("/{}", name_str)
|
||||
} else {
|
||||
format!("{}/{}", parent_path, name_str)
|
||||
};
|
||||
Some(VirtualPath::new(full_path))
|
||||
}
|
||||
|
||||
fn inode_to_path_inner(&self, tree: &VirtualTree, inode: u64) -> Option<String> {
|
||||
for (path, &ino) in tree.path_to_inode_iter() {
|
||||
if ino == inode {
|
||||
return Some(path.as_str().to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn get_or_create_query_inode(&self, query: &str) -> u64 {
|
||||
let query_inodes = self.query_inodes.read();
|
||||
if let Some(&inode) = query_inodes.get(query) {
|
||||
return inode;
|
||||
}
|
||||
drop(query_inodes);
|
||||
|
||||
let mut query_inodes = self.query_inodes.write();
|
||||
let mut inode_queries = self.inode_queries.write();
|
||||
let mut next_inode = self.next_query_inode.write();
|
||||
|
||||
if let Some(&inode) = query_inodes.get(query) {
|
||||
return inode;
|
||||
}
|
||||
|
||||
let inode = *next_inode;
|
||||
*next_inode += 1;
|
||||
query_inodes.insert(query.to_string(), inode);
|
||||
inode_queries.insert(inode, query.to_string());
|
||||
inode
|
||||
}
|
||||
|
||||
fn get_query_for_inode(&self, inode: u64) -> Option<String> {
|
||||
self.inode_queries.read().get(&inode).cloned()
|
||||
}
|
||||
|
||||
pub fn mount(self, mountpoint: &Path) -> Result<()> {
|
||||
info!("Mounting MusicFS at {:?}", mountpoint);
|
||||
|
||||
let options = vec![
|
||||
fuser::MountOption::FSName("musicfs".to_string()),
|
||||
fuser::MountOption::AutoUnmount,
|
||||
fuser::MountOption::AllowOther,
|
||||
];
|
||||
|
||||
fuser::mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn spawn_mount(self, mountpoint: &Path) -> Result<fuser::BackgroundSession> {
|
||||
info!("Mounting MusicFS at {:?}", mountpoint);
|
||||
|
||||
let options = vec![
|
||||
fuser::MountOption::FSName("musicfs".to_string()),
|
||||
fuser::MountOption::AutoUnmount,
|
||||
fuser::MountOption::AllowOther,
|
||||
];
|
||||
|
||||
let session =
|
||||
fuser::spawn_mount2(self, mountpoint, &options).map_err(musicfs_core::Error::Io)?;
|
||||
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
fn node_to_attr(&self, node: &VirtualNode) -> FileAttr {
|
||||
match node {
|
||||
VirtualNode::Directory(dir) => FileAttr {
|
||||
ino: dir.inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: dir.mtime,
|
||||
mtime: dir.mtime,
|
||||
ctime: dir.mtime,
|
||||
crtime: dir.mtime,
|
||||
kind: FileType::Directory,
|
||||
perm: 0o755,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: BLOCK_SIZE,
|
||||
flags: 0,
|
||||
},
|
||||
VirtualNode::File(file) => FileAttr {
|
||||
ino: file.inode,
|
||||
size: file.size,
|
||||
blocks: (file.size + BLOCK_SIZE as u64 - 1) / BLOCK_SIZE as u64,
|
||||
atime: file.mtime,
|
||||
mtime: file.mtime,
|
||||
ctime: file.mtime,
|
||||
crtime: file.mtime,
|
||||
kind: FileType::RegularFile,
|
||||
perm: 0o644,
|
||||
nlink: 1,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: BLOCK_SIZE,
|
||||
flags: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Filesystem for MusicFs {
|
||||
fn init(
|
||||
&mut self,
|
||||
_req: &Request<'_>,
|
||||
_config: &mut fuser::KernelConfig,
|
||||
) -> std::result::Result<(), libc::c_int> {
|
||||
info!("MusicFS initialized");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn destroy(&mut self) {
|
||||
info!("MusicFS destroyed");
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) {
|
||||
let name_str = name.to_string_lossy();
|
||||
|
||||
if parent == ROOT_INODE && SearchOps::is_search_dir_name(&name_str) {
|
||||
trace!(parent, name = %name_str, "search_dir_name matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.lookup_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if parent == SearchOps::search_dir_inode() {
|
||||
trace!(parent, name = %name_str, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
let inode = self.get_or_create_query_inode(&name_str);
|
||||
search_ops.lookup_query_dir(&name_str, inode, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(query) = self.get_query_for_inode(parent) {
|
||||
trace!(parent, name = %name_str, query = %query, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
let inode = self.get_or_create_query_inode(&format!("{}:{}", query, name_str));
|
||||
search_ops.lookup_result(inode, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(inode) = tree.lookup(parent, name) {
|
||||
trace!(parent, name = %name_str, ino = inode, "file found in tree");
|
||||
if let Some(node) = tree.get(inode) {
|
||||
let attr = self.node_to_attr(node);
|
||||
reply.entry(&TTL, &attr, 0);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
trace!(parent, name = %name_str, "file not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) {
|
||||
if ino == SearchOps::search_dir_inode() {
|
||||
trace!(ino, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if SearchOps::is_search_inode(ino) {
|
||||
trace!(ino, "search_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_result(ino, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if self.get_query_for_inode(ino).is_some() {
|
||||
trace!(ino, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.getattr_search_dir(reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(node) = tree.get(ino) {
|
||||
trace!(ino, "inode found in tree");
|
||||
let mut attr = self.node_to_attr(node);
|
||||
|
||||
if let VirtualNode::File(file) = node {
|
||||
if let Some(ref overlay) = self.overlay_reader {
|
||||
match overlay.estimate_virtual_size(file.file_id) {
|
||||
Ok(Some(virtual_size)) => {
|
||||
trace!(ino, file_id = ?file.file_id, virtual_size, "using overlay virtual size");
|
||||
attr.size = virtual_size;
|
||||
attr.blocks =
|
||||
(virtual_size + BLOCK_SIZE as u64 - 1) / BLOCK_SIZE as u64;
|
||||
}
|
||||
Ok(None) => {
|
||||
trace!(ino, file_id = ?file.file_id, "no overlay, using original size");
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(ino, file_id = ?file.file_id, error = %e, "overlay size estimation failed, using original");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
reply.attr(&TTL, &attr);
|
||||
} else {
|
||||
trace!(ino, "inode not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn readdir(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
offset: i64,
|
||||
mut reply: ReplyDirectory,
|
||||
) {
|
||||
if ino == SearchOps::search_dir_inode() {
|
||||
trace!(ino, offset, "search_dir_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readdir_search_root(offset, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(query) = self.get_query_for_inode(ino) {
|
||||
trace!(ino, offset, query = %query, "query_inode matched");
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readdir_query(&query, offset, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if let Some(children) = tree.readdir(ino) {
|
||||
trace!(
|
||||
ino,
|
||||
offset,
|
||||
children_count = children.len(),
|
||||
"directory found"
|
||||
);
|
||||
let parent_ino = tree.get_parent(ino).unwrap_or(ROOT_INODE);
|
||||
|
||||
let entries: Vec<(u64, FileType, &str)> = vec![
|
||||
(ino, FileType::Directory, "."),
|
||||
(parent_ino, FileType::Directory, ".."),
|
||||
];
|
||||
|
||||
let child_entries: Vec<(u64, FileType, String)> = children
|
||||
.iter()
|
||||
.map(|(name, child_ino, is_dir)| {
|
||||
let kind = if *is_dir {
|
||||
FileType::Directory
|
||||
} else {
|
||||
FileType::RegularFile
|
||||
};
|
||||
(*child_ino, kind, name.to_string_lossy().to_string())
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, name) {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let base_offset = entries.len();
|
||||
for (i, (inode, kind, name)) in child_entries.iter().enumerate() {
|
||||
let entry_offset = base_offset + i;
|
||||
if entry_offset < offset as usize {
|
||||
continue;
|
||||
}
|
||||
if reply.add(*inode, (entry_offset + 1) as i64, *kind, name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
reply.ok();
|
||||
} else {
|
||||
trace!(ino, offset, "directory not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn open(&mut self, _req: &Request, ino: u64, flags: i32, reply: ReplyOpen) {
|
||||
let write_flags = libc::O_WRONLY | libc::O_RDWR | libc::O_APPEND | libc::O_TRUNC;
|
||||
if flags & write_flags != 0 {
|
||||
trace!(ino, flags, "write flags detected");
|
||||
reply.error(libc::EROFS);
|
||||
return;
|
||||
}
|
||||
|
||||
let tree = self.tree.read();
|
||||
|
||||
if tree.get(ino).is_some() {
|
||||
trace!(ino, "inode found");
|
||||
reply.opened(0, 0);
|
||||
} else {
|
||||
trace!(ino, "inode not found");
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn read(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
offset: i64,
|
||||
size: u32,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
reply: ReplyData,
|
||||
) {
|
||||
let file_id = {
|
||||
let tree = self.tree.read();
|
||||
if let Some(VirtualNode::File(file)) = tree.get(ino) {
|
||||
trace!(ino, "file found in tree");
|
||||
file.file_id
|
||||
} else {
|
||||
trace!(ino, "file not found");
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let handle = self.runtime_handle.clone();
|
||||
|
||||
if let Some(ref overlay) = self.overlay_reader {
|
||||
let overlay = overlay.clone();
|
||||
let result = std::thread::scope(|_| {
|
||||
handle.block_on(async {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
overlay.read(file_id, offset as u64, size),
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(Ok(data)) => {
|
||||
trace!(
|
||||
ino,
|
||||
offset,
|
||||
size_bytes = size,
|
||||
bytes_read = data.len(),
|
||||
"overlay read successful"
|
||||
);
|
||||
reply.data(&data);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
let errno = match &e {
|
||||
OverlayError::NotFound(_) => libc::ENOENT,
|
||||
OverlayError::Database(_) => libc::EIO,
|
||||
OverlayError::Handler(_) => libc::EIO,
|
||||
OverlayError::Cas(_) => libc::EIO,
|
||||
OverlayError::NoHandler(_) => libc::EIO,
|
||||
};
|
||||
warn!(ino, offset, size_bytes = size, error = %e, "overlay read failed");
|
||||
reply.error(errno);
|
||||
}
|
||||
Err(_timeout) => {
|
||||
warn!(
|
||||
ino,
|
||||
offset,
|
||||
size_bytes = size,
|
||||
"overlay read timed out after 30s"
|
||||
);
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let Some(reader) = &self.reader else {
|
||||
trace!(ino, "no reader available");
|
||||
reply.data(&[]);
|
||||
return;
|
||||
};
|
||||
|
||||
let reader = reader.clone();
|
||||
let result = std::thread::scope(|_| {
|
||||
handle.block_on(async {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
reader.read(file_id, offset as u64, size),
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(Ok(data)) => {
|
||||
trace!(
|
||||
ino,
|
||||
offset,
|
||||
size_bytes = size,
|
||||
bytes_read = data.len(),
|
||||
"read successful"
|
||||
);
|
||||
reply.data(&data);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!(ino, offset, size_bytes = size, error = %e, "read failed");
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
Err(_timeout) => {
|
||||
warn!(ino, offset, size_bytes = size, "read timed out after 30s");
|
||||
reply.error(libc::EIO);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, reply))]
|
||||
fn release(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
ino: u64,
|
||||
_fh: u64,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
_flush: bool,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
trace!(ino, "releasing file handle");
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn readlink(&mut self, _req: &Request, ino: u64, reply: ReplyData) {
|
||||
debug!("readlink(ino={})", ino);
|
||||
|
||||
if SearchOps::is_search_inode(ino) {
|
||||
if let Some(ref search_ops) = self.search_ops {
|
||||
search_ops.readlink(ino, reply);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
reply.error(libc::EINVAL);
|
||||
}
|
||||
|
||||
fn write(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_fh: u64,
|
||||
_offset: i64,
|
||||
_data: &[u8],
|
||||
_write_flags: u32,
|
||||
_flags: i32,
|
||||
_lock_owner: Option<u64>,
|
||||
reply: fuser::ReplyWrite,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn mkdir(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
parent: u64,
|
||||
name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
let path = match self.resolve_path(parent, name) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut tree = self.tree.write();
|
||||
match tree.mkdir(&path) {
|
||||
Ok(inode) => {
|
||||
if let Some(ref db) = self.db {
|
||||
if let Err(e) = db.insert_directory(&path) {
|
||||
warn!(error = %e, "failed to persist directory to database");
|
||||
}
|
||||
}
|
||||
let attr = FileAttr {
|
||||
ino: inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: SystemTime::now(),
|
||||
mtime: SystemTime::now(),
|
||||
ctime: SystemTime::now(),
|
||||
crtime: SystemTime::now(),
|
||||
kind: FileType::Directory,
|
||||
perm: 0o755,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: BLOCK_SIZE,
|
||||
flags: 0,
|
||||
};
|
||||
debug!(path = %path.as_str(), inode, "mkdir successful");
|
||||
reply.entry(&TTL, &attr, 0);
|
||||
}
|
||||
Err(RenameError::TargetExists) => reply.error(libc::EEXIST),
|
||||
Err(RenameError::ParentNotFound) => reply.error(libc::ENOENT),
|
||||
Err(_) => reply.error(libc::EIO),
|
||||
}
|
||||
}
|
||||
|
||||
fn unlink(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: fuser::ReplyEmpty) {
|
||||
let path = match self.resolve_path(parent, name) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let (file_id, is_dir) = {
|
||||
let tree = self.tree.read();
|
||||
match tree.get_by_path(&path) {
|
||||
Some(VirtualNode::File(f)) => (Some(f.file_id), false),
|
||||
Some(VirtualNode::Directory(_)) => (None, true),
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if is_dir {
|
||||
reply.error(libc::EISDIR);
|
||||
return;
|
||||
}
|
||||
|
||||
let trash_path = VirtualPath::new(format!("/.trash{}", path.as_str()));
|
||||
|
||||
{
|
||||
let mut tree = self.tree.write();
|
||||
tree.ensure_trash_dir();
|
||||
|
||||
let trash_parent = std::path::Path::new(trash_path.as_str())
|
||||
.parent()
|
||||
.map(|p| VirtualPath::new(p.to_string_lossy().into_owned()))
|
||||
.unwrap_or_else(|| VirtualPath::new("/.trash"));
|
||||
|
||||
if let Err(e) = tree.mkdir_p(&trash_parent) {
|
||||
if !matches!(e, RenameError::TargetExists) {
|
||||
warn!(error = ?e, "failed to create trash parent directories");
|
||||
reply.error(libc::EIO);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = tree.rename_file(&path, &trash_path) {
|
||||
match e {
|
||||
RenameError::SourceNotFound => reply.error(libc::ENOENT),
|
||||
RenameError::TargetExists => reply.error(libc::EEXIST),
|
||||
_ => reply.error(libc::EIO),
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let (Some(ref db), Some(id)) = (&self.db, file_id) {
|
||||
if let Err(e) = db.update_virtual_path(id, &trash_path) {
|
||||
warn!(error = %e, "failed to update virtual path in database");
|
||||
}
|
||||
if let Err(e) = db.mark_trashed(id, &path) {
|
||||
warn!(error = %e, "failed to mark file as trashed in database");
|
||||
}
|
||||
}
|
||||
|
||||
debug!(path = %path.as_str(), trash = %trash_path.as_str(), "file moved to trash");
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn rmdir(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: fuser::ReplyEmpty) {
|
||||
let path = match self.resolve_path(parent, name) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if VirtualTree::is_trash_path(&path) {
|
||||
reply.error(libc::EPERM);
|
||||
return;
|
||||
}
|
||||
|
||||
{
|
||||
let mut tree = self.tree.write();
|
||||
match tree.remove_directory(&path) {
|
||||
Ok(()) => {}
|
||||
Err(RemoveError::NotFound) => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
Err(RemoveError::NotEmpty) => {
|
||||
reply.error(libc::ENOTEMPTY);
|
||||
return;
|
||||
}
|
||||
Err(RemoveError::NotDirectory) => {
|
||||
reply.error(libc::ENOTDIR);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref db) = self.db {
|
||||
if let Err(e) = db.delete_directory(&path) {
|
||||
warn!(error = %e, "failed to delete directory from database");
|
||||
}
|
||||
}
|
||||
|
||||
debug!(path = %path.as_str(), "directory removed");
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
fn rename(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
parent: u64,
|
||||
name: &OsStr,
|
||||
newparent: u64,
|
||||
newname: &OsStr,
|
||||
_flags: u32,
|
||||
reply: fuser::ReplyEmpty,
|
||||
) {
|
||||
let old_path = match self.resolve_path(parent, name) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let new_path = match self.resolve_path(newparent, newname) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if old_path.as_str() == new_path.as_str() {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
|
||||
let is_dir = {
|
||||
let tree = self.tree.read();
|
||||
tree.get_by_path(&old_path)
|
||||
.map(|n| n.is_dir())
|
||||
.unwrap_or(false)
|
||||
};
|
||||
|
||||
let result = if is_dir {
|
||||
let mut tree = self.tree.write();
|
||||
match tree.rename_directory(&old_path, &new_path) {
|
||||
Ok(count) => {
|
||||
if let Some(ref db) = self.db {
|
||||
let old_prefix = if old_path.as_str().ends_with('/') {
|
||||
old_path.as_str().to_string()
|
||||
} else {
|
||||
format!("{}/", old_path.as_str())
|
||||
};
|
||||
let new_prefix = if new_path.as_str().ends_with('/') {
|
||||
new_path.as_str().to_string()
|
||||
} else {
|
||||
format!("{}/", new_path.as_str())
|
||||
};
|
||||
if let Err(e) = db.rename_directory(&old_prefix, &new_prefix) {
|
||||
warn!(error = %e, "failed to persist file path rename to database");
|
||||
}
|
||||
if let Err(e) = db.rename_directories(&old_prefix, &new_prefix) {
|
||||
warn!(error = %e, "failed to persist directory rename to database");
|
||||
}
|
||||
}
|
||||
debug!(old = %old_path.as_str(), new = %new_path.as_str(), count, "directory renamed");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
} else {
|
||||
let file_id = {
|
||||
let tree = self.tree.read();
|
||||
match tree.get_by_path(&old_path) {
|
||||
Some(VirtualNode::File(f)) => Some(f.file_id),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
let mut tree = self.tree.write();
|
||||
match tree.rename_file(&old_path, &new_path) {
|
||||
Ok(()) => {
|
||||
if let (Some(ref db), Some(id)) = (&self.db, file_id) {
|
||||
if let Err(e) = db.update_virtual_path(id, &new_path) {
|
||||
warn!(error = %e, "failed to persist file rename to database");
|
||||
}
|
||||
let was_in_trash = VirtualTree::is_trash_path(&old_path);
|
||||
let now_in_trash = VirtualTree::is_trash_path(&new_path);
|
||||
if was_in_trash && !now_in_trash {
|
||||
if let Err(e) = db.unmark_trashed(id) {
|
||||
warn!(error = %e, "failed to unmark trashed after restore");
|
||||
}
|
||||
debug!(path = %new_path.as_str(), "file restored from trash");
|
||||
}
|
||||
}
|
||||
debug!(old = %old_path.as_str(), new = %new_path.as_str(), "file renamed");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => reply.ok(),
|
||||
Err(RenameError::SourceNotFound) => reply.error(libc::ENOENT),
|
||||
Err(RenameError::TargetExists) => reply.error(libc::EEXIST),
|
||||
Err(RenameError::ParentNotFound) => reply.error(libc::ENOENT),
|
||||
Err(RenameError::IsDirectory) => reply.error(libc::EISDIR),
|
||||
Err(RenameError::NotDirectory) => reply.error(libc::ENOTDIR),
|
||||
}
|
||||
}
|
||||
|
||||
fn create(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
_flags: i32,
|
||||
reply: fuser::ReplyCreate,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn setattr(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_mode: Option<u32>,
|
||||
_uid: Option<u32>,
|
||||
_gid: Option<u32>,
|
||||
_size: Option<u64>,
|
||||
_atime: Option<fuser::TimeOrNow>,
|
||||
_mtime: Option<fuser::TimeOrNow>,
|
||||
_ctime: Option<SystemTime>,
|
||||
_fh: Option<u64>,
|
||||
_crtime: Option<SystemTime>,
|
||||
_chgtime: Option<SystemTime>,
|
||||
_bkuptime: Option<SystemTime>,
|
||||
_flags: Option<u32>,
|
||||
reply: ReplyAttr,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn symlink(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_link: &Path,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn link(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_ino: u64,
|
||||
_newparent: u64,
|
||||
_newname: &OsStr,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
|
||||
fn mknod(
|
||||
&mut self,
|
||||
_req: &Request,
|
||||
_parent: u64,
|
||||
_name: &OsStr,
|
||||
_mode: u32,
|
||||
_umask: u32,
|
||||
_rdev: u32,
|
||||
reply: ReplyEntry,
|
||||
) {
|
||||
reply.error(libc::EROFS);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_cache::TreeBuilder;
|
||||
use musicfs_core::{FileId, FileMeta, OriginId, RealPath, VirtualPath};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn make_file_meta(id: i64, vpath: &str, size: u64) -> FileMeta {
|
||||
FileMeta {
|
||||
id: FileId(id),
|
||||
virtual_path: VirtualPath::new(vpath),
|
||||
real_path: RealPath {
|
||||
origin_id: OriginId::from("test"),
|
||||
path: PathBuf::from("/test"),
|
||||
},
|
||||
size,
|
||||
mtime: SystemTime::now(),
|
||||
content_hash: None,
|
||||
audio: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tree_integration() {
|
||||
let runtime = tokio::runtime::Runtime::new().unwrap();
|
||||
let handle = runtime.handle().clone();
|
||||
|
||||
let mut builder = TreeBuilder::new();
|
||||
builder.add_file(&make_file_meta(1, "/Artist/Album/Track.flac", 30_000_000));
|
||||
let tree = Arc::new(RwLock::new(builder.build()));
|
||||
|
||||
let _fs = MusicFs::new(tree.clone(), handle);
|
||||
|
||||
let tree_read = tree.read();
|
||||
assert!(tree_read.get(ROOT_INODE).is_some());
|
||||
assert!(tree_read
|
||||
.get_by_path(&VirtualPath::new("/Artist"))
|
||||
.is_some());
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
mod filesystem;
|
||||
pub mod ops;
|
||||
|
||||
pub use filesystem::MusicFs;
|
||||
pub use ops::SearchOps;
|
||||
@@ -1,5 +0,0 @@
|
||||
mod prefetch;
|
||||
mod search;
|
||||
|
||||
pub use prefetch::PrefetchOps;
|
||||
pub use search::SearchOps;
|
||||
@@ -1,298 +0,0 @@
|
||||
use fuser::{FileAttr, FileType, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry};
|
||||
use musicfs_cache::{PatternStore, PrefetchConfig, PrefetchEngine};
|
||||
use musicfs_cas::ContentFetcher;
|
||||
use musicfs_core::{EventBus, FileId};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
const PREFETCH_DIR_INODE: u64 = 0xFFFF_FFFF_0000_0002;
|
||||
const PREFETCH_STATUS_INODE: u64 = 0xFFFF_FFFF_0000_0003;
|
||||
const PREFETCH_HINTS_BASE: u64 = 0xFFFF_FFFF_2000_0000;
|
||||
|
||||
pub struct PrefetchOps {
|
||||
pattern_store: Arc<PatternStore>,
|
||||
engine: Option<Arc<PrefetchEngine>>,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
}
|
||||
|
||||
impl PrefetchOps {
|
||||
pub fn new(pattern_store: Arc<PatternStore>, uid: u32, gid: u32) -> Self {
|
||||
Self {
|
||||
pattern_store,
|
||||
engine: None,
|
||||
uid,
|
||||
gid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_engine(
|
||||
pattern_store: Arc<PatternStore>,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
config: PrefetchConfig,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
) -> Self {
|
||||
let engine = Arc::new(PrefetchEngine::new(config, pattern_store.clone(), fetcher));
|
||||
|
||||
Self {
|
||||
pattern_store,
|
||||
engine: Some(engine),
|
||||
uid,
|
||||
gid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_engine(&self, event_bus: Arc<EventBus>) -> Option<musicfs_cache::PrefetchHandle> {
|
||||
self.engine
|
||||
.as_ref()
|
||||
.map(|e| e.clone().start(event_bus, self.pattern_store.clone()))
|
||||
}
|
||||
|
||||
pub fn is_prefetch_dir_name(name: &str) -> bool {
|
||||
name == ".prefetch"
|
||||
}
|
||||
|
||||
pub fn is_prefetch_inode(inode: u64) -> bool {
|
||||
inode == PREFETCH_DIR_INODE
|
||||
|| inode == PREFETCH_STATUS_INODE
|
||||
|| inode >= PREFETCH_HINTS_BASE
|
||||
}
|
||||
|
||||
pub fn prefetch_dir_inode() -> u64 {
|
||||
PREFETCH_DIR_INODE
|
||||
}
|
||||
|
||||
pub fn lookup_prefetch_dir(&self, reply: ReplyEntry) {
|
||||
let attr = self.dir_attr(PREFETCH_DIR_INODE);
|
||||
reply.entry(&Duration::from_secs(60), &attr, 0);
|
||||
}
|
||||
|
||||
pub fn lookup_status(&self, reply: ReplyEntry) {
|
||||
let status = self.generate_status();
|
||||
let attr = self.file_attr(PREFETCH_STATUS_INODE, status.len() as u64);
|
||||
reply.entry(&Duration::from_secs(1), &attr, 0);
|
||||
}
|
||||
|
||||
pub fn lookup_hint(&self, name: &str, reply: ReplyEntry) {
|
||||
if let Some(inode) = self.hint_name_to_inode(name) {
|
||||
let attr = self.file_attr(inode, 256);
|
||||
reply.entry(&Duration::from_secs(1), &attr, 0);
|
||||
} else {
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn getattr_prefetch_dir(&self, reply: ReplyAttr) {
|
||||
let attr = self.dir_attr(PREFETCH_DIR_INODE);
|
||||
reply.attr(&Duration::from_secs(60), &attr);
|
||||
}
|
||||
|
||||
pub fn getattr_status(&self, reply: ReplyAttr) {
|
||||
let status = self.generate_status();
|
||||
let attr = self.file_attr(PREFETCH_STATUS_INODE, status.len() as u64);
|
||||
reply.attr(&Duration::from_secs(1), &attr);
|
||||
}
|
||||
|
||||
pub fn getattr_hint(&self, inode: u64, reply: ReplyAttr) {
|
||||
let attr = self.file_attr(inode, 256);
|
||||
reply.attr(&Duration::from_secs(1), &attr);
|
||||
}
|
||||
|
||||
pub fn readdir_prefetch_root(&self, offset: i64, mut reply: ReplyDirectory) {
|
||||
let entries: Vec<(u64, FileType, &str)> = vec![
|
||||
(PREFETCH_DIR_INODE, FileType::Directory, "."),
|
||||
(1, FileType::Directory, ".."),
|
||||
(PREFETCH_STATUS_INODE, FileType::RegularFile, "status"),
|
||||
];
|
||||
|
||||
let recently_played = self.pattern_store.recently_played(7).unwrap_or_default();
|
||||
let predictions: Vec<(u64, FileType, String)> = recently_played
|
||||
.iter()
|
||||
.take(10)
|
||||
.enumerate()
|
||||
.map(|(i, file_id)| {
|
||||
let inode = PREFETCH_HINTS_BASE + i as u64;
|
||||
let name = format!("hint_{:04}", file_id.0);
|
||||
(inode, FileType::RegularFile, name)
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, *name) {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let base_offset = entries.len();
|
||||
for (i, (inode, kind, name)) in predictions.iter().enumerate() {
|
||||
let entry_offset = base_offset + i;
|
||||
if entry_offset < offset as usize {
|
||||
continue;
|
||||
}
|
||||
if reply.add(*inode, (entry_offset + 1) as i64, *kind, name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
pub fn read_status(&self, offset: i64, size: u32, reply: ReplyData) {
|
||||
let status = self.generate_status();
|
||||
let start = offset as usize;
|
||||
let end = std::cmp::min(start + size as usize, status.len());
|
||||
|
||||
if start >= status.len() {
|
||||
reply.data(&[]);
|
||||
} else {
|
||||
reply.data(&status.as_bytes()[start..end]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn read_hint(&self, inode: u64, offset: i64, size: u32, reply: ReplyData) {
|
||||
let file_id = self.inode_to_file_id(inode);
|
||||
let predictions = self.pattern_store.predict_next(file_id, 5);
|
||||
|
||||
let content = predictions
|
||||
.iter()
|
||||
.map(|id| format!("{}", id.0))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let start = offset as usize;
|
||||
let end = std::cmp::min(start + size as usize, content.len());
|
||||
|
||||
if start >= content.len() {
|
||||
reply.data(&[]);
|
||||
} else {
|
||||
reply.data(&content.as_bytes()[start..end]);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_status(&self) -> String {
|
||||
let engine_status = if let Some(engine) = &self.engine {
|
||||
format!(
|
||||
"running: {}\nin_flight: {}",
|
||||
engine.is_running(),
|
||||
engine.in_flight_count()
|
||||
)
|
||||
} else {
|
||||
"engine: disabled".to_string()
|
||||
};
|
||||
|
||||
let most_played = self
|
||||
.pattern_store
|
||||
.most_played(5)
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|id| format!("{}", id.0))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
|
||||
format!(
|
||||
"MusicFS Prefetch Status\n\
|
||||
=======================\n\
|
||||
{}\n\
|
||||
most_played: [{}]\n",
|
||||
engine_status, most_played
|
||||
)
|
||||
}
|
||||
|
||||
fn hint_name_to_inode(&self, name: &str) -> Option<u64> {
|
||||
if name.starts_with("hint_") {
|
||||
let id_str = name.strip_prefix("hint_")?;
|
||||
let id: i64 = id_str.parse().ok()?;
|
||||
Some(PREFETCH_HINTS_BASE + id as u64)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn inode_to_file_id(&self, inode: u64) -> FileId {
|
||||
FileId((inode - PREFETCH_HINTS_BASE) as i64)
|
||||
}
|
||||
|
||||
fn dir_attr(&self, inode: u64) -> FileAttr {
|
||||
FileAttr {
|
||||
ino: inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
kind: FileType::Directory,
|
||||
perm: 0o555,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: 512,
|
||||
flags: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn file_attr(&self, inode: u64, size: u64) -> FileAttr {
|
||||
FileAttr {
|
||||
ino: inode,
|
||||
size,
|
||||
blocks: (size + 511) / 512,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
kind: FileType::RegularFile,
|
||||
perm: 0o444,
|
||||
nlink: 1,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: 512,
|
||||
flags: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_prefetch_ops_new() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let pattern_store =
|
||||
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
|
||||
let _ops = PrefetchOps::new(pattern_store, 1000, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_prefetch_inode() {
|
||||
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_DIR_INODE));
|
||||
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_STATUS_INODE));
|
||||
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_HINTS_BASE));
|
||||
assert!(PrefetchOps::is_prefetch_inode(PREFETCH_HINTS_BASE + 100));
|
||||
assert!(!PrefetchOps::is_prefetch_inode(1));
|
||||
assert!(!PrefetchOps::is_prefetch_inode(1000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hint_name_to_inode() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let pattern_store =
|
||||
Arc::new(PatternStore::new(&dir.path().join("patterns.db"), 30).unwrap());
|
||||
let ops = PrefetchOps::new(pattern_store, 1000, 1000);
|
||||
|
||||
assert_eq!(
|
||||
ops.hint_name_to_inode("hint_0001"),
|
||||
Some(PREFETCH_HINTS_BASE + 1)
|
||||
);
|
||||
assert_eq!(
|
||||
ops.hint_name_to_inode("hint_9999"),
|
||||
Some(PREFETCH_HINTS_BASE + 9999)
|
||||
);
|
||||
assert_eq!(ops.hint_name_to_inode("invalid"), None);
|
||||
}
|
||||
}
|
||||
@@ -1,273 +0,0 @@
|
||||
use fuser::{FileAttr, FileType, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry};
|
||||
use moka::sync::Cache;
|
||||
use musicfs_search::{SearchHit, SearchIndex};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
const SEARCH_DIR_INODE: u64 = 0xFFFF_FFFF_0000_0001;
|
||||
const SEARCH_RESULT_BASE: u64 = 0xFFFF_FFFF_1000_0000;
|
||||
const RESULT_CACHE_MAX_ENTRIES: u64 = 1000;
|
||||
const RESULT_CACHE_TTL_SECS: u64 = 300;
|
||||
const INODE_CACHE_MAX_ENTRIES: u64 = 10000;
|
||||
const MAX_QUERY_LENGTH: usize = 256;
|
||||
|
||||
pub struct SearchOps {
|
||||
index: Arc<SearchIndex>,
|
||||
result_cache: Cache<String, Vec<SearchHit>>,
|
||||
inode_to_result: Cache<u64, (String, usize)>,
|
||||
mount_point: String,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
}
|
||||
|
||||
impl SearchOps {
|
||||
pub fn new(index: Arc<SearchIndex>, mount_point: &str, uid: u32, gid: u32) -> Self {
|
||||
let result_cache = Cache::builder()
|
||||
.max_capacity(RESULT_CACHE_MAX_ENTRIES)
|
||||
.time_to_live(Duration::from_secs(RESULT_CACHE_TTL_SECS))
|
||||
.build();
|
||||
|
||||
let inode_to_result = Cache::builder()
|
||||
.max_capacity(INODE_CACHE_MAX_ENTRIES)
|
||||
.time_to_live(Duration::from_secs(RESULT_CACHE_TTL_SECS))
|
||||
.build();
|
||||
|
||||
Self {
|
||||
index,
|
||||
result_cache,
|
||||
inode_to_result,
|
||||
mount_point: mount_point.to_string(),
|
||||
uid,
|
||||
gid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_search_dir_name(name: &str) -> bool {
|
||||
name == ".search"
|
||||
}
|
||||
|
||||
pub fn is_search_inode(inode: u64) -> bool {
|
||||
inode == SEARCH_DIR_INODE || inode >= SEARCH_RESULT_BASE
|
||||
}
|
||||
|
||||
pub fn search_dir_inode() -> u64 {
|
||||
SEARCH_DIR_INODE
|
||||
}
|
||||
|
||||
pub fn lookup_search_dir(&self, reply: ReplyEntry) {
|
||||
let attr = self.dir_attr(SEARCH_DIR_INODE);
|
||||
reply.entry(&Duration::from_secs(60), &attr, 0);
|
||||
}
|
||||
|
||||
pub fn lookup_query_dir(&self, query: &str, inode: u64, reply: ReplyEntry) {
|
||||
let results = self.execute_query(query);
|
||||
if results.is_empty() {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
|
||||
let attr = self.dir_attr(inode);
|
||||
reply.entry(&Duration::from_secs(1), &attr, 0);
|
||||
}
|
||||
|
||||
pub fn lookup_result(&self, inode: u64, reply: ReplyEntry) {
|
||||
if self.inode_to_result.contains_key(&inode) {
|
||||
let attr = self.symlink_attr(inode, 256);
|
||||
reply.entry(&Duration::from_secs(1), &attr, 0);
|
||||
} else {
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn getattr_search_dir(&self, reply: ReplyAttr) {
|
||||
let attr = self.dir_attr(SEARCH_DIR_INODE);
|
||||
reply.attr(&Duration::from_secs(60), &attr);
|
||||
}
|
||||
|
||||
pub fn getattr_result(&self, inode: u64, reply: ReplyAttr) {
|
||||
let attr = self.symlink_attr(inode, 256);
|
||||
reply.attr(&Duration::from_secs(1), &attr);
|
||||
}
|
||||
|
||||
pub fn readdir_search_root(&self, offset: i64, mut reply: ReplyDirectory) {
|
||||
let entries = vec![
|
||||
(SEARCH_DIR_INODE, FileType::Directory, "."),
|
||||
(1, FileType::Directory, ".."),
|
||||
];
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
pub fn readdir_query(&self, query: &str, offset: i64, mut reply: ReplyDirectory) {
|
||||
let results = self.execute_query(query);
|
||||
|
||||
let entries = vec![
|
||||
(SEARCH_DIR_INODE + 1, FileType::Directory, ".".to_string()),
|
||||
(SEARCH_DIR_INODE, FileType::Directory, "..".to_string()),
|
||||
];
|
||||
|
||||
for (i, (inode, kind, name)) in entries.iter().enumerate().skip(offset as usize) {
|
||||
if reply.add(*inode, (i + 1) as i64, *kind, name) {
|
||||
reply.ok();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let base_offset = entries.len();
|
||||
for (i, hit) in results.iter().enumerate() {
|
||||
let entry_offset = base_offset + i;
|
||||
if entry_offset < offset as usize {
|
||||
continue;
|
||||
}
|
||||
|
||||
let inode = SEARCH_RESULT_BASE + i as u64;
|
||||
let name = self.result_filename(hit, i);
|
||||
|
||||
self.inode_to_result.insert(inode, (query.to_string(), i));
|
||||
|
||||
if reply.add(inode, (entry_offset + 1) as i64, FileType::Symlink, &name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
pub fn readlink(&self, inode: u64, reply: ReplyData) {
|
||||
let (query, index) = match self.inode_to_result.get(&inode) {
|
||||
Some((q, i)) => (q, i),
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let results = self.execute_query(&query);
|
||||
if let Some(hit) = results.get(index) {
|
||||
if let Some(target) = self.safe_symlink_target(hit.virtual_path.as_str()) {
|
||||
reply.data(target.as_bytes());
|
||||
} else {
|
||||
reply.error(libc::EINVAL);
|
||||
}
|
||||
} else {
|
||||
reply.error(libc::ENOENT);
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_symlink_target(&self, virtual_path: &str) -> Option<String> {
|
||||
let normalized = Path::new(virtual_path).components().fold(
|
||||
std::path::PathBuf::new(),
|
||||
|mut acc, comp| {
|
||||
match comp {
|
||||
std::path::Component::Normal(s) => acc.push(s),
|
||||
std::path::Component::RootDir => acc.push("/"),
|
||||
_ => {}
|
||||
}
|
||||
acc
|
||||
},
|
||||
);
|
||||
|
||||
let path_str = normalized.to_string_lossy();
|
||||
if path_str.contains("..") {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(format!("{}{}", self.mount_point, path_str))
|
||||
}
|
||||
|
||||
fn execute_query(&self, query: &str) -> Vec<SearchHit> {
|
||||
let query = if query.len() > MAX_QUERY_LENGTH {
|
||||
&query[..MAX_QUERY_LENGTH]
|
||||
} else {
|
||||
query
|
||||
};
|
||||
|
||||
if let Some(results) = self.result_cache.get(query) {
|
||||
return results;
|
||||
}
|
||||
|
||||
let results = self.index.search(query, 1000).unwrap_or_default();
|
||||
self.result_cache.insert(query.to_string(), results.clone());
|
||||
results
|
||||
}
|
||||
|
||||
fn result_filename(&self, hit: &SearchHit, index: usize) -> String {
|
||||
let artist = hit.artist.as_deref().unwrap_or("Unknown");
|
||||
let title = hit.title.as_deref().unwrap_or("Unknown");
|
||||
let ext = hit
|
||||
.virtual_path
|
||||
.as_str()
|
||||
.rsplit('.')
|
||||
.next()
|
||||
.unwrap_or("flac");
|
||||
format!("{:03}. {} - {}.{}", index + 1, artist, title, ext)
|
||||
}
|
||||
|
||||
fn dir_attr(&self, inode: u64) -> FileAttr {
|
||||
FileAttr {
|
||||
ino: inode,
|
||||
size: 0,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
kind: FileType::Directory,
|
||||
perm: 0o555,
|
||||
nlink: 2,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: 512,
|
||||
flags: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn symlink_attr(&self, inode: u64, target_len: u64) -> FileAttr {
|
||||
FileAttr {
|
||||
ino: inode,
|
||||
size: target_len,
|
||||
blocks: 0,
|
||||
atime: SystemTime::UNIX_EPOCH,
|
||||
mtime: SystemTime::UNIX_EPOCH,
|
||||
ctime: SystemTime::UNIX_EPOCH,
|
||||
crtime: SystemTime::UNIX_EPOCH,
|
||||
kind: FileType::Symlink,
|
||||
perm: 0o777,
|
||||
nlink: 1,
|
||||
uid: self.uid,
|
||||
gid: self.gid,
|
||||
rdev: 0,
|
||||
blksize: 512,
|
||||
flags: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_search::SearchIndex;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_search_ops_new() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
|
||||
let _ops = SearchOps::new(index, "/mnt/music", 1000, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_search_inode() {
|
||||
assert!(SearchOps::is_search_inode(SEARCH_DIR_INODE));
|
||||
assert!(SearchOps::is_search_inode(SEARCH_RESULT_BASE));
|
||||
assert!(SearchOps::is_search_inode(SEARCH_RESULT_BASE + 100));
|
||||
assert!(!SearchOps::is_search_inode(1));
|
||||
assert!(!SearchOps::is_search_inode(1000));
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
[package]
|
||||
name = "musicfs-grpc"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-cache = { path = "../musicfs-cache" }
|
||||
musicfs-cas = { path = "../musicfs-cas" }
|
||||
musicfs-metadata = { path = "../musicfs-metadata" }
|
||||
musicfs-search = { path = "../musicfs-search" }
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
parking_lot.workspace = true
|
||||
tonic.workspace = true
|
||||
prost.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
chrono.workspace = true
|
||||
csv = "1.3"
|
||||
reqwest = { version = "0.11", features = ["json"] }
|
||||
hmac = "0.12"
|
||||
sha2 = "0.10"
|
||||
hex.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
tonic-build.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
@@ -1,4 +0,0 @@
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
tonic_build::compile_protos("proto/musicfs.proto")?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,322 +0,0 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package musicfs.v1;
|
||||
|
||||
option go_package = "homelab.lan/music-agregator/gen/musicfs/v1;musicfsv1";
|
||||
|
||||
service MusicFS {
|
||||
rpc Search(SearchRequest) returns (SearchResponse);
|
||||
rpc SearchStream(SearchRequest) returns (stream SearchResult);
|
||||
rpc GetStatus(Empty) returns (StatusResponse);
|
||||
rpc Shutdown(ShutdownRequest) returns (Empty);
|
||||
rpc GetCacheStats(Empty) returns (CacheStats);
|
||||
rpc ClearCache(ClearCacheRequest) returns (ClearCacheResponse);
|
||||
rpc Prefetch(PrefetchRequest) returns (stream PrefetchProgress);
|
||||
rpc ListOrigins(Empty) returns (OriginsResponse);
|
||||
rpc GetOriginHealth(OriginRequest) returns (OriginHealthResponse);
|
||||
rpc RescanOrigin(OriginRequest) returns (stream SyncProgress);
|
||||
rpc SubscribeEvents(EventFilter) returns (stream Event);
|
||||
}
|
||||
|
||||
service MetadataService {
|
||||
rpc GetMetadata(GetMetadataRequest) returns (MetadataResponse);
|
||||
rpc UpdateMetadata(UpdateMetadataRequest) returns (UpdateMetadataResponse);
|
||||
rpc ClearOverlay(ClearOverlayRequest) returns (ClearOverlayResponse);
|
||||
rpc BatchUpdateMetadata(BatchUpdateRequest) returns (stream BatchUpdateProgress);
|
||||
rpc ImportMetadata(ImportMetadataRequest) returns (stream ImportProgress);
|
||||
}
|
||||
|
||||
message Empty {}
|
||||
|
||||
message SearchRequest {
|
||||
string query = 1;
|
||||
optional uint32 limit = 2;
|
||||
optional uint32 offset = 3;
|
||||
optional string origin_id = 4;
|
||||
}
|
||||
|
||||
message SearchResponse {
|
||||
repeated SearchResult results = 1;
|
||||
uint64 total_matches = 2;
|
||||
uint32 query_time_ms = 3;
|
||||
}
|
||||
|
||||
message SearchResult {
|
||||
int64 file_id = 1;
|
||||
string virtual_path = 2;
|
||||
optional string artist = 3;
|
||||
optional string album = 4;
|
||||
optional string title = 5;
|
||||
float score = 6;
|
||||
map<string, string> highlights = 7;
|
||||
}
|
||||
|
||||
enum MountState {
|
||||
MOUNT_UNKNOWN = 0;
|
||||
MOUNT_MOUNTING = 1;
|
||||
MOUNT_READY = 2;
|
||||
MOUNT_SYNCING = 3;
|
||||
MOUNT_DEGRADED = 4;
|
||||
MOUNT_UNMOUNTING = 5;
|
||||
}
|
||||
|
||||
message StatusResponse {
|
||||
string version = 1;
|
||||
uint64 uptime_secs = 2;
|
||||
string mount_point = 3;
|
||||
MountState state = 4;
|
||||
uint32 open_file_handles = 5;
|
||||
uint64 fuse_ops_total = 6;
|
||||
uint64 files_indexed = 7;
|
||||
uint64 cache_size_bytes = 8;
|
||||
repeated OriginStatus origins = 9;
|
||||
}
|
||||
|
||||
message OriginStatus {
|
||||
string id = 1;
|
||||
string origin_type = 2;
|
||||
HealthStatus health = 3;
|
||||
uint64 files_count = 4;
|
||||
}
|
||||
|
||||
enum HealthStatus {
|
||||
HEALTH_UNKNOWN = 0;
|
||||
HEALTH_HEALTHY = 1;
|
||||
HEALTH_DEGRADED = 2;
|
||||
HEALTH_UNHEALTHY = 3;
|
||||
}
|
||||
|
||||
message ShutdownRequest {
|
||||
bool graceful = 1;
|
||||
uint32 timeout_secs = 2;
|
||||
}
|
||||
|
||||
message TierStats {
|
||||
uint64 entries = 1;
|
||||
uint64 size_bytes = 2;
|
||||
uint64 hits = 3;
|
||||
uint64 misses = 4;
|
||||
}
|
||||
|
||||
message CacheStats {
|
||||
uint64 total_size_bytes = 1;
|
||||
uint64 used_size_bytes = 2;
|
||||
uint64 size_limit_bytes = 3;
|
||||
uint64 chunk_count = 4;
|
||||
uint64 chunks_unique = 5;
|
||||
double dedup_ratio = 6;
|
||||
uint64 hit_count = 7;
|
||||
uint64 miss_count = 8;
|
||||
double hit_ratio = 9;
|
||||
uint64 metadata_entries = 10;
|
||||
uint64 metadata_bytes = 11;
|
||||
TierStats l1_metadata = 12;
|
||||
TierStats l2_headers = 13;
|
||||
TierStats l3_chunks = 14;
|
||||
}
|
||||
|
||||
message ClearCacheRequest {
|
||||
optional string origin_id = 1;
|
||||
bool clear_metadata = 2;
|
||||
bool clear_chunks = 3;
|
||||
}
|
||||
|
||||
message ClearCacheResponse {
|
||||
uint64 bytes_cleared = 1;
|
||||
uint64 chunks_cleared = 2;
|
||||
}
|
||||
|
||||
message PrefetchRequest {
|
||||
repeated string paths = 1;
|
||||
optional string origin_id = 2;
|
||||
}
|
||||
|
||||
message PrefetchProgress {
|
||||
string current_path = 1;
|
||||
uint32 completed = 2;
|
||||
uint32 total = 3;
|
||||
uint64 bytes_fetched = 4;
|
||||
}
|
||||
|
||||
message OriginsResponse {
|
||||
repeated OriginInfo origins = 1;
|
||||
}
|
||||
|
||||
message OriginInfo {
|
||||
string id = 1;
|
||||
string origin_type = 2;
|
||||
string display_name = 3;
|
||||
string root_path = 4;
|
||||
HealthStatus health = 5;
|
||||
uint64 files_count = 6;
|
||||
uint64 total_size_bytes = 7;
|
||||
}
|
||||
|
||||
message OriginRequest {
|
||||
string origin_id = 1;
|
||||
// Optional subdirectory to scope the scan (relative to origin root).
|
||||
// If empty, scans the entire origin.
|
||||
// Example: "Metallica - Master of Puppets (1986) [FLAC]"
|
||||
optional string subdir = 2;
|
||||
}
|
||||
|
||||
message OriginHealthResponse {
|
||||
string origin_id = 1;
|
||||
HealthStatus status = 2;
|
||||
optional string message = 3;
|
||||
uint64 last_check_secs = 4;
|
||||
}
|
||||
|
||||
message SyncProgress {
|
||||
string phase = 1;
|
||||
uint32 current = 2;
|
||||
uint32 total = 3;
|
||||
string current_path = 4;
|
||||
uint64 bytes_synced = 5;
|
||||
repeated SyncedFile new_files = 6;
|
||||
}
|
||||
|
||||
message SyncedFile {
|
||||
string path = 1;
|
||||
int64 file_id = 2;
|
||||
string virtual_path = 3;
|
||||
}
|
||||
|
||||
message EventFilter {
|
||||
repeated string event_types = 1;
|
||||
optional string origin_id = 2;
|
||||
}
|
||||
|
||||
message Event {
|
||||
string event_type = 1;
|
||||
int64 timestamp_ms = 2;
|
||||
optional string origin_id = 3;
|
||||
optional string path = 4;
|
||||
optional int64 file_id = 5;
|
||||
map<string, string> metadata = 6;
|
||||
}
|
||||
|
||||
// MetadataService messages
|
||||
|
||||
message GetMetadataRequest {
|
||||
string virtual_path = 1;
|
||||
}
|
||||
|
||||
message MetadataResponse {
|
||||
int64 file_id = 1;
|
||||
optional string title = 2;
|
||||
optional string artist = 3;
|
||||
optional string album = 4;
|
||||
optional string album_artist = 5;
|
||||
optional uint32 year = 6;
|
||||
optional uint32 track = 7;
|
||||
optional uint32 disc = 8;
|
||||
optional string genre = 9;
|
||||
optional string format = 10;
|
||||
optional uint64 duration_ms = 11;
|
||||
optional uint64 bitrate = 12;
|
||||
optional uint32 track_total = 13;
|
||||
optional uint32 disc_total = 14;
|
||||
optional string date = 15;
|
||||
optional string composer = 16;
|
||||
optional string comment = 17;
|
||||
optional string lyrics = 18;
|
||||
optional string copyright = 19;
|
||||
optional bool compilation = 20;
|
||||
optional string artist_sort = 21;
|
||||
optional string album_artist_sort = 22;
|
||||
optional string album_sort = 23;
|
||||
optional string title_sort = 24;
|
||||
optional string mb_recording_id = 25;
|
||||
optional string mb_album_id = 26;
|
||||
optional string mb_artist_id = 27;
|
||||
optional string mb_album_artist_id = 28;
|
||||
optional string mb_release_group_id = 29;
|
||||
optional float replaygain_track_gain = 30;
|
||||
optional float replaygain_track_peak = 31;
|
||||
optional float replaygain_album_gain = 32;
|
||||
optional float replaygain_album_peak = 33;
|
||||
optional uint32 channels = 34;
|
||||
optional uint32 bits_per_sample = 35;
|
||||
optional string encoder = 36;
|
||||
optional string label = 40;
|
||||
optional string album_type = 41;
|
||||
optional string cover_url = 42;
|
||||
map<string, string> custom_tags = 50;
|
||||
}
|
||||
|
||||
message UpdateMetadataRequest {
|
||||
int64 file_id = 1;
|
||||
optional string title = 2;
|
||||
optional string artist = 3;
|
||||
optional string album = 4;
|
||||
optional string album_artist = 5;
|
||||
optional uint32 track_number = 6;
|
||||
optional uint32 disc_number = 7;
|
||||
optional string date = 8;
|
||||
optional string genre = 9;
|
||||
optional string composer = 10;
|
||||
optional string comment = 11;
|
||||
optional string lyrics = 12;
|
||||
optional string copyright = 13;
|
||||
optional bool compilation = 14;
|
||||
optional string artist_sort = 15;
|
||||
optional string album_artist_sort = 16;
|
||||
optional string album_sort = 17;
|
||||
optional string title_sort = 18;
|
||||
optional string mb_recording_id = 20;
|
||||
optional string mb_album_id = 21;
|
||||
optional string mb_artist_id = 22;
|
||||
optional float replaygain_track_gain = 30;
|
||||
optional float replaygain_track_peak = 31;
|
||||
optional float replaygain_album_gain = 32;
|
||||
optional float replaygain_album_peak = 33;
|
||||
optional string label = 40;
|
||||
optional string album_type = 41;
|
||||
optional string cover_url = 42;
|
||||
map<string, string> custom_tags = 50;
|
||||
}
|
||||
|
||||
message UpdateMetadataResponse {
|
||||
int64 file_id = 1;
|
||||
bool success = 2;
|
||||
optional string error_message = 3;
|
||||
}
|
||||
|
||||
message ClearOverlayRequest {
|
||||
int64 file_id = 1;
|
||||
}
|
||||
|
||||
message ClearOverlayResponse {
|
||||
int64 file_id = 1;
|
||||
bool success = 2;
|
||||
optional string error_message = 3;
|
||||
}
|
||||
|
||||
message BatchUpdateRequest {
|
||||
repeated BatchUpdateItem items = 1;
|
||||
}
|
||||
|
||||
message BatchUpdateItem {
|
||||
int64 file_id = 1;
|
||||
UpdateMetadataRequest metadata = 2;
|
||||
}
|
||||
|
||||
message BatchUpdateProgress {
|
||||
uint32 completed = 1;
|
||||
uint32 total = 2;
|
||||
optional int64 current_file_id = 3;
|
||||
optional string error_message = 4;
|
||||
}
|
||||
|
||||
message ImportMetadataRequest {
|
||||
string source_path = 1;
|
||||
optional string format = 2;
|
||||
}
|
||||
|
||||
message ImportProgress {
|
||||
uint32 imported = 1;
|
||||
uint32 total = 2;
|
||||
optional string current_file = 3;
|
||||
optional string error_message = 4;
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
pub mod proto {
|
||||
pub mod musicfs {
|
||||
pub mod v1 {
|
||||
tonic::include_proto!("musicfs.v1");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod metadata;
|
||||
pub mod scanner;
|
||||
mod search_service;
|
||||
mod server;
|
||||
mod webhook;
|
||||
|
||||
pub use metadata::MetadataServiceImpl;
|
||||
pub use proto::musicfs::v1::metadata_service_server::MetadataServiceServer;
|
||||
pub use proto::musicfs::v1::music_fs_server::{MusicFs, MusicFsServer as MusicFsGrpcServer};
|
||||
pub use proto::musicfs::v1::*;
|
||||
pub use search_service::SearchService;
|
||||
pub use server::MusicFsServer;
|
||||
pub use webhook::{WebhookConfig, WebhookHandler, WebhookPayload};
|
||||
@@ -1,794 +0,0 @@
|
||||
//! MetadataService gRPC handlers for metadata overlay operations.
|
||||
|
||||
use crate::proto::musicfs::v1::{
|
||||
metadata_service_server::MetadataService, BatchUpdateProgress, BatchUpdateRequest,
|
||||
ClearOverlayRequest, ClearOverlayResponse, GetMetadataRequest, ImportMetadataRequest,
|
||||
ImportProgress, MetadataResponse, UpdateMetadataRequest, UpdateMetadataResponse,
|
||||
};
|
||||
use musicfs_cache::{Database, EnrichmentUpdate};
|
||||
use musicfs_core::{AudioMeta, FileId, VirtualPath};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::{debug, info, instrument, warn};
|
||||
|
||||
/// gRPC service implementation for metadata operations.
|
||||
pub struct MetadataServiceImpl {
|
||||
db: Arc<Database>,
|
||||
}
|
||||
|
||||
impl MetadataServiceImpl {
|
||||
/// Create a new MetadataServiceImpl with the given database.
|
||||
pub fn new(db: Arc<Database>) -> Self {
|
||||
Self { db }
|
||||
}
|
||||
|
||||
/// Convert AudioMeta to MetadataResponse proto message.
|
||||
fn audio_meta_to_response(file_id: FileId, meta: &AudioMeta) -> MetadataResponse {
|
||||
MetadataResponse {
|
||||
file_id: file_id.0,
|
||||
title: meta.title.clone(),
|
||||
artist: meta.artist.clone(),
|
||||
album: meta.album.clone(),
|
||||
album_artist: meta.album_artist.clone(),
|
||||
year: meta.year,
|
||||
track: meta.track,
|
||||
disc: meta.disc,
|
||||
genre: meta.genre.clone(),
|
||||
format: Some(format!("{:?}", meta.format)),
|
||||
duration_ms: meta.duration_ms,
|
||||
bitrate: meta.bitrate.map(|b| b as u64),
|
||||
track_total: meta.track_total,
|
||||
disc_total: meta.disc_total,
|
||||
date: meta.date.clone(),
|
||||
composer: meta.composer.clone(),
|
||||
comment: meta.comment.clone(),
|
||||
lyrics: meta.lyrics.clone(),
|
||||
copyright: meta.copyright.clone(),
|
||||
compilation: meta.compilation,
|
||||
artist_sort: meta.artist_sort.clone(),
|
||||
album_artist_sort: meta.album_artist_sort.clone(),
|
||||
album_sort: meta.album_sort.clone(),
|
||||
title_sort: meta.title_sort.clone(),
|
||||
mb_recording_id: meta.mb_recording_id.clone(),
|
||||
mb_album_id: meta.mb_album_id.clone(),
|
||||
mb_artist_id: meta.mb_artist_id.clone(),
|
||||
mb_album_artist_id: meta.mb_album_artist_id.clone(),
|
||||
mb_release_group_id: meta.mb_release_group_id.clone(),
|
||||
replaygain_track_gain: meta.replaygain_track_gain,
|
||||
replaygain_track_peak: meta.replaygain_track_peak,
|
||||
replaygain_album_gain: meta.replaygain_album_gain,
|
||||
replaygain_album_peak: meta.replaygain_album_peak,
|
||||
channels: meta.channels,
|
||||
bits_per_sample: meta.bits_per_sample,
|
||||
encoder: meta.encoder.clone(),
|
||||
label: None,
|
||||
album_type: None,
|
||||
cover_url: None,
|
||||
custom_tags: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert UpdateMetadataRequest to AudioMeta for database update.
|
||||
fn request_to_audio_meta(req: &UpdateMetadataRequest) -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: req.title.clone(),
|
||||
artist: req.artist.clone(),
|
||||
album: req.album.clone(),
|
||||
album_artist: req.album_artist.clone(),
|
||||
genre: req.genre.clone(),
|
||||
year: None,
|
||||
track: req.track_number,
|
||||
disc: req.disc_number,
|
||||
duration_ms: None,
|
||||
bitrate: None,
|
||||
sample_rate: None,
|
||||
format: musicfs_core::AudioFormat::Unknown,
|
||||
track_total: None,
|
||||
disc_total: None,
|
||||
date: req.date.clone(),
|
||||
composer: req.composer.clone(),
|
||||
comment: req.comment.clone(),
|
||||
lyrics: req.lyrics.clone(),
|
||||
copyright: req.copyright.clone(),
|
||||
compilation: req.compilation,
|
||||
artist_sort: req.artist_sort.clone(),
|
||||
album_artist_sort: req.album_artist_sort.clone(),
|
||||
album_sort: req.album_sort.clone(),
|
||||
title_sort: req.title_sort.clone(),
|
||||
mb_recording_id: req.mb_recording_id.clone(),
|
||||
mb_album_id: req.mb_album_id.clone(),
|
||||
mb_artist_id: req.mb_artist_id.clone(),
|
||||
mb_album_artist_id: None,
|
||||
mb_release_group_id: None,
|
||||
replaygain_track_gain: req.replaygain_track_gain,
|
||||
replaygain_track_peak: req.replaygain_track_peak,
|
||||
replaygain_album_gain: req.replaygain_album_gain,
|
||||
replaygain_album_peak: req.replaygain_album_peak,
|
||||
channels: None,
|
||||
bits_per_sample: None,
|
||||
encoder: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl MetadataService for MetadataServiceImpl {
|
||||
#[instrument(level = "debug", skip(self, request), fields(method = "get_metadata"))]
|
||||
async fn get_metadata(
|
||||
&self,
|
||||
request: Request<GetMetadataRequest>,
|
||||
) -> Result<Response<MetadataResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
debug!(virtual_path = %req.virtual_path, "GetMetadata request");
|
||||
|
||||
if req.virtual_path.is_empty() {
|
||||
return Err(Status::invalid_argument("virtual_path cannot be empty"));
|
||||
}
|
||||
|
||||
let vpath = VirtualPath::new(&req.virtual_path);
|
||||
|
||||
let file_meta = self
|
||||
.db
|
||||
.get_file_by_virtual_path(&vpath)
|
||||
.map_err(|e| Status::internal(format!("Database error: {}", e)))?
|
||||
.ok_or_else(|| Status::not_found(format!("File not found: {}", req.virtual_path)))?;
|
||||
|
||||
let audio_meta = self
|
||||
.db
|
||||
.get_file_metadata_row(file_meta.id)
|
||||
.map_err(|e| Status::internal(format!("Failed to get metadata: {}", e)))?;
|
||||
|
||||
let response = Self::audio_meta_to_response(file_meta.id, &audio_meta);
|
||||
Ok(Response::new(response))
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "update_metadata")
|
||||
)]
|
||||
async fn update_metadata(
|
||||
&self,
|
||||
request: Request<UpdateMetadataRequest>,
|
||||
) -> Result<Response<UpdateMetadataResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let file_id = FileId(req.file_id);
|
||||
info!(file_id = req.file_id, "UpdateMetadata request");
|
||||
|
||||
if req.file_id <= 0 {
|
||||
return Err(Status::invalid_argument("file_id must be positive"));
|
||||
}
|
||||
|
||||
let audio_meta = Self::request_to_audio_meta(&req);
|
||||
|
||||
if let Err(e) = self.db.update_metadata(file_id, &audio_meta) {
|
||||
warn!(file_id = req.file_id, error = %e, "Failed to update metadata");
|
||||
return Ok(Response::new(UpdateMetadataResponse {
|
||||
file_id: req.file_id,
|
||||
success: false,
|
||||
error_message: Some(e.to_string()),
|
||||
}));
|
||||
}
|
||||
|
||||
if req.label.is_some() || req.album_type.is_some() || req.cover_url.is_some() {
|
||||
let enrichment = EnrichmentUpdate {
|
||||
label: req.label.clone(),
|
||||
album_type: req.album_type.clone(),
|
||||
cover_url: req.cover_url.clone(),
|
||||
genres_json: None,
|
||||
primary_genre: None,
|
||||
source: "orchestrator".to_string(),
|
||||
};
|
||||
if let Err(e) = self.db.update_enrichment(file_id, &enrichment) {
|
||||
warn!(file_id = req.file_id, error = %e, "Failed to update enrichment");
|
||||
return Ok(Response::new(UpdateMetadataResponse {
|
||||
file_id: req.file_id,
|
||||
success: false,
|
||||
error_message: Some(e.to_string()),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
debug!(file_id = req.file_id, "Metadata updated successfully");
|
||||
Ok(Response::new(UpdateMetadataResponse {
|
||||
file_id: req.file_id,
|
||||
success: true,
|
||||
error_message: None,
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "clear_overlay"))]
|
||||
async fn clear_overlay(
|
||||
&self,
|
||||
request: Request<ClearOverlayRequest>,
|
||||
) -> Result<Response<ClearOverlayResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let file_id = FileId(req.file_id);
|
||||
info!(file_id = req.file_id, "ClearOverlay request");
|
||||
|
||||
if req.file_id <= 0 {
|
||||
return Err(Status::invalid_argument("file_id must be positive"));
|
||||
}
|
||||
|
||||
match self.db.clear_overlay(file_id) {
|
||||
Ok(()) => {
|
||||
debug!(file_id = req.file_id, "Overlay cleared successfully");
|
||||
Ok(Response::new(ClearOverlayResponse {
|
||||
file_id: req.file_id,
|
||||
success: true,
|
||||
error_message: None,
|
||||
}))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(file_id = req.file_id, error = %e, "Failed to clear overlay");
|
||||
Ok(Response::new(ClearOverlayResponse {
|
||||
file_id: req.file_id,
|
||||
success: false,
|
||||
error_message: Some(e.to_string()),
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type BatchUpdateMetadataStream = ReceiverStream<Result<BatchUpdateProgress, Status>>;
|
||||
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "batch_update_metadata")
|
||||
)]
|
||||
async fn batch_update_metadata(
|
||||
&self,
|
||||
request: Request<BatchUpdateRequest>,
|
||||
) -> Result<Response<Self::BatchUpdateMetadataStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
let total = req.items.len() as u32;
|
||||
info!(item_count = total, "BatchUpdateMetadata request");
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
let db = Arc::clone(&self.db);
|
||||
|
||||
tokio::spawn(async move {
|
||||
for (i, item) in req.items.into_iter().enumerate() {
|
||||
let file_id = FileId(item.file_id);
|
||||
let completed = (i + 1) as u32;
|
||||
|
||||
let error_message = if let Some(ref metadata_req) = item.metadata {
|
||||
let audio_meta = MetadataServiceImpl::request_to_audio_meta(metadata_req);
|
||||
match db.update_metadata(file_id, &audio_meta) {
|
||||
Ok(()) => {
|
||||
if metadata_req.label.is_some()
|
||||
|| metadata_req.album_type.is_some()
|
||||
|| metadata_req.cover_url.is_some()
|
||||
{
|
||||
let enrichment = EnrichmentUpdate {
|
||||
label: metadata_req.label.clone(),
|
||||
album_type: metadata_req.album_type.clone(),
|
||||
cover_url: metadata_req.cover_url.clone(),
|
||||
genres_json: None,
|
||||
primary_genre: None,
|
||||
source: "orchestrator".to_string(),
|
||||
};
|
||||
if let Err(e) = db.update_enrichment(file_id, &enrichment) {
|
||||
Some(e.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(e) => Some(e.to_string()),
|
||||
}
|
||||
} else {
|
||||
Some("Missing metadata in batch item".to_string())
|
||||
};
|
||||
|
||||
let progress = BatchUpdateProgress {
|
||||
completed,
|
||||
total,
|
||||
current_file_id: Some(item.file_id),
|
||||
error_message,
|
||||
};
|
||||
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
type ImportMetadataStream = ReceiverStream<Result<ImportProgress, Status>>;
|
||||
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "import_metadata")
|
||||
)]
|
||||
async fn import_metadata(
|
||||
&self,
|
||||
request: Request<ImportMetadataRequest>,
|
||||
) -> Result<Response<Self::ImportMetadataStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(source_path = %req.source_path, format = ?req.format, "ImportMetadata request");
|
||||
|
||||
if req.source_path.is_empty() {
|
||||
return Err(Status::invalid_argument("source_path cannot be empty"));
|
||||
}
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
let db = Arc::clone(&self.db);
|
||||
let source_path = req.source_path.clone();
|
||||
let format = req.format.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let file_format = format.as_deref().unwrap_or_else(|| {
|
||||
if source_path.ends_with(".csv") {
|
||||
"csv"
|
||||
} else if source_path.ends_with(".json") {
|
||||
"json"
|
||||
} else {
|
||||
"unknown"
|
||||
}
|
||||
});
|
||||
|
||||
let content = match tokio::fs::read_to_string(&source_path).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Ok(ImportProgress {
|
||||
imported: 0,
|
||||
total: 0,
|
||||
current_file: None,
|
||||
error_message: Some(format!("Failed to read file: {}", e)),
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let entries: Vec<ImportEntry> = match file_format {
|
||||
"json" => match serde_json::from_str::<Vec<ImportEntry>>(&content) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Ok(ImportProgress {
|
||||
imported: 0,
|
||||
total: 0,
|
||||
current_file: None,
|
||||
error_message: Some(format!("Failed to parse JSON: {}", e)),
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
"csv" => match parse_csv_entries(&content) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Ok(ImportProgress {
|
||||
imported: 0,
|
||||
total: 0,
|
||||
current_file: None,
|
||||
error_message: Some(format!("Failed to parse CSV: {}", e)),
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
let _ = tx
|
||||
.send(Ok(ImportProgress {
|
||||
imported: 0,
|
||||
total: 0,
|
||||
current_file: None,
|
||||
error_message: Some(format!("Unsupported format: {}", file_format)),
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let total = entries.len() as u32;
|
||||
let mut imported = 0u32;
|
||||
|
||||
for entry in entries {
|
||||
let vpath = VirtualPath::new(&entry.virtual_path);
|
||||
|
||||
let file_meta = match db.get_file_by_virtual_path(&vpath) {
|
||||
Ok(Some(f)) => f,
|
||||
Ok(None) => {
|
||||
let progress = ImportProgress {
|
||||
imported,
|
||||
total,
|
||||
current_file: Some(entry.virtual_path.clone()),
|
||||
error_message: Some(format!("File not found: {}", entry.virtual_path)),
|
||||
};
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
let progress = ImportProgress {
|
||||
imported,
|
||||
total,
|
||||
current_file: Some(entry.virtual_path.clone()),
|
||||
error_message: Some(format!("Database error: {}", e)),
|
||||
};
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let audio_meta = entry.to_audio_meta();
|
||||
let error_message = match db.update_metadata(file_meta.id, &audio_meta) {
|
||||
Ok(()) => {
|
||||
imported += 1;
|
||||
None
|
||||
}
|
||||
Err(e) => Some(e.to_string()),
|
||||
};
|
||||
|
||||
let progress = ImportProgress {
|
||||
imported,
|
||||
total,
|
||||
current_file: Some(entry.virtual_path),
|
||||
error_message,
|
||||
};
|
||||
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Entry from import file (CSV or JSON).
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
struct ImportEntry {
|
||||
virtual_path: String,
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
artist: Option<String>,
|
||||
#[serde(default)]
|
||||
album: Option<String>,
|
||||
#[serde(default)]
|
||||
album_artist: Option<String>,
|
||||
#[serde(default)]
|
||||
genre: Option<String>,
|
||||
#[serde(default)]
|
||||
year: Option<u32>,
|
||||
#[serde(default)]
|
||||
track: Option<u32>,
|
||||
#[serde(default)]
|
||||
disc: Option<u32>,
|
||||
#[serde(default)]
|
||||
date: Option<String>,
|
||||
#[serde(default)]
|
||||
composer: Option<String>,
|
||||
#[serde(default)]
|
||||
comment: Option<String>,
|
||||
}
|
||||
|
||||
impl ImportEntry {
|
||||
fn to_audio_meta(&self) -> AudioMeta {
|
||||
AudioMeta {
|
||||
title: self.title.clone(),
|
||||
artist: self.artist.clone(),
|
||||
album: self.album.clone(),
|
||||
album_artist: self.album_artist.clone(),
|
||||
genre: self.genre.clone(),
|
||||
year: self.year,
|
||||
track: self.track,
|
||||
disc: self.disc,
|
||||
duration_ms: None,
|
||||
bitrate: None,
|
||||
sample_rate: None,
|
||||
format: musicfs_core::AudioFormat::Unknown,
|
||||
track_total: None,
|
||||
disc_total: None,
|
||||
date: self.date.clone(),
|
||||
composer: self.composer.clone(),
|
||||
comment: self.comment.clone(),
|
||||
lyrics: None,
|
||||
copyright: None,
|
||||
compilation: None,
|
||||
artist_sort: None,
|
||||
album_artist_sort: None,
|
||||
album_sort: None,
|
||||
title_sort: None,
|
||||
mb_recording_id: None,
|
||||
mb_album_id: None,
|
||||
mb_artist_id: None,
|
||||
mb_album_artist_id: None,
|
||||
mb_release_group_id: None,
|
||||
replaygain_track_gain: None,
|
||||
replaygain_track_peak: None,
|
||||
replaygain_album_gain: None,
|
||||
replaygain_album_peak: None,
|
||||
channels: None,
|
||||
bits_per_sample: None,
|
||||
encoder: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse CSV content into ImportEntry list.
|
||||
fn parse_csv_entries(content: &str) -> Result<Vec<ImportEntry>, String> {
|
||||
let mut reader = csv::Reader::from_reader(content.as_bytes());
|
||||
let mut entries = Vec::new();
|
||||
|
||||
for result in reader.deserialize() {
|
||||
let entry: ImportEntry = result.map_err(|e| format!("CSV parse error: {}", e))?;
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::proto::musicfs::v1::BatchUpdateItem;
|
||||
use musicfs_core::{AudioFormat, OriginId};
|
||||
use std::path::Path;
|
||||
use std::time::UNIX_EPOCH;
|
||||
use tempfile::TempDir;
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
fn create_test_db() -> (TempDir, Arc<Database>) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let db = Arc::new(Database::open_memory().unwrap());
|
||||
(dir, db)
|
||||
}
|
||||
|
||||
fn insert_test_file(db: &Database, vpath: &str) -> FileId {
|
||||
let real_path = format!("/music{}", vpath);
|
||||
db.upsert_file(
|
||||
&OriginId::from("local"),
|
||||
Path::new(&real_path),
|
||||
&VirtualPath::new(vpath),
|
||||
&AudioMeta {
|
||||
title: Some("Test Track".to_string()),
|
||||
artist: Some("Test Artist".to_string()),
|
||||
album: Some("Test Album".to_string()),
|
||||
format: AudioFormat::Flac,
|
||||
..Default::default()
|
||||
},
|
||||
UNIX_EPOCH,
|
||||
1000,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_metadata_success() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let vpath = "/Artist/Album/Track.flac";
|
||||
insert_test_file(&db, vpath);
|
||||
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
let request = Request::new(GetMetadataRequest {
|
||||
virtual_path: vpath.to_string(),
|
||||
});
|
||||
|
||||
let response = service.get_metadata(request).await.unwrap();
|
||||
let meta = response.into_inner();
|
||||
|
||||
assert_eq!(meta.title, Some("Test Track".to_string()));
|
||||
assert_eq!(meta.artist, Some("Test Artist".to_string()));
|
||||
assert_eq!(meta.album, Some("Test Album".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_metadata_not_found() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(GetMetadataRequest {
|
||||
virtual_path: "/nonexistent.flac".to_string(),
|
||||
});
|
||||
|
||||
let result = service.get_metadata(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_metadata_empty_path() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(GetMetadataRequest {
|
||||
virtual_path: String::new(),
|
||||
});
|
||||
|
||||
let result = service.get_metadata(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_update_metadata_success() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let vpath = "/Artist/Album/Track.flac";
|
||||
let file_id = insert_test_file(&db, vpath);
|
||||
|
||||
let service = MetadataServiceImpl::new(db.clone());
|
||||
let request = Request::new(UpdateMetadataRequest {
|
||||
file_id: file_id.0,
|
||||
title: Some("Updated Title".to_string()),
|
||||
artist: Some("Updated Artist".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let response = service.update_metadata(request).await.unwrap();
|
||||
let result = response.into_inner();
|
||||
|
||||
assert!(result.success);
|
||||
assert!(result.error_message.is_none());
|
||||
|
||||
let meta = db.get_file_metadata_row(file_id).unwrap();
|
||||
assert_eq!(meta.title, Some("Updated Title".to_string()));
|
||||
assert_eq!(meta.artist, Some("Updated Artist".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_update_metadata_invalid_id() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(UpdateMetadataRequest {
|
||||
file_id: 0,
|
||||
title: Some("Title".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let result = service.update_metadata(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clear_overlay_success() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let vpath = "/Artist/Album/Track.flac";
|
||||
let file_id = insert_test_file(&db, vpath);
|
||||
|
||||
let service = MetadataServiceImpl::new(db.clone());
|
||||
let request = Request::new(ClearOverlayRequest { file_id: file_id.0 });
|
||||
|
||||
let response = service.clear_overlay(request).await.unwrap();
|
||||
let result = response.into_inner();
|
||||
|
||||
assert!(result.success);
|
||||
assert!(result.error_message.is_none());
|
||||
|
||||
let meta = db.get_file_metadata_row(file_id).unwrap();
|
||||
assert!(meta.title.is_none());
|
||||
assert!(meta.artist.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clear_overlay_invalid_id() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(ClearOverlayRequest { file_id: -1 });
|
||||
|
||||
let result = service.clear_overlay(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_batch_update_metadata() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let file_id1 = insert_test_file(&db, "/Track1.flac");
|
||||
let file_id2 = insert_test_file(&db, "/Track2.flac");
|
||||
|
||||
let service = MetadataServiceImpl::new(db.clone());
|
||||
let request = Request::new(BatchUpdateRequest {
|
||||
items: vec![
|
||||
BatchUpdateItem {
|
||||
file_id: file_id1.0,
|
||||
metadata: Some(UpdateMetadataRequest {
|
||||
file_id: file_id1.0,
|
||||
title: Some("Batch Title 1".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
BatchUpdateItem {
|
||||
file_id: file_id2.0,
|
||||
metadata: Some(UpdateMetadataRequest {
|
||||
file_id: file_id2.0,
|
||||
title: Some("Batch Title 2".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
let response = service.batch_update_metadata(request).await.unwrap();
|
||||
let mut stream = response.into_inner();
|
||||
|
||||
let mut progress_count = 0;
|
||||
while let Some(Ok(result)) = stream.next().await {
|
||||
progress_count += 1;
|
||||
assert!(result.error_message.is_none());
|
||||
}
|
||||
|
||||
assert_eq!(progress_count, 2);
|
||||
|
||||
let meta1 = db.get_file_metadata_row(file_id1).unwrap();
|
||||
assert_eq!(meta1.title, Some("Batch Title 1".to_string()));
|
||||
|
||||
let meta2 = db.get_file_metadata_row(file_id2).unwrap();
|
||||
assert_eq!(meta2.title, Some("Batch Title 2".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_import_metadata_empty_path() {
|
||||
let (_dir, db) = create_test_db();
|
||||
let service = MetadataServiceImpl::new(db);
|
||||
|
||||
let request = Request::new(ImportMetadataRequest {
|
||||
source_path: String::new(),
|
||||
format: None,
|
||||
});
|
||||
|
||||
let result = service.import_metadata(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_csv_entries() {
|
||||
let csv_content = r#"virtual_path,title,artist,album
|
||||
/Track1.flac,Title 1,Artist 1,Album 1
|
||||
/Track2.flac,Title 2,Artist 2,Album 2"#;
|
||||
|
||||
let entries = parse_csv_entries(csv_content).unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(entries[0].virtual_path, "/Track1.flac");
|
||||
assert_eq!(entries[0].title, Some("Title 1".to_string()));
|
||||
assert_eq!(entries[1].virtual_path, "/Track2.flac");
|
||||
assert_eq!(entries[1].artist, Some("Artist 2".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_import_entry_to_audio_meta() {
|
||||
let entry = ImportEntry {
|
||||
virtual_path: "/test.flac".to_string(),
|
||||
title: Some("Test".to_string()),
|
||||
artist: Some("Artist".to_string()),
|
||||
album: None,
|
||||
album_artist: None,
|
||||
genre: Some("Rock".to_string()),
|
||||
year: Some(2024),
|
||||
track: Some(1),
|
||||
disc: None,
|
||||
date: None,
|
||||
composer: None,
|
||||
comment: None,
|
||||
};
|
||||
|
||||
let meta = entry.to_audio_meta();
|
||||
assert_eq!(meta.title, Some("Test".to_string()));
|
||||
assert_eq!(meta.artist, Some("Artist".to_string()));
|
||||
assert_eq!(meta.genre, Some("Rock".to_string()));
|
||||
assert_eq!(meta.year, Some(2024));
|
||||
assert_eq!(meta.track, Some(1));
|
||||
}
|
||||
}
|
||||
@@ -1,261 +0,0 @@
|
||||
use musicfs_cache::{Database, VirtualTree};
|
||||
use musicfs_cas::ContentFetcher;
|
||||
use musicfs_core::{
|
||||
AudioMeta, Error, Event, EventBus, FileId, FileMeta, OriginId, RealPath, Result, VirtualPath,
|
||||
};
|
||||
use musicfs_metadata::MetadataParser;
|
||||
use parking_lot::RwLock;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::UNIX_EPOCH;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub struct ScanResult {
|
||||
pub new_files: Vec<SyncedFileInfo>,
|
||||
pub changed: u32,
|
||||
pub deleted: u32,
|
||||
pub unchanged: u32,
|
||||
pub bytes_synced: u64,
|
||||
}
|
||||
|
||||
pub struct SyncedFileInfo {
|
||||
pub path: String,
|
||||
pub file_id: FileId,
|
||||
pub virtual_path: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScanProgress {
|
||||
pub phase: String,
|
||||
pub current: u32,
|
||||
pub total: u32,
|
||||
pub current_path: String,
|
||||
pub bytes_synced: u64,
|
||||
}
|
||||
|
||||
pub struct OriginScanner {
|
||||
db: Arc<Database>,
|
||||
event_bus: Arc<EventBus>,
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
parser: MetadataParser,
|
||||
}
|
||||
|
||||
impl OriginScanner {
|
||||
pub fn new(
|
||||
db: Arc<Database>,
|
||||
event_bus: Arc<EventBus>,
|
||||
tree: Arc<RwLock<VirtualTree>>,
|
||||
fetcher: Arc<ContentFetcher>,
|
||||
) -> Self {
|
||||
Self {
|
||||
db,
|
||||
event_bus,
|
||||
tree,
|
||||
fetcher,
|
||||
parser: MetadataParser,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn scan(
|
||||
&self,
|
||||
origin_id: &OriginId,
|
||||
origin_root: &Path,
|
||||
subdir: Option<&str>,
|
||||
progress_tx: mpsc::Sender<ScanProgress>,
|
||||
) -> Result<ScanResult> {
|
||||
let scan_root = match subdir {
|
||||
Some(sub) if !sub.is_empty() => origin_root.join(sub),
|
||||
_ => origin_root.to_path_buf(),
|
||||
};
|
||||
|
||||
if !scan_root.exists() {
|
||||
return Err(Error::Origin(format!(
|
||||
"scan path does not exist: {}",
|
||||
scan_root.display()
|
||||
)));
|
||||
}
|
||||
|
||||
// Phase 1: Scanning
|
||||
let audio_files = self.collect_audio_files(&scan_root, &progress_tx)?;
|
||||
let total_files = audio_files.len() as u32;
|
||||
info!(files = total_files, "scan phase complete");
|
||||
|
||||
// Phase 2: Hashing + categorization
|
||||
let mut new_files = Vec::new();
|
||||
let mut unchanged = 0u32;
|
||||
|
||||
for (i, abs_path) in audio_files.iter().enumerate() {
|
||||
let _ = progress_tx.try_send(ScanProgress {
|
||||
phase: "hashing".to_string(),
|
||||
current: i as u32 + 1,
|
||||
total: total_files,
|
||||
current_path: abs_path.display().to_string(),
|
||||
bytes_synced: 0,
|
||||
});
|
||||
|
||||
let rel_path = abs_path.strip_prefix(origin_root).unwrap_or(abs_path);
|
||||
|
||||
let existing = self.db.get_file_by_real_path(origin_id, rel_path)?;
|
||||
if existing.is_some() {
|
||||
unchanged += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let size = std::fs::metadata(abs_path).map(|m| m.len()).unwrap_or(0);
|
||||
|
||||
new_files.push(DiscoveredFile {
|
||||
abs_path: abs_path.clone(),
|
||||
rel_path: rel_path.to_path_buf(),
|
||||
size,
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
new = new_files.len(),
|
||||
unchanged = unchanged,
|
||||
"hash phase complete"
|
||||
);
|
||||
|
||||
// Phase 3: Indexing
|
||||
let mut synced = Vec::new();
|
||||
let mut bytes_synced = 0u64;
|
||||
let ingest_total = new_files.len() as u32;
|
||||
|
||||
for (i, file) in new_files.iter().enumerate() {
|
||||
let _ = progress_tx.try_send(ScanProgress {
|
||||
phase: "indexing".to_string(),
|
||||
current: i as u32 + 1,
|
||||
total: ingest_total,
|
||||
current_path: file.abs_path.display().to_string(),
|
||||
bytes_synced,
|
||||
});
|
||||
|
||||
let audio_meta = match self.parser.parse_file(&file.abs_path) {
|
||||
Ok(meta) => meta,
|
||||
Err(e) => {
|
||||
warn!(path = %file.abs_path.display(), error = %e, "parse failed, using defaults");
|
||||
AudioMeta::default()
|
||||
}
|
||||
};
|
||||
|
||||
let virtual_path = derive_virtual_path(&audio_meta, &file.rel_path);
|
||||
|
||||
let file_id = self.db.upsert_file(
|
||||
origin_id,
|
||||
&file.rel_path,
|
||||
&virtual_path,
|
||||
&audio_meta,
|
||||
UNIX_EPOCH,
|
||||
file.size,
|
||||
)?;
|
||||
|
||||
let file_meta = FileMeta {
|
||||
id: file_id,
|
||||
virtual_path: virtual_path.clone(),
|
||||
real_path: RealPath {
|
||||
origin_id: origin_id.clone(),
|
||||
path: file.rel_path.clone(),
|
||||
},
|
||||
size: file.size,
|
||||
mtime: UNIX_EPOCH,
|
||||
content_hash: None,
|
||||
audio: Some(audio_meta),
|
||||
};
|
||||
|
||||
{
|
||||
let mut tree = self.tree.write();
|
||||
tree.insert_file(&file_meta);
|
||||
}
|
||||
|
||||
self.fetcher.register_file(file_meta.clone());
|
||||
|
||||
self.event_bus.publish(Event::FileAdded {
|
||||
path: virtual_path.clone(),
|
||||
origin_id: origin_id.clone(),
|
||||
});
|
||||
|
||||
bytes_synced += file.size;
|
||||
|
||||
synced.push(SyncedFileInfo {
|
||||
path: file.abs_path.display().to_string(),
|
||||
file_id,
|
||||
virtual_path: virtual_path.as_str().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(ScanResult {
|
||||
new_files: synced,
|
||||
changed: 0,
|
||||
deleted: 0,
|
||||
unchanged,
|
||||
bytes_synced,
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_audio_files(
|
||||
&self,
|
||||
scan_root: &Path,
|
||||
progress_tx: &mpsc::Sender<ScanProgress>,
|
||||
) -> Result<Vec<PathBuf>> {
|
||||
let mut files = Vec::new();
|
||||
self.walk_dir(scan_root, &mut files, progress_tx)?;
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn walk_dir(
|
||||
&self,
|
||||
dir: &Path,
|
||||
files: &mut Vec<PathBuf>,
|
||||
progress_tx: &mpsc::Sender<ScanProgress>,
|
||||
) -> Result<()> {
|
||||
let entries = std::fs::read_dir(dir)
|
||||
.map_err(|e| Error::Origin(format!("read_dir {}: {}", dir.display(), e)))?;
|
||||
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
self.walk_dir(&path, files, progress_tx)?;
|
||||
} else if is_audio_file(&path) {
|
||||
files.push(path.clone());
|
||||
let _ = progress_tx.try_send(ScanProgress {
|
||||
phase: "scanning".to_string(),
|
||||
current: files.len() as u32,
|
||||
total: 0,
|
||||
current_path: path.display().to_string(),
|
||||
bytes_synced: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn derive_virtual_path(meta: &AudioMeta, rel_path: &Path) -> VirtualPath {
|
||||
let artist = meta.artist.as_deref().unwrap_or("Unknown Artist");
|
||||
let album = meta.album.as_deref().unwrap_or("Unknown Album");
|
||||
let filename = rel_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("unknown");
|
||||
|
||||
VirtualPath::new(format!("/{}/{}/{}", artist, album, filename))
|
||||
}
|
||||
|
||||
fn is_audio_file(path: &Path) -> bool {
|
||||
matches!(
|
||||
path.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.to_lowercase())
|
||||
.as_deref(),
|
||||
Some("flac" | "mp3" | "ogg" | "wav" | "m4a" | "aac" | "opus")
|
||||
)
|
||||
}
|
||||
|
||||
struct DiscoveredFile {
|
||||
abs_path: PathBuf,
|
||||
rel_path: PathBuf,
|
||||
size: u64,
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
use crate::proto::musicfs::v1::{
|
||||
music_fs_server::MusicFs, CacheStats, ClearCacheRequest, ClearCacheResponse, Empty, Event,
|
||||
EventFilter, OriginHealthResponse, OriginRequest, OriginsResponse, PrefetchProgress,
|
||||
PrefetchRequest, SearchRequest, SearchResponse, SearchResult, ShutdownRequest, StatusResponse,
|
||||
SyncProgress,
|
||||
};
|
||||
use musicfs_search::SearchIndex;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::debug;
|
||||
|
||||
pub struct SearchService {
|
||||
index: Arc<SearchIndex>,
|
||||
}
|
||||
|
||||
impl SearchService {
|
||||
pub fn new(index: Arc<SearchIndex>) -> Self {
|
||||
Self { index }
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl MusicFs for SearchService {
|
||||
async fn search(
|
||||
&self,
|
||||
request: Request<SearchRequest>,
|
||||
) -> Result<Response<SearchResponse>, Status> {
|
||||
let start = Instant::now();
|
||||
let req = request.into_inner();
|
||||
|
||||
if req.query.is_empty() {
|
||||
return Err(Status::invalid_argument("Query cannot be empty"));
|
||||
}
|
||||
|
||||
if req.query.len() > 256 {
|
||||
return Err(Status::invalid_argument(
|
||||
"Query exceeds maximum length (256)",
|
||||
));
|
||||
}
|
||||
|
||||
let limit = req.limit.unwrap_or(100).min(10000) as usize;
|
||||
let offset = req.offset.unwrap_or(0) as usize;
|
||||
|
||||
let results = self
|
||||
.index
|
||||
.search(&req.query, limit + offset)
|
||||
.map_err(|e| Status::internal(format!("Search failed: {}", e)))?;
|
||||
|
||||
let hits: Vec<SearchResult> = results
|
||||
.into_iter()
|
||||
.skip(offset)
|
||||
.take(limit)
|
||||
.map(|hit| SearchResult {
|
||||
file_id: hit.file_id.0,
|
||||
virtual_path: hit.virtual_path.as_str().to_string(),
|
||||
artist: hit.artist,
|
||||
album: hit.album,
|
||||
title: hit.title,
|
||||
score: hit.score,
|
||||
highlights: Default::default(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total_matches = self.index.count();
|
||||
let query_time_ms = start.elapsed().as_millis() as u32;
|
||||
|
||||
debug!(
|
||||
"Search '{}' returned {} results in {}ms",
|
||||
req.query,
|
||||
hits.len(),
|
||||
query_time_ms
|
||||
);
|
||||
|
||||
Ok(Response::new(SearchResponse {
|
||||
results: hits,
|
||||
total_matches,
|
||||
query_time_ms,
|
||||
}))
|
||||
}
|
||||
|
||||
type SearchStreamStream = ReceiverStream<Result<SearchResult, Status>>;
|
||||
|
||||
async fn search_stream(
|
||||
&self,
|
||||
request: Request<SearchRequest>,
|
||||
) -> Result<Response<Self::SearchStreamStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
|
||||
if req.query.is_empty() {
|
||||
return Err(Status::invalid_argument("Query cannot be empty"));
|
||||
}
|
||||
|
||||
let limit = req.limit.unwrap_or(1000).min(10000) as usize;
|
||||
|
||||
let results = self
|
||||
.index
|
||||
.search(&req.query, limit)
|
||||
.map_err(|e| Status::internal(format!("Search failed: {}", e)))?;
|
||||
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(100);
|
||||
|
||||
tokio::spawn(async move {
|
||||
for hit in results {
|
||||
let result = SearchResult {
|
||||
file_id: hit.file_id.0,
|
||||
virtual_path: hit.virtual_path.as_str().to_string(),
|
||||
artist: hit.artist,
|
||||
album: hit.album,
|
||||
title: hit.title,
|
||||
score: hit.score,
|
||||
highlights: Default::default(),
|
||||
};
|
||||
if tx.send(Ok(result)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
async fn get_status(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<StatusResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn shutdown(
|
||||
&self,
|
||||
_request: Request<ShutdownRequest>,
|
||||
) -> Result<Response<Empty>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_cache_stats(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<CacheStats>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn clear_cache(
|
||||
&self,
|
||||
_request: Request<ClearCacheRequest>,
|
||||
) -> Result<Response<ClearCacheResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
type PrefetchStream = ReceiverStream<Result<PrefetchProgress, Status>>;
|
||||
|
||||
async fn prefetch(
|
||||
&self,
|
||||
_request: Request<PrefetchRequest>,
|
||||
) -> Result<Response<Self::PrefetchStream>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_origins(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<OriginsResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_origin_health(
|
||||
&self,
|
||||
_request: Request<OriginRequest>,
|
||||
) -> Result<Response<OriginHealthResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
type RescanOriginStream = ReceiverStream<Result<SyncProgress, Status>>;
|
||||
|
||||
async fn rescan_origin(
|
||||
&self,
|
||||
_request: Request<OriginRequest>,
|
||||
) -> Result<Response<Self::RescanOriginStream>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
|
||||
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
|
||||
|
||||
async fn subscribe_events(
|
||||
&self,
|
||||
_request: Request<EventFilter>,
|
||||
) -> Result<Response<Self::SubscribeEventsStream>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use MusicFsServer for control operations",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_grpc_search_empty_query() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
|
||||
let service = SearchService::new(index);
|
||||
|
||||
let request = Request::new(SearchRequest {
|
||||
query: String::new(),
|
||||
limit: Some(10),
|
||||
offset: None,
|
||||
origin_id: None,
|
||||
});
|
||||
|
||||
let result = service.search(request).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_grpc_search_returns_response() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let index = Arc::new(SearchIndex::open(dir.path()).unwrap());
|
||||
let service = SearchService::new(index);
|
||||
|
||||
let request = Request::new(SearchRequest {
|
||||
query: "test".to_string(),
|
||||
limit: Some(10),
|
||||
offset: None,
|
||||
origin_id: None,
|
||||
});
|
||||
|
||||
let response = service.search(request).await.unwrap();
|
||||
assert!(response.get_ref().results.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -1,561 +0,0 @@
|
||||
use crate::proto::musicfs::v1::{
|
||||
music_fs_server::MusicFs, CacheStats, ClearCacheRequest, ClearCacheResponse, Empty, Event,
|
||||
EventFilter, HealthStatus, MountState, OriginHealthResponse, OriginRequest, OriginsResponse,
|
||||
PrefetchProgress, PrefetchRequest, SearchRequest, SearchResponse, SearchResult,
|
||||
ShutdownRequest, StatusResponse, SyncProgress, SyncedFile, TierStats,
|
||||
};
|
||||
use musicfs_core::{Event as CoreEvent, EventBus};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::{debug, info, instrument};
|
||||
|
||||
pub struct MusicFsServer {
|
||||
start_time: Instant,
|
||||
event_bus: Arc<EventBus>,
|
||||
version: String,
|
||||
scanner: Arc<crate::scanner::OriginScanner>,
|
||||
origin_root: std::path::PathBuf,
|
||||
}
|
||||
|
||||
impl MusicFsServer {
|
||||
pub fn new(
|
||||
event_bus: Arc<EventBus>,
|
||||
db: Arc<musicfs_cache::Database>,
|
||||
tree: Arc<parking_lot::RwLock<musicfs_cache::VirtualTree>>,
|
||||
fetcher: Arc<musicfs_cas::ContentFetcher>,
|
||||
origin_root: std::path::PathBuf,
|
||||
) -> Self {
|
||||
let scanner = Arc::new(crate::scanner::OriginScanner::new(
|
||||
db,
|
||||
event_bus.clone(),
|
||||
tree,
|
||||
fetcher,
|
||||
));
|
||||
Self {
|
||||
start_time: Instant::now(),
|
||||
event_bus,
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
scanner,
|
||||
origin_root,
|
||||
}
|
||||
}
|
||||
|
||||
fn event_to_proto(event: &CoreEvent) -> Event {
|
||||
let (event_type, origin_id, path, file_id) = match event {
|
||||
CoreEvent::FileAccessed {
|
||||
file_id,
|
||||
origin_id,
|
||||
path,
|
||||
..
|
||||
} => (
|
||||
"file_accessed".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
Some(path.as_str().to_string()),
|
||||
Some(file_id.0),
|
||||
),
|
||||
CoreEvent::FileAdded { path, origin_id } => (
|
||||
"file_added".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
Some(path.as_str().to_string()),
|
||||
None,
|
||||
),
|
||||
CoreEvent::FileRemoved { path, file_id } => (
|
||||
"file_removed".to_string(),
|
||||
None,
|
||||
Some(path.as_str().to_string()),
|
||||
file_id.map(|id| id.0),
|
||||
),
|
||||
CoreEvent::FileModified { path } => (
|
||||
"file_modified".to_string(),
|
||||
None,
|
||||
Some(path.as_str().to_string()),
|
||||
None,
|
||||
),
|
||||
CoreEvent::SyncStarted { origin_id } => (
|
||||
"sync_started".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
CoreEvent::SyncCompleted {
|
||||
origin_id,
|
||||
files_changed,
|
||||
} => {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("files_changed".to_string(), files_changed.to_string());
|
||||
return Event {
|
||||
event_type: "sync_completed".to_string(),
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id: Some(origin_id.to_string()),
|
||||
path: None,
|
||||
file_id: None,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
CoreEvent::OriginHealthChanged { origin_id, healthy } => {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("healthy".to_string(), healthy.to_string());
|
||||
return Event {
|
||||
event_type: "origin_health_changed".to_string(),
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id: Some(origin_id.to_string()),
|
||||
path: None,
|
||||
file_id: None,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
CoreEvent::CacheEviction { bytes_freed } => {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("bytes_freed".to_string(), bytes_freed.to_string());
|
||||
return Event {
|
||||
event_type: "cache_eviction".to_string(),
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id: None,
|
||||
path: None,
|
||||
file_id: None,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
CoreEvent::OriginConnected { origin_id } => (
|
||||
"origin_connected".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
CoreEvent::OriginDisconnected { origin_id } => (
|
||||
"origin_disconnected".to_string(),
|
||||
Some(origin_id.to_string()),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
CoreEvent::AllOriginsUnhealthy { candidate_count } => {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("candidate_count".to_string(), candidate_count.to_string());
|
||||
return Event {
|
||||
event_type: "all_origins_unhealthy".to_string(),
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id: None,
|
||||
path: None,
|
||||
file_id: None,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
Event {
|
||||
event_type,
|
||||
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||
origin_id,
|
||||
path,
|
||||
file_id,
|
||||
metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_filter(event: &CoreEvent, filter: &EventFilter) -> bool {
|
||||
if !filter.event_types.is_empty() {
|
||||
let event_type = match event {
|
||||
CoreEvent::FileAccessed { .. } => "file_accessed",
|
||||
CoreEvent::FileAdded { .. } => "file_added",
|
||||
CoreEvent::FileRemoved { .. } => "file_removed",
|
||||
CoreEvent::FileModified { .. } => "file_modified",
|
||||
CoreEvent::SyncStarted { .. } => "sync_started",
|
||||
CoreEvent::SyncCompleted { .. } => "sync_completed",
|
||||
CoreEvent::OriginHealthChanged { .. } => "origin_health_changed",
|
||||
CoreEvent::CacheEviction { .. } => "cache_eviction",
|
||||
CoreEvent::OriginConnected { .. } => "origin_connected",
|
||||
CoreEvent::OriginDisconnected { .. } => "origin_disconnected",
|
||||
CoreEvent::AllOriginsUnhealthy { .. } => "all_origins_unhealthy",
|
||||
};
|
||||
|
||||
if !filter.event_types.iter().any(|t| t == event_type) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref origin_filter) = filter.origin_id {
|
||||
let event_origin = match event {
|
||||
CoreEvent::FileAccessed { origin_id, .. }
|
||||
| CoreEvent::FileAdded { origin_id, .. }
|
||||
| CoreEvent::SyncStarted { origin_id }
|
||||
| CoreEvent::SyncCompleted { origin_id, .. }
|
||||
| CoreEvent::OriginHealthChanged { origin_id, .. }
|
||||
| CoreEvent::OriginConnected { origin_id }
|
||||
| CoreEvent::OriginDisconnected { origin_id } => Some(origin_id.to_string()),
|
||||
CoreEvent::FileRemoved { .. }
|
||||
| CoreEvent::FileModified { .. }
|
||||
| CoreEvent::CacheEviction { .. }
|
||||
| CoreEvent::AllOriginsUnhealthy { .. } => None,
|
||||
};
|
||||
|
||||
if event_origin.as_ref() != Some(origin_filter) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl MusicFs for MusicFsServer {
|
||||
async fn search(
|
||||
&self,
|
||||
_request: Request<SearchRequest>,
|
||||
) -> Result<Response<SearchResponse>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use SearchService for search operations",
|
||||
))
|
||||
}
|
||||
|
||||
type SearchStreamStream = ReceiverStream<Result<SearchResult, Status>>;
|
||||
|
||||
async fn search_stream(
|
||||
&self,
|
||||
_request: Request<SearchRequest>,
|
||||
) -> Result<Response<Self::SearchStreamStream>, Status> {
|
||||
Err(Status::unimplemented(
|
||||
"Use SearchService for search operations",
|
||||
))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _request), fields(method = "get_status"))]
|
||||
async fn get_status(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<StatusResponse>, Status> {
|
||||
debug!("gRPC get_status called");
|
||||
let uptime = self.start_time.elapsed().as_secs();
|
||||
|
||||
Ok(Response::new(StatusResponse {
|
||||
version: self.version.clone(),
|
||||
uptime_secs: uptime,
|
||||
mount_point: String::new(),
|
||||
state: MountState::MountReady as i32,
|
||||
open_file_handles: 0,
|
||||
fuse_ops_total: 0,
|
||||
files_indexed: 0,
|
||||
cache_size_bytes: 0,
|
||||
origins: vec![],
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "shutdown"))]
|
||||
async fn shutdown(&self, request: Request<ShutdownRequest>) -> Result<Response<Empty>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(
|
||||
graceful = req.graceful,
|
||||
timeout_secs = req.timeout_secs,
|
||||
"gRPC shutdown requested"
|
||||
);
|
||||
|
||||
Ok(Response::new(Empty {}))
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip(self, _request),
|
||||
fields(method = "get_cache_stats")
|
||||
)]
|
||||
async fn get_cache_stats(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<CacheStats>, Status> {
|
||||
debug!("gRPC get_cache_stats called");
|
||||
Ok(Response::new(CacheStats {
|
||||
total_size_bytes: 0,
|
||||
used_size_bytes: 0,
|
||||
size_limit_bytes: 0,
|
||||
chunk_count: 0,
|
||||
chunks_unique: 0,
|
||||
dedup_ratio: 0.0,
|
||||
hit_count: 0,
|
||||
miss_count: 0,
|
||||
hit_ratio: 0.0,
|
||||
metadata_entries: 0,
|
||||
metadata_bytes: 0,
|
||||
l1_metadata: Some(TierStats {
|
||||
entries: 0,
|
||||
size_bytes: 0,
|
||||
hits: 0,
|
||||
misses: 0,
|
||||
}),
|
||||
l2_headers: Some(TierStats {
|
||||
entries: 0,
|
||||
size_bytes: 0,
|
||||
hits: 0,
|
||||
misses: 0,
|
||||
}),
|
||||
l3_chunks: Some(TierStats {
|
||||
entries: 0,
|
||||
size_bytes: 0,
|
||||
hits: 0,
|
||||
misses: 0,
|
||||
}),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "clear_cache"))]
|
||||
async fn clear_cache(
|
||||
&self,
|
||||
request: Request<ClearCacheRequest>,
|
||||
) -> Result<Response<ClearCacheResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
info!(
|
||||
origin_id = ?req.origin_id,
|
||||
clear_metadata = req.clear_metadata,
|
||||
clear_chunks = req.clear_chunks,
|
||||
"gRPC clear_cache"
|
||||
);
|
||||
|
||||
Ok(Response::new(ClearCacheResponse {
|
||||
bytes_cleared: 0,
|
||||
chunks_cleared: 0,
|
||||
}))
|
||||
}
|
||||
|
||||
type PrefetchStream = ReceiverStream<Result<PrefetchProgress, Status>>;
|
||||
|
||||
#[instrument(level = "debug", skip(self, request), fields(method = "prefetch"))]
|
||||
async fn prefetch(
|
||||
&self,
|
||||
request: Request<PrefetchRequest>,
|
||||
) -> Result<Response<Self::PrefetchStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
let total = req.paths.len() as u32;
|
||||
debug!(file_count = total, "gRPC prefetch started");
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
|
||||
tokio::spawn(async move {
|
||||
for (i, path) in req.paths.into_iter().enumerate() {
|
||||
let progress = PrefetchProgress {
|
||||
current_path: path,
|
||||
completed: i as u32 + 1,
|
||||
total,
|
||||
bytes_fetched: 0,
|
||||
};
|
||||
if tx.send(Ok(progress)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _request), fields(method = "list_origins"))]
|
||||
async fn list_origins(
|
||||
&self,
|
||||
_request: Request<Empty>,
|
||||
) -> Result<Response<OriginsResponse>, Status> {
|
||||
debug!("gRPC list_origins called");
|
||||
Ok(Response::new(OriginsResponse { origins: vec![] }))
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip(self, request),
|
||||
fields(method = "get_origin_health")
|
||||
)]
|
||||
async fn get_origin_health(
|
||||
&self,
|
||||
request: Request<OriginRequest>,
|
||||
) -> Result<Response<OriginHealthResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
debug!(origin_id = %req.origin_id, "gRPC get_origin_health");
|
||||
|
||||
Ok(Response::new(OriginHealthResponse {
|
||||
origin_id: req.origin_id,
|
||||
status: HealthStatus::HealthUnknown as i32,
|
||||
message: None,
|
||||
last_check_secs: 0,
|
||||
}))
|
||||
}
|
||||
|
||||
type RescanOriginStream = ReceiverStream<Result<SyncProgress, Status>>;
|
||||
|
||||
#[instrument(level = "info", skip(self, request), fields(method = "rescan_origin"))]
|
||||
async fn rescan_origin(
|
||||
&self,
|
||||
request: Request<OriginRequest>,
|
||||
) -> Result<Response<Self::RescanOriginStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
let subdir = req.subdir.as_deref().filter(|s| !s.is_empty());
|
||||
info!(
|
||||
origin_id = %req.origin_id,
|
||||
subdir = ?subdir,
|
||||
"gRPC rescan_origin started"
|
||||
);
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
let (progress_tx, mut progress_rx) = mpsc::channel::<crate::scanner::ScanProgress>(64);
|
||||
|
||||
let origin_id = musicfs_core::OriginId::from(req.origin_id.as_str());
|
||||
let scanner = self.scanner.clone();
|
||||
let origin_root = self.origin_root.clone();
|
||||
let subdir_owned = subdir.map(|s| s.to_string());
|
||||
|
||||
tokio::spawn(async move {
|
||||
let forward_handle = {
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some(progress) = progress_rx.recv().await {
|
||||
let proto = SyncProgress {
|
||||
phase: progress.phase,
|
||||
current: progress.current,
|
||||
total: progress.total,
|
||||
current_path: progress.current_path,
|
||||
bytes_synced: progress.bytes_synced,
|
||||
new_files: vec![],
|
||||
};
|
||||
if tx.send(Ok(proto)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let result = scanner
|
||||
.scan(
|
||||
&origin_id,
|
||||
&origin_root,
|
||||
subdir_owned.as_deref(),
|
||||
progress_tx,
|
||||
)
|
||||
.await;
|
||||
|
||||
forward_handle.abort();
|
||||
|
||||
match result {
|
||||
Ok(scan_result) => {
|
||||
let synced_files: Vec<SyncedFile> = scan_result
|
||||
.new_files
|
||||
.iter()
|
||||
.map(|f| SyncedFile {
|
||||
path: f.path.clone(),
|
||||
file_id: f.file_id.0,
|
||||
virtual_path: f.virtual_path.clone(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let _ = tx
|
||||
.send(Ok(SyncProgress {
|
||||
phase: "complete".to_string(),
|
||||
current: scan_result.new_files.len() as u32
|
||||
+ scan_result.changed
|
||||
+ scan_result.deleted,
|
||||
total: scan_result.new_files.len() as u32
|
||||
+ scan_result.changed
|
||||
+ scan_result.deleted
|
||||
+ scan_result.unchanged,
|
||||
current_path: String::new(),
|
||||
bytes_synced: scan_result.bytes_synced,
|
||||
new_files: synced_files,
|
||||
}))
|
||||
.await;
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Err(Status::internal(format!("rescan failed: {}", e))))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
type SubscribeEventsStream = ReceiverStream<Result<Event, Status>>;
|
||||
|
||||
#[instrument(
|
||||
level = "info",
|
||||
skip(self, request),
|
||||
fields(method = "subscribe_events")
|
||||
)]
|
||||
async fn subscribe_events(
|
||||
&self,
|
||||
request: Request<EventFilter>,
|
||||
) -> Result<Response<Self::SubscribeEventsStream>, Status> {
|
||||
info!("gRPC subscribe_events: client connected");
|
||||
let filter = request.into_inner();
|
||||
let mut rx = self.event_bus.subscribe();
|
||||
let (tx, out_rx) = mpsc::channel(100);
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(event) => {
|
||||
if Self::matches_filter(&event, &filter) {
|
||||
let proto_event = Self::event_to_proto(&event);
|
||||
if tx.send(Ok(proto_event)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
tracing::warn!(skipped = n, "Event subscriber lagged, skipped events");
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
tracing::debug!("Event channel closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(out_rx)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
async fn make_test_server() -> (MusicFsServer, tempfile::TempDir) {
|
||||
let event_bus = Arc::new(EventBus::new(16));
|
||||
let db = Arc::new(musicfs_cache::Database::open_memory().unwrap());
|
||||
let tree = Arc::new(parking_lot::RwLock::new(
|
||||
musicfs_cache::TreeBuilder::new().build(),
|
||||
));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cfg = musicfs_cas::CasConfig {
|
||||
chunks_dir: dir.path().join("chunks"),
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(musicfs_cas::CasStore::open(cfg).await.unwrap());
|
||||
let fetcher = Arc::new(musicfs_cas::ContentFetcher::new(store));
|
||||
let origin_root = std::path::PathBuf::from("/tmp/test-origin");
|
||||
(
|
||||
MusicFsServer::new(event_bus, db, tree, fetcher, origin_root),
|
||||
dir,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_status() {
|
||||
let (server, _dir) = make_test_server().await;
|
||||
|
||||
let response = server.get_status(Request::new(Empty {})).await.unwrap();
|
||||
let status = response.into_inner();
|
||||
|
||||
assert!(!status.version.is_empty());
|
||||
assert!(status.uptime_secs < 5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_cache_stats() {
|
||||
let (server, _dir) = make_test_server().await;
|
||||
|
||||
let response = server
|
||||
.get_cache_stats(Request::new(Empty {}))
|
||||
.await
|
||||
.unwrap();
|
||||
let stats = response.into_inner();
|
||||
|
||||
assert_eq!(stats.hit_ratio, 0.0);
|
||||
}
|
||||
}
|
||||
@@ -1,327 +0,0 @@
|
||||
use musicfs_core::Event;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WebhookPayload {
|
||||
pub event_type: String,
|
||||
pub timestamp: i64,
|
||||
pub data: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
pub struct WebhookConfig {
|
||||
pub url: String,
|
||||
#[serde(skip_serializing)]
|
||||
pub secret: Option<String>,
|
||||
pub events: Vec<String>,
|
||||
#[serde(default = "default_retry_count")]
|
||||
pub retry_count: u32,
|
||||
#[serde(default = "default_timeout_ms")]
|
||||
pub timeout_ms: u64,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for WebhookConfig {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("WebhookConfig")
|
||||
.field("url", &self.url)
|
||||
.field("secret", &self.secret.as_ref().map(|_| "[REDACTED]"))
|
||||
.field("events", &self.events)
|
||||
.field("retry_count", &self.retry_count)
|
||||
.field("timeout_ms", &self.timeout_ms)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
fn default_retry_count() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
fn default_timeout_ms() -> u64 {
|
||||
5000
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum WebhookError {
|
||||
#[error("Failed to initialize HTTP client: {0}")]
|
||||
ClientInit(String),
|
||||
}
|
||||
|
||||
pub struct WebhookHandler {
|
||||
client: reqwest::Client,
|
||||
configs: Vec<WebhookConfig>,
|
||||
}
|
||||
|
||||
impl WebhookHandler {
|
||||
pub fn new(configs: Vec<WebhookConfig>) -> Result<Self, WebhookError> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
error!(error = %e, "Failed to create webhook HTTP client");
|
||||
WebhookError::ClientInit(e.to_string())
|
||||
})?;
|
||||
|
||||
Ok(Self { client, configs })
|
||||
}
|
||||
|
||||
pub async fn run(&self, mut rx: broadcast::Receiver<Event>) {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(event) => {
|
||||
for config in &self.configs {
|
||||
if self.matches_filter(&event, config) {
|
||||
self.dispatch(config, &event).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
warn!(skipped = n, "Webhook handler lagged, skipped events");
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => {
|
||||
debug!("Event channel closed, webhook handler stopping");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch(&self, config: &WebhookConfig, event: &Event) {
|
||||
let payload = WebhookPayload {
|
||||
event_type: self.event_type_name(event),
|
||||
timestamp: chrono::Utc::now().timestamp_millis(),
|
||||
data: self.event_to_json(event),
|
||||
};
|
||||
|
||||
let signature = self.sign(&payload, config);
|
||||
|
||||
let mut attempts = 0u32;
|
||||
loop {
|
||||
let result = self
|
||||
.client
|
||||
.post(&config.url)
|
||||
.timeout(Duration::from_millis(config.timeout_ms))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-MusicFS-Signature", &signature)
|
||||
.header("X-MusicFS-Event", &payload.event_type)
|
||||
.json(&payload)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
debug!(
|
||||
"Webhook delivered to {} for {}",
|
||||
config.url, payload.event_type
|
||||
);
|
||||
break;
|
||||
}
|
||||
Ok(resp) => {
|
||||
warn!(
|
||||
"Webhook to {} returned status {}, attempt {}/{}",
|
||||
config.url,
|
||||
resp.status(),
|
||||
attempts + 1,
|
||||
config.retry_count + 1
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"Webhook to {} failed: {}, attempt {}/{}",
|
||||
config.url,
|
||||
e,
|
||||
attempts + 1,
|
||||
config.retry_count + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if attempts >= config.retry_count {
|
||||
warn!(
|
||||
"Webhook delivery to {} failed after {} attempts",
|
||||
config.url,
|
||||
attempts + 1
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
attempts += 1;
|
||||
let delay = Duration::from_millis(100 * 2u64.pow(attempts));
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn sign(&self, payload: &WebhookPayload, config: &WebhookConfig) -> String {
|
||||
match &config.secret {
|
||||
Some(secret) => {
|
||||
use hmac::{Hmac, Mac};
|
||||
use sha2::Sha256;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
let body = serde_json::to_string(payload).unwrap_or_default();
|
||||
let mac = match HmacSha256::new_from_slice(secret.as_bytes()) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
error!(error = %e, "Invalid HMAC key for webhook signature");
|
||||
return String::new();
|
||||
}
|
||||
};
|
||||
let mut mac = mac;
|
||||
mac.update(body.as_bytes());
|
||||
let result = mac.finalize();
|
||||
|
||||
format!("sha256={}", hex::encode(result.into_bytes()))
|
||||
}
|
||||
None => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_filter(&self, event: &Event, config: &WebhookConfig) -> bool {
|
||||
if config.events.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let event_type = self.event_type_name(event);
|
||||
config.events.iter().any(|e| e == &event_type)
|
||||
}
|
||||
|
||||
fn event_type_name(&self, event: &Event) -> String {
|
||||
match event {
|
||||
Event::FileAccessed { .. } => "file_accessed",
|
||||
Event::FileAdded { .. } => "file_added",
|
||||
Event::FileRemoved { .. } => "file_removed",
|
||||
Event::FileModified { .. } => "file_modified",
|
||||
Event::SyncStarted { .. } => "sync_started",
|
||||
Event::SyncCompleted { .. } => "sync_completed",
|
||||
Event::OriginHealthChanged { .. } => "origin_health_changed",
|
||||
Event::CacheEviction { .. } => "cache_eviction",
|
||||
Event::OriginConnected { .. } => "origin_connected",
|
||||
Event::OriginDisconnected { .. } => "origin_disconnected",
|
||||
Event::AllOriginsUnhealthy { .. } => "all_origins_unhealthy",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn event_to_json(&self, event: &Event) -> serde_json::Value {
|
||||
match event {
|
||||
Event::FileAccessed {
|
||||
file_id,
|
||||
origin_id,
|
||||
path,
|
||||
offset,
|
||||
size,
|
||||
} => serde_json::json!({
|
||||
"file_id": file_id.0,
|
||||
"origin_id": origin_id.to_string(),
|
||||
"path": path.as_str(),
|
||||
"offset": offset,
|
||||
"size": size,
|
||||
}),
|
||||
Event::FileAdded { path, origin_id } => serde_json::json!({
|
||||
"path": path.as_str(),
|
||||
"origin_id": origin_id.to_string(),
|
||||
}),
|
||||
Event::FileRemoved { path, file_id } => serde_json::json!({
|
||||
"path": path.as_str(),
|
||||
"file_id": file_id.map(|id| id.0),
|
||||
}),
|
||||
Event::FileModified { path } => serde_json::json!({
|
||||
"path": path.as_str(),
|
||||
}),
|
||||
Event::SyncStarted { origin_id } => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
}),
|
||||
Event::SyncCompleted {
|
||||
origin_id,
|
||||
files_changed,
|
||||
} => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
"files_changed": files_changed,
|
||||
}),
|
||||
Event::OriginHealthChanged { origin_id, healthy } => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
"healthy": healthy,
|
||||
}),
|
||||
Event::CacheEviction { bytes_freed } => serde_json::json!({
|
||||
"bytes_freed": bytes_freed,
|
||||
}),
|
||||
Event::OriginConnected { origin_id } => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
}),
|
||||
Event::OriginDisconnected { origin_id } => serde_json::json!({
|
||||
"origin_id": origin_id.to_string(),
|
||||
}),
|
||||
Event::AllOriginsUnhealthy { candidate_count } => serde_json::json!({
|
||||
"candidate_count": candidate_count,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use musicfs_core::OriginId;
|
||||
|
||||
#[test]
|
||||
fn test_webhook_config_defaults() {
|
||||
let json = r#"{"url": "http://example.com", "events": []}"#;
|
||||
let config: WebhookConfig = serde_json::from_str(json).unwrap();
|
||||
|
||||
assert_eq!(config.retry_count, 3);
|
||||
assert_eq!(config.timeout_ms, 5000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_event_type_name() {
|
||||
let handler = WebhookHandler::new(vec![]).unwrap();
|
||||
|
||||
let event = Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
};
|
||||
assert_eq!(handler.event_type_name(&event), "sync_started");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_matches_filter_empty() {
|
||||
let handler = WebhookHandler::new(vec![]).unwrap();
|
||||
let config = WebhookConfig {
|
||||
url: "http://example.com".to_string(),
|
||||
secret: None,
|
||||
events: vec![],
|
||||
retry_count: 3,
|
||||
timeout_ms: 5000,
|
||||
};
|
||||
|
||||
let event = Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
};
|
||||
assert!(handler.matches_filter(&event, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_matches_filter_specific() {
|
||||
let handler = WebhookHandler::new(vec![]).unwrap();
|
||||
let config = WebhookConfig {
|
||||
url: "http://example.com".to_string(),
|
||||
secret: None,
|
||||
events: vec!["sync_started".to_string()],
|
||||
retry_count: 3,
|
||||
timeout_ms: 5000,
|
||||
};
|
||||
|
||||
let event = Event::SyncStarted {
|
||||
origin_id: OriginId::from("test"),
|
||||
};
|
||||
assert!(handler.matches_filter(&event, &config));
|
||||
|
||||
let event2 = Event::SyncCompleted {
|
||||
origin_id: OriginId::from("test"),
|
||||
files_changed: 0,
|
||||
};
|
||||
assert!(!handler.matches_filter(&event2, &config));
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
[package]
|
||||
name = "musicfs-metadata"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
musicfs-core = { path = "../musicfs-core" }
|
||||
symphonia.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
image.workspace = true
|
||||
@@ -1,116 +0,0 @@
|
||||
use image::ImageFormat;
|
||||
use std::io::Cursor;
|
||||
use symphonia::core::meta::Visual;
|
||||
use tracing::debug;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Artwork {
|
||||
pub art_type: ArtType,
|
||||
pub mime_type: String,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ArtType {
|
||||
Front,
|
||||
Back,
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum ArtSize {
|
||||
Thumbnail,
|
||||
Medium,
|
||||
Full,
|
||||
}
|
||||
|
||||
impl ArtSize {
|
||||
pub fn max_dimension(&self) -> Option<u32> {
|
||||
match self {
|
||||
ArtSize::Thumbnail => Some(150),
|
||||
ArtSize::Medium => Some(300),
|
||||
ArtSize::Full => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ArtworkExtractor;
|
||||
|
||||
impl ArtworkExtractor {
|
||||
pub fn extract_from_visual(visual: &Visual) -> Option<Artwork> {
|
||||
let data = visual.data.to_vec();
|
||||
|
||||
let img = image::load_from_memory(&data).ok()?;
|
||||
|
||||
let art_type = match visual.usage {
|
||||
Some(symphonia::core::meta::StandardVisualKey::FrontCover) => ArtType::Front,
|
||||
Some(symphonia::core::meta::StandardVisualKey::BackCover) => ArtType::Back,
|
||||
_ => ArtType::Other,
|
||||
};
|
||||
|
||||
let mime_type = if visual.media_type.is_empty() {
|
||||
"image/jpeg".to_string()
|
||||
} else {
|
||||
visual.media_type.clone()
|
||||
};
|
||||
|
||||
Some(Artwork {
|
||||
art_type,
|
||||
mime_type,
|
||||
width: img.width(),
|
||||
height: img.height(),
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn resize(artwork: &Artwork, size: ArtSize) -> Option<Artwork> {
|
||||
let max_dim = size.max_dimension()?;
|
||||
|
||||
if artwork.width <= max_dim && artwork.height <= max_dim {
|
||||
return Some(artwork.clone());
|
||||
}
|
||||
|
||||
let img = image::load_from_memory(&artwork.data).ok()?;
|
||||
let resized = img.thumbnail(max_dim, max_dim);
|
||||
|
||||
let mut output = Vec::new();
|
||||
let mut cursor = Cursor::new(&mut output);
|
||||
resized.write_to(&mut cursor, ImageFormat::Jpeg).ok()?;
|
||||
|
||||
debug!(
|
||||
"Resized artwork from {}x{} to {}x{}",
|
||||
artwork.width,
|
||||
artwork.height,
|
||||
resized.width(),
|
||||
resized.height()
|
||||
);
|
||||
|
||||
Some(Artwork {
|
||||
art_type: artwork.art_type,
|
||||
mime_type: "image/jpeg".to_string(),
|
||||
width: resized.width(),
|
||||
height: resized.height(),
|
||||
data: output,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_art_size_dimensions() {
|
||||
assert_eq!(ArtSize::Thumbnail.max_dimension(), Some(150));
|
||||
assert_eq!(ArtSize::Medium.max_dimension(), Some(300));
|
||||
assert_eq!(ArtSize::Full.max_dimension(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_art_type_equality() {
|
||||
assert_eq!(ArtType::Front, ArtType::Front);
|
||||
assert_ne!(ArtType::Front, ArtType::Back);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user