Move the files around
This commit is contained in:
@@ -0,0 +1,226 @@
|
||||
use crate::registry::OriginRegistry;
|
||||
use crate::traits::Origin;
|
||||
use musicfs_core::{Error, RealPath, Result};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tracing::{trace, warn};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RetryConfig {
|
||||
pub max_attempts: u32,
|
||||
pub delays: Vec<Duration>,
|
||||
}
|
||||
|
||||
impl Default for RetryConfig {
|
||||
fn default() -> Self {
|
||||
Self::spec_compliant()
|
||||
}
|
||||
}
|
||||
|
||||
impl RetryConfig {
|
||||
pub fn spec_compliant() -> Self {
|
||||
Self {
|
||||
max_attempts: 3,
|
||||
delays: vec![
|
||||
Duration::from_millis(100),
|
||||
Duration::from_millis(500),
|
||||
Duration::from_millis(2000),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_delays(delays: Vec<Duration>) -> Self {
|
||||
Self {
|
||||
max_attempts: delays.len() as u32,
|
||||
delays,
|
||||
}
|
||||
}
|
||||
|
||||
fn delay_for_attempt(&self, attempt: u32) -> Duration {
|
||||
self.delays
|
||||
.get(attempt as usize)
|
||||
.copied()
|
||||
.unwrap_or(*self.delays.last().unwrap_or(&Duration::from_millis(100)))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FailoverExecutor {
|
||||
registry: Arc<OriginRegistry>,
|
||||
retry_config: RetryConfig,
|
||||
}
|
||||
|
||||
impl FailoverExecutor {
|
||||
pub fn new(registry: Arc<OriginRegistry>, retry_config: RetryConfig) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
retry_config,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn read_with_failover(
|
||||
&self,
|
||||
path: &RealPath,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> Result<Vec<u8>> {
|
||||
let origins = self.registry.route_all(path);
|
||||
|
||||
if origins.is_empty() {
|
||||
if let Some(origin) = self.registry.route_with_fallback(path) {
|
||||
warn!("No healthy origins, using fallback origin {}", origin.id());
|
||||
return self
|
||||
.read_with_retry(&origin, &path.path, offset, size)
|
||||
.await;
|
||||
}
|
||||
return Err(Error::NoOriginAvailable);
|
||||
}
|
||||
|
||||
let mut last_error = None;
|
||||
|
||||
for origin in origins {
|
||||
trace!(origin_id = %origin.id(), "Attempting read from origin");
|
||||
let start = std::time::Instant::now();
|
||||
match self
|
||||
.read_with_retry(&origin, &path.path, offset, size)
|
||||
.await
|
||||
{
|
||||
Ok(data) => {
|
||||
let latency = start.elapsed().as_millis() as u64;
|
||||
self.registry.record_latency(origin.id(), latency);
|
||||
return Ok(data);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(origin_id = %origin.id(), error = %e, "Origin failed, trying next");
|
||||
last_error = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_error.unwrap_or(Error::NoOriginAvailable))
|
||||
}
|
||||
|
||||
async fn read_with_retry(
|
||||
&self,
|
||||
origin: &Arc<dyn Origin>,
|
||||
path: &std::path::Path,
|
||||
offset: u64,
|
||||
size: u32,
|
||||
) -> Result<Vec<u8>> {
|
||||
for attempt in 0..self.retry_config.max_attempts {
|
||||
match origin.read(path, offset, size).await {
|
||||
Ok(data) => return Ok(data),
|
||||
Err(e) if attempt + 1 < self.retry_config.max_attempts => {
|
||||
let delay = self.retry_config.delay_for_attempt(attempt);
|
||||
warn!(
|
||||
origin_id = %origin.id(),
|
||||
attempt = attempt + 1,
|
||||
max_attempts = self.retry_config.max_attempts,
|
||||
error = %e,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Retrying read operation"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::MaxRetriesExceeded)
|
||||
}
|
||||
|
||||
pub async fn read_full_with_failover(&self, path: &RealPath) -> Result<Vec<u8>> {
|
||||
let origins = self.registry.route_all(path);
|
||||
|
||||
if origins.is_empty() {
|
||||
if let Some(origin) = self.registry.route_with_fallback(path) {
|
||||
warn!(
|
||||
"No healthy origins for full read, using fallback {}",
|
||||
origin.id()
|
||||
);
|
||||
return self.read_full_with_retry(&origin, &path.path).await;
|
||||
}
|
||||
return Err(Error::NoOriginAvailable);
|
||||
}
|
||||
|
||||
let mut last_error = None;
|
||||
|
||||
for origin in origins {
|
||||
trace!(origin_id = %origin.id(), "Attempting full read from origin");
|
||||
let start = std::time::Instant::now();
|
||||
match self.read_full_with_retry(&origin, &path.path).await {
|
||||
Ok(data) => {
|
||||
let latency = start.elapsed().as_millis() as u64;
|
||||
self.registry.record_latency(origin.id(), latency);
|
||||
return Ok(data);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(origin_id = %origin.id(), error = %e, "Origin failed full read, trying next");
|
||||
last_error = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_error.unwrap_or(Error::NoOriginAvailable))
|
||||
}
|
||||
|
||||
async fn read_full_with_retry(
|
||||
&self,
|
||||
origin: &Arc<dyn Origin>,
|
||||
path: &std::path::Path,
|
||||
) -> Result<Vec<u8>> {
|
||||
for attempt in 0..self.retry_config.max_attempts {
|
||||
match origin.read_full(path).await {
|
||||
Ok(data) => return Ok(data),
|
||||
Err(e) if attempt + 1 < self.retry_config.max_attempts => {
|
||||
let delay = self.retry_config.delay_for_attempt(attempt);
|
||||
warn!(
|
||||
origin_id = %origin.id(),
|
||||
attempt = attempt + 1,
|
||||
max_attempts = self.retry_config.max_attempts,
|
||||
error = %e,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Retrying full read operation"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::MaxRetriesExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_retry_config_default() {
|
||||
let config = RetryConfig::default();
|
||||
assert_eq!(config.max_attempts, 3);
|
||||
assert_eq!(config.delays[0], Duration::from_millis(100));
|
||||
assert_eq!(config.delays[1], Duration::from_millis(500));
|
||||
assert_eq!(config.delays[2], Duration::from_millis(2000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delay_for_attempt() {
|
||||
let config = RetryConfig::spec_compliant();
|
||||
|
||||
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(100));
|
||||
assert_eq!(config.delay_for_attempt(1), Duration::from_millis(500));
|
||||
assert_eq!(config.delay_for_attempt(2), Duration::from_millis(2000));
|
||||
assert_eq!(config.delay_for_attempt(10), Duration::from_millis(2000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_delays() {
|
||||
let config =
|
||||
RetryConfig::with_delays(vec![Duration::from_millis(50), Duration::from_millis(100)]);
|
||||
|
||||
assert_eq!(config.max_attempts, 2);
|
||||
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(50));
|
||||
assert_eq!(config.delay_for_attempt(1), Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
use crate::traits::Origin;
|
||||
use dashmap::DashMap;
|
||||
use futures::future::join_all;
|
||||
use musicfs_core::{Event, EventBus, HealthStatus, OriginId, OriginType};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, info_span, warn, Instrument};
|
||||
|
||||
pub struct HealthMonitor {
|
||||
origins: DashMap<OriginId, Arc<dyn Origin>>,
|
||||
state: DashMap<OriginId, OriginHealthState>,
|
||||
check_interval: Duration,
|
||||
default_threshold: u32,
|
||||
per_type_thresholds: HashMap<OriginType, u32>,
|
||||
event_bus: Option<Arc<EventBus>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OriginHealthState {
|
||||
pub status: HealthStatus,
|
||||
pub last_check: Instant,
|
||||
pub consecutive_failures: u32,
|
||||
pub last_latency_ms: Option<u64>,
|
||||
}
|
||||
|
||||
impl Default for OriginHealthState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
status: HealthStatus::Unknown,
|
||||
last_check: Instant::now(),
|
||||
consecutive_failures: 0,
|
||||
last_latency_ms: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealthSnapshot {
|
||||
pub healthy: Vec<OriginId>,
|
||||
pub degraded: Vec<OriginId>,
|
||||
pub unhealthy: Vec<OriginId>,
|
||||
pub failure_counts: HashMap<OriginId, u32>,
|
||||
}
|
||||
|
||||
impl HealthSnapshot {
|
||||
pub fn is_healthy(&self, id: &OriginId) -> bool {
|
||||
self.healthy.contains(id)
|
||||
}
|
||||
|
||||
pub fn is_degraded(&self, id: &OriginId) -> bool {
|
||||
self.degraded.contains(id)
|
||||
}
|
||||
|
||||
pub fn is_unhealthy(&self, id: &OriginId) -> bool {
|
||||
self.unhealthy.contains(id)
|
||||
}
|
||||
|
||||
pub fn failure_count(&self, id: &OriginId) -> Option<u32> {
|
||||
self.failure_counts.get(id).copied()
|
||||
}
|
||||
|
||||
pub fn all_unhealthy(&self) -> bool {
|
||||
self.healthy.is_empty() && self.degraded.is_empty()
|
||||
}
|
||||
|
||||
pub fn total_candidates(&self) -> usize {
|
||||
self.healthy.len() + self.degraded.len() + self.unhealthy.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthMonitor {
|
||||
pub fn new(check_interval: Duration) -> Self {
|
||||
let mut per_type = HashMap::new();
|
||||
per_type.insert(OriginType::Local, 1);
|
||||
per_type.insert(OriginType::Nfs, 3);
|
||||
per_type.insert(OriginType::Smb, 3);
|
||||
per_type.insert(OriginType::S3, 3);
|
||||
per_type.insert(OriginType::Sftp, 3);
|
||||
|
||||
Self {
|
||||
origins: DashMap::new(),
|
||||
state: DashMap::new(),
|
||||
check_interval,
|
||||
default_threshold: 3,
|
||||
per_type_thresholds: per_type,
|
||||
event_bus: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_threshold(mut self, threshold: u32) -> Self {
|
||||
self.default_threshold = threshold;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_per_type_thresholds(mut self, thresholds: HashMap<OriginType, u32>) -> Self {
|
||||
self.per_type_thresholds = thresholds;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_event_bus(mut self, bus: Arc<EventBus>) -> Self {
|
||||
self.event_bus = Some(bus);
|
||||
self
|
||||
}
|
||||
|
||||
fn threshold_for(&self, origin_type: OriginType) -> u32 {
|
||||
self.per_type_thresholds
|
||||
.get(&origin_type)
|
||||
.copied()
|
||||
.unwrap_or(self.default_threshold)
|
||||
}
|
||||
|
||||
pub fn add_origin(&self, origin: Arc<dyn Origin>) {
|
||||
let id = origin.id().clone();
|
||||
self.origins.insert(id.clone(), origin);
|
||||
self.state.insert(id, OriginHealthState::default());
|
||||
}
|
||||
|
||||
pub fn remove_origin(&self, id: &OriginId) {
|
||||
self.origins.remove(id);
|
||||
self.state.remove(id);
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> HealthSnapshot {
|
||||
let mut healthy = Vec::new();
|
||||
let mut degraded = Vec::new();
|
||||
let mut unhealthy = Vec::new();
|
||||
let mut failure_counts = HashMap::new();
|
||||
|
||||
for entry in self.state.iter() {
|
||||
let id = entry.key().clone();
|
||||
failure_counts.insert(id.clone(), entry.value().consecutive_failures);
|
||||
|
||||
match entry.value().status {
|
||||
HealthStatus::Healthy => healthy.push(id),
|
||||
HealthStatus::Degraded => degraded.push(id),
|
||||
HealthStatus::Unhealthy => unhealthy.push(id),
|
||||
HealthStatus::Unknown => degraded.push(id),
|
||||
}
|
||||
}
|
||||
|
||||
HealthSnapshot {
|
||||
healthy,
|
||||
degraded,
|
||||
unhealthy,
|
||||
failure_counts,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(self: Arc<Self>) -> HealthCheckHandle {
|
||||
let (stop_tx, mut stop_rx) = mpsc::channel::<()>(1);
|
||||
let monitor = self.clone();
|
||||
let interval_secs = monitor.check_interval.as_secs();
|
||||
|
||||
info!(
|
||||
interval_secs = interval_secs,
|
||||
origin_count = monitor.origins.len(),
|
||||
"Health monitor starting"
|
||||
);
|
||||
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let mut interval = tokio::time::interval(monitor.check_interval);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
monitor.check_all().await;
|
||||
}
|
||||
_ = stop_rx.recv() => {
|
||||
info!("Health monitor stopping");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.instrument(info_span!("health_monitor")),
|
||||
);
|
||||
|
||||
HealthCheckHandle { stop_tx }
|
||||
}
|
||||
|
||||
pub async fn check_all(&self) {
|
||||
let origins: Vec<_> = self
|
||||
.origins
|
||||
.iter()
|
||||
.map(|e| (e.key().clone(), e.value().clone()))
|
||||
.collect();
|
||||
|
||||
let checks: Vec<_> = origins
|
||||
.iter()
|
||||
.map(|(id, origin)| self.check_one(id, origin))
|
||||
.collect();
|
||||
|
||||
join_all(checks).await;
|
||||
}
|
||||
|
||||
async fn check_one(&self, id: &OriginId, origin: &Arc<dyn Origin>) {
|
||||
let start = Instant::now();
|
||||
let health_timeout = Duration::from_millis(1500);
|
||||
|
||||
let status = match tokio::time::timeout(health_timeout, origin.health()).await {
|
||||
Ok(status) => status,
|
||||
Err(_) => {
|
||||
warn!(
|
||||
origin_id = %id,
|
||||
timeout_ms = health_timeout.as_millis() as u64,
|
||||
"Health check timed out"
|
||||
);
|
||||
HealthStatus::Unhealthy
|
||||
}
|
||||
};
|
||||
|
||||
let latency_ms = start.elapsed().as_millis() as u64;
|
||||
|
||||
let threshold = self.threshold_for(origin.origin_type());
|
||||
let prev_healthy = self
|
||||
.state
|
||||
.get(id)
|
||||
.map(|s| s.status == HealthStatus::Healthy)
|
||||
.unwrap_or(false);
|
||||
|
||||
let mut state = self.state.entry(id.clone()).or_default();
|
||||
|
||||
match status {
|
||||
HealthStatus::Healthy => {
|
||||
if state.status != HealthStatus::Healthy {
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to healthy"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Healthy;
|
||||
state.consecutive_failures = 0;
|
||||
}
|
||||
HealthStatus::Degraded => {
|
||||
if state.status != HealthStatus::Degraded {
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to degraded"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Degraded;
|
||||
}
|
||||
HealthStatus::Unhealthy => {
|
||||
state.consecutive_failures += 1;
|
||||
if state.consecutive_failures >= threshold {
|
||||
if state.status != HealthStatus::Unhealthy {
|
||||
info!(
|
||||
origin_id = %id,
|
||||
previous_status = ?state.status,
|
||||
consecutive_failures = state.consecutive_failures,
|
||||
threshold = threshold,
|
||||
duration_ms = latency_ms,
|
||||
"Origin health state transition to unhealthy"
|
||||
);
|
||||
}
|
||||
state.status = HealthStatus::Unhealthy;
|
||||
} else {
|
||||
debug!(
|
||||
origin_id = %id,
|
||||
consecutive_failures = state.consecutive_failures,
|
||||
threshold = threshold,
|
||||
"Origin health check failed"
|
||||
);
|
||||
state.status = HealthStatus::Degraded;
|
||||
}
|
||||
}
|
||||
HealthStatus::Unknown => {
|
||||
state.status = HealthStatus::Unknown;
|
||||
}
|
||||
}
|
||||
|
||||
state.last_check = Instant::now();
|
||||
state.last_latency_ms = Some(latency_ms);
|
||||
|
||||
let now_healthy = state.status == HealthStatus::Healthy;
|
||||
if prev_healthy != now_healthy {
|
||||
if let Some(bus) = &self.event_bus {
|
||||
bus.publish(Event::OriginHealthChanged {
|
||||
origin_id: id.clone(),
|
||||
healthy: now_healthy,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn check_now(&self, id: &OriginId) {
|
||||
if let Some(origin) = self.origins.get(id) {
|
||||
self.check_one(id, &origin.clone()).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_state(&self, id: &OriginId) -> Option<OriginHealthState> {
|
||||
self.state.get(id).map(|e| e.value().clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HealthCheckHandle {
|
||||
stop_tx: mpsc::Sender<()>,
|
||||
}
|
||||
|
||||
impl HealthCheckHandle {
|
||||
pub async fn stop(self) {
|
||||
let _ = self.stop_tx.send(()).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::LocalOrigin;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_health_monitor_basic() {
|
||||
let monitor = HealthMonitor::new(Duration::from_secs(30));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
|
||||
|
||||
monitor.add_origin(origin);
|
||||
|
||||
let snapshot = monitor.snapshot();
|
||||
assert!(!snapshot.is_healthy(&OriginId::from("test")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_health_check() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
|
||||
|
||||
monitor.add_origin(origin);
|
||||
monitor.check_now(&OriginId::from("test")).await;
|
||||
|
||||
let snapshot = monitor.snapshot();
|
||||
assert!(snapshot.is_healthy(&OriginId::from("test")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_failure_tracking() {
|
||||
let mut thresholds = HashMap::new();
|
||||
thresholds.insert(OriginType::Local, 3);
|
||||
|
||||
let monitor =
|
||||
HealthMonitor::new(Duration::from_secs(30)).with_per_type_thresholds(thresholds);
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
"missing",
|
||||
std::path::Path::new("/nonexistent"),
|
||||
));
|
||||
monitor.add_origin(origin);
|
||||
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
assert_eq!(state.status, HealthStatus::Degraded);
|
||||
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
|
||||
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 3);
|
||||
assert_eq!(state.status, HealthStatus::Unhealthy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_threshold_is_one() {
|
||||
let monitor = HealthMonitor::new(Duration::from_secs(30));
|
||||
|
||||
let origin = Arc::new(LocalOrigin::new(
|
||||
"missing",
|
||||
std::path::Path::new("/nonexistent"),
|
||||
));
|
||||
monitor.add_origin(origin);
|
||||
|
||||
monitor.check_now(&OriginId::from("missing")).await;
|
||||
let state = monitor.get_state(&OriginId::from("missing")).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
assert_eq!(state.status, HealthStatus::Unhealthy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_snapshot_all_unhealthy() {
|
||||
let snapshot = HealthSnapshot {
|
||||
healthy: Vec::new(),
|
||||
degraded: Vec::new(),
|
||||
unhealthy: vec![OriginId::from("a")],
|
||||
failure_counts: HashMap::new(),
|
||||
};
|
||||
assert!(snapshot.all_unhealthy());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
mod failover;
|
||||
mod health;
|
||||
mod local;
|
||||
mod nfs;
|
||||
mod registry;
|
||||
mod router;
|
||||
mod s3;
|
||||
mod sftp;
|
||||
mod smb;
|
||||
mod traits;
|
||||
|
||||
pub use failover::{FailoverExecutor, RetryConfig};
|
||||
pub use health::{HealthCheckHandle, HealthMonitor, HealthSnapshot, OriginHealthState};
|
||||
pub use local::LocalOrigin;
|
||||
pub use musicfs_core::OriginType;
|
||||
pub use nfs::NfsOrigin;
|
||||
pub use registry::OriginRegistry;
|
||||
pub use router::{LatencyStats, Router};
|
||||
pub use smb::SmbOrigin;
|
||||
pub use traits::{Origin, WatchCallback, WatchEvent, WatchHandle};
|
||||
@@ -0,0 +1,218 @@
|
||||
use crate::traits::{Origin, WatchCallback, WatchHandle};
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::fs;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::debug;
|
||||
|
||||
pub struct LocalOrigin {
|
||||
id: OriginId,
|
||||
root: PathBuf,
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
impl LocalOrigin {
|
||||
pub fn new(id: impl Into<OriginId>, root: impl Into<PathBuf>) -> Self {
|
||||
let root = root.into();
|
||||
let display_name = format!("Local: {}", root.display());
|
||||
Self {
|
||||
id: id.into(),
|
||||
root,
|
||||
display_name,
|
||||
}
|
||||
}
|
||||
|
||||
fn full_path(&self, path: &Path) -> PathBuf {
|
||||
if path.as_os_str().is_empty() || path == Path::new("/") {
|
||||
self.root.clone()
|
||||
} else {
|
||||
self.root.join(path.strip_prefix("/").unwrap_or(path))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for LocalOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
&self.id
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
OriginType::Local
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
&self.display_name
|
||||
}
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
|
||||
let full_path = self.full_path(path);
|
||||
debug!("LocalOrigin::readdir({:?})", full_path);
|
||||
|
||||
let mut entries = Vec::new();
|
||||
let mut dir = fs::read_dir(&full_path).await?;
|
||||
|
||||
while let Some(entry) = dir.next_entry().await? {
|
||||
let metadata = entry.metadata().await?;
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
|
||||
entries.push(DirEntry {
|
||||
name,
|
||||
is_dir: metadata.is_dir(),
|
||||
size: metadata.len(),
|
||||
mtime: metadata.modified().unwrap_or(std::time::UNIX_EPOCH),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat> {
|
||||
let full_path = self.full_path(path);
|
||||
debug!("LocalOrigin::stat({:?})", full_path);
|
||||
|
||||
let metadata = fs::metadata(&full_path).await?;
|
||||
|
||||
Ok(FileStat {
|
||||
size: metadata.len(),
|
||||
mtime: metadata.modified().unwrap_or(std::time::UNIX_EPOCH),
|
||||
is_dir: metadata.is_dir(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||
|
||||
let full_path = self.full_path(path);
|
||||
debug!(
|
||||
"LocalOrigin::read({:?}, offset={}, size={})",
|
||||
full_path, offset, size
|
||||
);
|
||||
|
||||
let mut file = fs::File::open(&full_path).await?;
|
||||
file.seek(std::io::SeekFrom::Start(offset)).await?;
|
||||
|
||||
// FIX: Loop until all requested bytes are read or EOF
|
||||
// Single read() only returns kernel buffer (~2MB), not full request
|
||||
let mut buffer = Vec::with_capacity(size as usize);
|
||||
let mut temp_buf = vec![0u8; 64 * 1024]; // 64KB chunks
|
||||
let mut total_read = 0usize;
|
||||
|
||||
while total_read < size as usize {
|
||||
let to_read = std::cmp::min(temp_buf.len(), size as usize - total_read);
|
||||
let n = file.read(&mut temp_buf[..to_read]).await?;
|
||||
if n == 0 {
|
||||
break; // EOF
|
||||
}
|
||||
buffer.extend_from_slice(&temp_buf[..n]);
|
||||
total_read += n;
|
||||
}
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
let full_path = self.full_path(path);
|
||||
debug!("LocalOrigin::read_full({:?})", full_path);
|
||||
Ok(fs::read(&full_path).await?)
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
let full_path = self.full_path(path);
|
||||
Ok(fs::try_exists(&full_path).await?)
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
match fs::try_exists(&self.root).await {
|
||||
Ok(true) => HealthStatus::Healthy,
|
||||
Ok(false) => HealthStatus::Unhealthy,
|
||||
Err(_) => HealthStatus::Unhealthy,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
|
||||
let full_path = self.full_path(path);
|
||||
let file = fs::File::open(&full_path).await?;
|
||||
Ok(Box::new(file))
|
||||
}
|
||||
|
||||
async fn watch(&self, path: &Path, _callback: WatchCallback) -> Result<WatchHandle> {
|
||||
debug!("LocalOrigin::watch({:?}) - stub implementation", path);
|
||||
let (tx, _rx) = tokio::sync::oneshot::channel();
|
||||
Ok(WatchHandle::new(tx))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_readdir() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello").unwrap();
|
||||
std::fs::create_dir(dir.path().join("subdir")).unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
let entries = origin.readdir(Path::new("/")).await.unwrap();
|
||||
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert!(entries.iter().any(|e| e.name == "test.txt" && !e.is_dir));
|
||||
assert!(entries.iter().any(|e| e.name == "subdir" && e.is_dir));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_stat() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
let stat = origin.stat(Path::new("/test.txt")).await.unwrap();
|
||||
|
||||
assert_eq!(stat.size, 11);
|
||||
assert!(!stat.is_dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_read() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
let data = origin.read(Path::new("/test.txt"), 0, 5).await.unwrap();
|
||||
|
||||
assert_eq!(data, b"hello");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_read_offset() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello world").unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
let data = origin.read(Path::new("/test.txt"), 6, 5).await.unwrap();
|
||||
|
||||
assert_eq!(data, b"world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_exists() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.txt"), "hello").unwrap();
|
||||
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
|
||||
assert!(origin.exists(Path::new("/test.txt")).await.unwrap());
|
||||
assert!(!origin.exists(Path::new("/nonexistent.txt")).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_origin_health() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = LocalOrigin::new("test", dir.path());
|
||||
|
||||
assert_eq!(origin.health().await, HealthStatus::Healthy);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
use crate::local::LocalOrigin;
|
||||
use crate::traits::{Origin, WatchCallback, WatchHandle};
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
pub struct NfsOrigin {
|
||||
inner: LocalOrigin,
|
||||
max_retries: u32,
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
impl NfsOrigin {
|
||||
pub fn new(id: impl Into<OriginId>, mount_point: impl Into<PathBuf>) -> Self {
|
||||
let mount_point = mount_point.into();
|
||||
let display_name = format!("NFS: {}", mount_point.display());
|
||||
|
||||
Self {
|
||||
inner: LocalOrigin::new(id, &mount_point),
|
||||
max_retries: 3,
|
||||
display_name,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_max_retries(mut self, retries: u32) -> Self {
|
||||
self.max_retries = retries;
|
||||
self
|
||||
}
|
||||
|
||||
async fn retry_on_stale<T, F, Fut>(&self, op: F) -> Result<T>
|
||||
where
|
||||
F: Fn() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
let mut delay = Duration::from_millis(100);
|
||||
|
||||
for attempt in 0..self.max_retries {
|
||||
match op().await {
|
||||
Ok(result) => return Ok(result),
|
||||
Err(e) => {
|
||||
if let Some(io_err) = e.downcast_io() {
|
||||
#[cfg(unix)]
|
||||
if io_err.raw_os_error() == Some(libc::ESTALE) {
|
||||
warn!(
|
||||
"NFS stale handle (attempt {}/{}), retrying after {:?}",
|
||||
attempt + 1,
|
||||
self.max_retries,
|
||||
delay
|
||||
);
|
||||
sleep(delay).await;
|
||||
delay *= 2;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(musicfs_core::Error::NfsStaleHandle)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for NfsOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
self.inner.id()
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
OriginType::Nfs
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
&self.display_name
|
||||
}
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
|
||||
self.retry_on_stale(|| self.inner.readdir(path)).await
|
||||
}
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat> {
|
||||
self.retry_on_stale(|| self.inner.stat(path)).await
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
self.retry_on_stale(|| self.inner.read(path, offset, size))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
self.retry_on_stale(|| self.inner.read_full(path)).await
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
self.retry_on_stale(|| self.inner.exists(path)).await
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
let health_timeout = Duration::from_secs(5);
|
||||
match tokio::time::timeout(health_timeout, self.inner.stat(Path::new("/"))).await {
|
||||
Ok(Ok(_)) => HealthStatus::Healthy,
|
||||
Ok(Err(_)) | Err(_) => HealthStatus::Unhealthy,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn tokio::io::AsyncRead + Send + Unpin>> {
|
||||
self.inner.open_read(path).await
|
||||
}
|
||||
|
||||
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
|
||||
debug!("NFS watch - inotify may be unreliable over NFS, consider polling");
|
||||
self.inner.watch(path, callback).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nfs_origin_basic() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.flac"), b"audio").unwrap();
|
||||
|
||||
let origin = NfsOrigin::new("nfs-test", dir.path());
|
||||
|
||||
let entries = origin.readdir(Path::new("/")).await.unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
|
||||
let data = origin.read(Path::new("/test.flac"), 0, 5).await.unwrap();
|
||||
assert_eq!(&data, b"audio");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nfs_origin_health() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = NfsOrigin::new("nfs-test", dir.path());
|
||||
|
||||
assert_eq!(origin.health().await, HealthStatus::Healthy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nfs_origin_type() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = NfsOrigin::new("nfs-test", dir.path());
|
||||
|
||||
assert_eq!(origin.origin_type(), OriginType::Nfs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retry_uses_fn_not_fnmut() {
|
||||
fn assert_fn<F: Fn() -> Fut, Fut>(_: F) {}
|
||||
|
||||
let closure = || async { Ok::<_, musicfs_core::Error>(()) };
|
||||
assert_fn(closure);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
use crate::health::{HealthMonitor, HealthSnapshot};
|
||||
use crate::router::Router;
|
||||
use crate::traits::{Origin, WatchHandle};
|
||||
use musicfs_core::{OriginId, RealPath};
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub struct OriginRegistry {
|
||||
origins: RwLock<HashMap<OriginId, Arc<dyn Origin>>>,
|
||||
router: Router,
|
||||
health_monitor: Arc<HealthMonitor>,
|
||||
watch_handles: RwLock<HashMap<OriginId, Vec<WatchHandle>>>,
|
||||
}
|
||||
|
||||
impl OriginRegistry {
|
||||
pub fn new(health_monitor: Arc<HealthMonitor>) -> Self {
|
||||
Self {
|
||||
origins: RwLock::new(HashMap::new()),
|
||||
router: Router::new(),
|
||||
health_monitor,
|
||||
watch_handles: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register(&self, origin: Arc<dyn Origin>, priority: u8) {
|
||||
let id = origin.id().clone();
|
||||
info!("Registering origin {} with priority {}", id, priority);
|
||||
|
||||
self.router.set_priority(id.clone(), priority);
|
||||
self.health_monitor.add_origin(origin.clone());
|
||||
self.origins.write().insert(id, origin);
|
||||
}
|
||||
|
||||
pub fn unregister(&self, id: &OriginId) {
|
||||
info!("Unregistering origin {}", id);
|
||||
|
||||
if let Some(handles) = self.watch_handles.write().remove(id) {
|
||||
info!("Dropping {} watch handles for origin {}", handles.len(), id);
|
||||
}
|
||||
|
||||
self.origins.write().remove(id);
|
||||
self.router.remove_priority(id);
|
||||
self.health_monitor.remove_origin(id);
|
||||
}
|
||||
|
||||
pub fn register_watch(&self, origin_id: &OriginId, handle: WatchHandle) {
|
||||
self.watch_handles
|
||||
.write()
|
||||
.entry(origin_id.clone())
|
||||
.or_default()
|
||||
.push(handle);
|
||||
}
|
||||
|
||||
pub fn get(&self, id: &OriginId) -> Option<Arc<dyn Origin>> {
|
||||
self.origins.read().get(id).cloned()
|
||||
}
|
||||
|
||||
pub fn list(&self) -> Vec<Arc<dyn Origin>> {
|
||||
self.origins.read().values().cloned().collect()
|
||||
}
|
||||
|
||||
pub fn route(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let candidates: Vec<_> = origins
|
||||
.iter()
|
||||
.filter(|(id, _)| self.can_serve(id, path))
|
||||
.map(|(id, origin)| (id.clone(), origin.clone()))
|
||||
.collect();
|
||||
|
||||
if candidates.is_empty() {
|
||||
warn!("No origin can serve path: {:?}", path);
|
||||
return None;
|
||||
}
|
||||
|
||||
let candidate_ids: Vec<_> = candidates.iter().map(|(id, _)| id.clone()).collect();
|
||||
let selected = self.router.select(&candidate_ids, &health)?;
|
||||
|
||||
candidates
|
||||
.into_iter()
|
||||
.find(|(id, _)| id == &selected)
|
||||
.map(|(_, origin)| origin)
|
||||
}
|
||||
|
||||
pub fn route_with_fallback(&self, path: &RealPath) -> Option<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let candidates: Vec<_> = origins
|
||||
.iter()
|
||||
.filter(|(id, _)| self.can_serve(id, path))
|
||||
.map(|(id, origin)| (id.clone(), origin.clone()))
|
||||
.collect();
|
||||
|
||||
if candidates.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let candidate_ids: Vec<_> = candidates.iter().map(|(id, _)| id.clone()).collect();
|
||||
let selected = self.router.select_with_fallback(&candidate_ids, &health)?;
|
||||
|
||||
candidates
|
||||
.into_iter()
|
||||
.find(|(id, _)| id == &selected)
|
||||
.map(|(_, origin)| origin)
|
||||
}
|
||||
|
||||
pub fn route_all(&self, path: &RealPath) -> Vec<Arc<dyn Origin>> {
|
||||
let origins = self.origins.read();
|
||||
let health = self.health_monitor.snapshot();
|
||||
|
||||
let mut result: Vec<_> = origins
|
||||
.iter()
|
||||
.filter(|(id, _)| self.can_serve(id, path) && health.is_healthy(id))
|
||||
.map(|(_, origin)| origin.clone())
|
||||
.collect();
|
||||
|
||||
result.sort_by_key(|o| self.router.get_priority(o.id()));
|
||||
result
|
||||
}
|
||||
|
||||
fn can_serve(&self, origin_id: &OriginId, path: &RealPath) -> bool {
|
||||
path.origin_id == *origin_id
|
||||
}
|
||||
|
||||
pub fn health(&self) -> HealthSnapshot {
|
||||
self.health_monitor.snapshot()
|
||||
}
|
||||
|
||||
pub fn record_latency(&self, id: &OriginId, latency_ms: u64) {
|
||||
self.router.record_latency(id, latency_ms);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::LocalOrigin;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_register_and_get() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
let registry = OriginRegistry::new(monitor);
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
|
||||
|
||||
registry.register(origin.clone(), 1);
|
||||
|
||||
let retrieved = registry.get(&OriginId::from("test"));
|
||||
assert!(retrieved.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unregister() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
let registry = OriginRegistry::new(monitor);
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = Arc::new(LocalOrigin::new("test", dir.path()));
|
||||
|
||||
registry.register(origin, 1);
|
||||
registry.unregister(&OriginId::from("test"));
|
||||
|
||||
assert!(registry.get(&OriginId::from("test")).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_route_by_priority() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
let registry = OriginRegistry::new(monitor.clone());
|
||||
|
||||
let dir1 = TempDir::new().unwrap();
|
||||
let dir2 = TempDir::new().unwrap();
|
||||
|
||||
let origin1 = Arc::new(LocalOrigin::new("primary", dir1.path()));
|
||||
let origin2 = Arc::new(LocalOrigin::new("backup", dir2.path()));
|
||||
|
||||
registry.register(origin1, 1);
|
||||
registry.register(origin2, 2);
|
||||
|
||||
monitor.check_now(&OriginId::from("primary")).await;
|
||||
monitor.check_now(&OriginId::from("backup")).await;
|
||||
|
||||
let path = RealPath {
|
||||
origin_id: OriginId::from("primary"),
|
||||
path: PathBuf::from("/test.flac"),
|
||||
};
|
||||
|
||||
let routed = registry.route(&path);
|
||||
assert!(routed.is_some());
|
||||
assert_eq!(routed.unwrap().id(), &OriginId::from("primary"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_origins() {
|
||||
let monitor = Arc::new(HealthMonitor::new(Duration::from_secs(30)));
|
||||
let registry = OriginRegistry::new(monitor);
|
||||
|
||||
let dir1 = TempDir::new().unwrap();
|
||||
let dir2 = TempDir::new().unwrap();
|
||||
|
||||
registry.register(Arc::new(LocalOrigin::new("a", dir1.path())), 1);
|
||||
registry.register(Arc::new(LocalOrigin::new("b", dir2.path())), 2);
|
||||
|
||||
let list = registry.list();
|
||||
assert_eq!(list.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
use crate::health::HealthSnapshot;
|
||||
use dashmap::DashMap;
|
||||
use musicfs_core::{Event, EventBus, OriginId};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
pub struct Router {
|
||||
priorities: DashMap<OriginId, u8>,
|
||||
latency_stats: DashMap<OriginId, LatencyStats>,
|
||||
event_bus: Option<Arc<EventBus>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct LatencyStats {
|
||||
pub samples: Vec<u64>,
|
||||
pub p50_ms: u64,
|
||||
pub p99_ms: u64,
|
||||
pub last_update: Option<Instant>,
|
||||
}
|
||||
|
||||
impl LatencyStats {
|
||||
pub fn record(&mut self, latency_ms: u64) {
|
||||
self.samples.push(latency_ms);
|
||||
|
||||
if self.samples.len() > 100 {
|
||||
self.samples.remove(0);
|
||||
}
|
||||
|
||||
if !self.samples.is_empty() {
|
||||
let mut sorted = self.samples.clone();
|
||||
sorted.sort_unstable();
|
||||
|
||||
let p50_idx = sorted.len() / 2;
|
||||
let p99_idx = (sorted.len() * 99) / 100;
|
||||
|
||||
self.p50_ms = sorted[p50_idx];
|
||||
self.p99_ms = sorted.get(p99_idx).copied().unwrap_or(self.p50_ms);
|
||||
}
|
||||
|
||||
self.last_update = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
impl Router {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
priorities: DashMap::new(),
|
||||
latency_stats: DashMap::new(),
|
||||
event_bus: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_event_bus(mut self, bus: Arc<EventBus>) -> Self {
|
||||
self.event_bus = Some(bus);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn set_priority(&self, id: OriginId, priority: u8) {
|
||||
self.priorities.insert(id, priority);
|
||||
}
|
||||
|
||||
pub fn remove_priority(&self, id: &OriginId) {
|
||||
self.priorities.remove(id);
|
||||
self.latency_stats.remove(id);
|
||||
}
|
||||
|
||||
pub fn get_priority(&self, id: &OriginId) -> u8 {
|
||||
self.priorities.get(id).map(|p| *p).unwrap_or(100)
|
||||
}
|
||||
|
||||
pub fn record_latency(&self, id: &OriginId, latency_ms: u64) {
|
||||
self.latency_stats
|
||||
.entry(id.clone())
|
||||
.or_default()
|
||||
.record(latency_ms);
|
||||
}
|
||||
|
||||
pub fn select(&self, candidates: &[OriginId], health: &HealthSnapshot) -> Option<OriginId> {
|
||||
let selected = candidates
|
||||
.iter()
|
||||
.filter(|id| health.is_healthy(id))
|
||||
.min_by_key(|id| {
|
||||
let priority = self.get_priority(id);
|
||||
let latency = self.latency_stats.get(*id).map(|s| s.p50_ms).unwrap_or(0);
|
||||
(priority, latency)
|
||||
})
|
||||
.cloned();
|
||||
|
||||
if let Some(ref id) = selected {
|
||||
let priority = self.get_priority(id);
|
||||
let latency = self.latency_stats.get(id).map(|s| s.p50_ms).unwrap_or(0);
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
priority = priority,
|
||||
latency_ms = latency,
|
||||
"Selected healthy origin"
|
||||
);
|
||||
}
|
||||
|
||||
selected
|
||||
}
|
||||
|
||||
pub fn select_with_fallback(
|
||||
&self,
|
||||
candidates: &[OriginId],
|
||||
health: &HealthSnapshot,
|
||||
) -> Option<OriginId> {
|
||||
if let Some(id) = self.select(candidates, health) {
|
||||
return Some(id);
|
||||
}
|
||||
|
||||
debug!("No healthy origins, trying degraded");
|
||||
if let Some(id) = candidates
|
||||
.iter()
|
||||
.filter(|id| health.is_degraded(id))
|
||||
.min_by_key(|id| self.get_priority(id))
|
||||
.cloned()
|
||||
{
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
priority = self.get_priority(&id),
|
||||
"Selected degraded origin as fallback"
|
||||
);
|
||||
return Some(id);
|
||||
}
|
||||
|
||||
warn!("All origins unhealthy, selecting least-bad by failure count");
|
||||
|
||||
if let Some(bus) = &self.event_bus {
|
||||
bus.publish(Event::AllOriginsUnhealthy {
|
||||
candidate_count: candidates.len(),
|
||||
});
|
||||
}
|
||||
|
||||
let selected = candidates
|
||||
.iter()
|
||||
.min_by_key(|id| {
|
||||
let failures = health.failure_count(id).unwrap_or(u32::MAX);
|
||||
let priority = self.get_priority(id);
|
||||
(failures, priority)
|
||||
})
|
||||
.cloned();
|
||||
|
||||
if let Some(ref id) = selected {
|
||||
let failures = health.failure_count(id).unwrap_or(u32::MAX);
|
||||
trace!(
|
||||
origin_id = %id,
|
||||
failure_count = failures,
|
||||
priority = self.get_priority(id),
|
||||
"Selected least-bad unhealthy origin"
|
||||
);
|
||||
}
|
||||
|
||||
selected
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Router {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn mock_health(healthy: &[&str], degraded: &[&str]) -> HealthSnapshot {
|
||||
HealthSnapshot {
|
||||
healthy: healthy.iter().map(|s| OriginId::from(*s)).collect(),
|
||||
degraded: degraded.iter().map(|s| OriginId::from(*s)).collect(),
|
||||
unhealthy: Vec::new(),
|
||||
failure_counts: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_select_by_priority() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("high"), 1);
|
||||
router.set_priority(OriginId::from("low"), 2);
|
||||
|
||||
let candidates = vec![OriginId::from("low"), OriginId::from("high")];
|
||||
let health = mock_health(&["high", "low"], &[]);
|
||||
|
||||
let selected = router.select(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("high")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_select_skips_unhealthy() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("high"), 1);
|
||||
router.set_priority(OriginId::from("low"), 2);
|
||||
|
||||
let candidates = vec![OriginId::from("high"), OriginId::from("low")];
|
||||
let health = mock_health(&["low"], &[]);
|
||||
|
||||
let selected = router.select(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("low")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_latency_affects_tiebreak() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("a"), 1);
|
||||
router.set_priority(OriginId::from("b"), 1);
|
||||
|
||||
router.record_latency(&OriginId::from("a"), 100);
|
||||
router.record_latency(&OriginId::from("b"), 10);
|
||||
|
||||
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
|
||||
let health = mock_health(&["a", "b"], &[]);
|
||||
|
||||
let selected = router.select(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("b")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fallback_to_degraded() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("a"), 1);
|
||||
router.set_priority(OriginId::from("b"), 2);
|
||||
|
||||
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
|
||||
let health = mock_health(&[], &["b"]);
|
||||
|
||||
let selected = router.select_with_fallback(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("b")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fallback_least_bad() {
|
||||
let router = Router::new();
|
||||
router.set_priority(OriginId::from("a"), 1);
|
||||
router.set_priority(OriginId::from("b"), 2);
|
||||
|
||||
let candidates = vec![OriginId::from("a"), OriginId::from("b")];
|
||||
let mut failure_counts = HashMap::new();
|
||||
failure_counts.insert(OriginId::from("a"), 5);
|
||||
failure_counts.insert(OriginId::from("b"), 2);
|
||||
|
||||
let health = HealthSnapshot {
|
||||
healthy: Vec::new(),
|
||||
degraded: Vec::new(),
|
||||
unhealthy: vec![OriginId::from("a"), OriginId::from("b")],
|
||||
failure_counts,
|
||||
};
|
||||
|
||||
let selected = router.select_with_fallback(&candidates, &health);
|
||||
assert_eq!(selected, Some(OriginId::from("b")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
//! S3-compatible object storage origin
|
||||
//!
|
||||
//! This module is feature-gated behind the `s3` feature to avoid heavy AWS SDK dependencies.
|
||||
//!
|
||||
//! # Oracle Security Fixes (MUST IMPLEMENT)
|
||||
//!
|
||||
//! 1. **Range EOF** - Clamp range to `min(requested_end, file_size)` to avoid 416 errors
|
||||
//! 2. **Health check** - Use `head_bucket` not `list_objects_v2` (lighter operation)
|
||||
//! 3. **Timeout handling** - Wrap all remote calls with `tokio::time::timeout(30s)`
|
||||
//!
|
||||
//! # Example Implementation (when feature enabled)
|
||||
//!
|
||||
//! ```ignore
|
||||
//! async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
//! // Oracle fix: Clamp range to file size to avoid 416 error
|
||||
//! let file_size = self.stat(path).await?.size;
|
||||
//! let end = std::cmp::min(offset + size as u64, file_size).saturating_sub(1);
|
||||
//!
|
||||
//! if offset >= file_size {
|
||||
//! return Ok(Vec::new()); // EOF
|
||||
//! }
|
||||
//!
|
||||
//! let range = format!("bytes={}-{}", offset, end);
|
||||
//!
|
||||
//! // Oracle fix: Add timeout to prevent hung connections
|
||||
//! let resp = tokio::time::timeout(
|
||||
//! Duration::from_secs(30),
|
||||
//! self.client.get_object().bucket(&self.bucket).key(&key).range(range).send()
|
||||
//! )
|
||||
//! .await
|
||||
//! .map_err(|_| Error::Timeout("S3 read timed out".into()))?
|
||||
//! .map_err(|e| Error::S3(e.to_string()))?;
|
||||
//!
|
||||
//! // ...
|
||||
//! }
|
||||
//!
|
||||
//! async fn health(&self) -> HealthStatus {
|
||||
//! // Oracle fix: Use head_bucket instead of list_objects_v2 (lighter)
|
||||
//! match self.client.head_bucket().bucket(&self.bucket).send().await {
|
||||
//! Ok(_) => HealthStatus::Healthy,
|
||||
//! Err(_) => HealthStatus::Unhealthy,
|
||||
//! }
|
||||
//! }
|
||||
//! ```
|
||||
|
||||
#[cfg(feature = "s3")]
|
||||
mod implementation {
|
||||
// Full S3 implementation would go here when aws-sdk-s3 is enabled
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#![allow(dead_code)]
|
||||
//! SFTP origin - feature-gated to avoid russh/deadpool dependencies
|
||||
|
||||
#[cfg(feature = "sftp")]
|
||||
mod implementation {
|
||||
// Full SFTP implementation with connection pooling
|
||||
// Oracle fixes to implement:
|
||||
// 1. Use deadpool connection pool, not Arc<Mutex<SftpSession>>
|
||||
// 2. Verify SSH host keys against ~/.ssh/known_hosts
|
||||
// 3. Wrap all operations with tokio::time::timeout(30s)
|
||||
// 4. Cap open_read to actual file size, not u32::MAX
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
use crate::local::LocalOrigin;
|
||||
use crate::traits::{Origin, WatchCallback, WatchHandle};
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
pub struct SmbOrigin {
|
||||
inner: LocalOrigin,
|
||||
share_path: String,
|
||||
}
|
||||
|
||||
impl SmbOrigin {
|
||||
pub fn from_mount(
|
||||
id: impl Into<OriginId>,
|
||||
mount_point: impl Into<PathBuf>,
|
||||
share_path: impl Into<String>,
|
||||
) -> Self {
|
||||
let mount_point = mount_point.into();
|
||||
let share_path = share_path.into();
|
||||
|
||||
Self {
|
||||
inner: LocalOrigin::new(id, &mount_point),
|
||||
share_path,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn is_mounted(&self) -> bool {
|
||||
self.inner.exists(Path::new("/")).await.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn retry_on_disconnect<T, F, Fut>(&self, op: F) -> Result<T>
|
||||
where
|
||||
F: Fn() -> Fut,
|
||||
Fut: Future<Output = Result<T>>,
|
||||
{
|
||||
const MAX_RETRIES: u32 = 3;
|
||||
|
||||
for attempt in 0..MAX_RETRIES {
|
||||
match op().await {
|
||||
Ok(val) => return Ok(val),
|
||||
Err(e) => {
|
||||
if Self::is_enotconn(&e) && attempt < MAX_RETRIES - 1 {
|
||||
debug!(attempt, "SMB ENOTCONN, retrying");
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
continue;
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn is_enotconn(err: &musicfs_core::Error) -> bool {
|
||||
if let musicfs_core::Error::Io(io_err) = err {
|
||||
io_err.raw_os_error() == Some(libc::ENOTCONN)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn is_enotconn(_err: &musicfs_core::Error) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Origin for SmbOrigin {
|
||||
fn id(&self) -> &OriginId {
|
||||
self.inner.id()
|
||||
}
|
||||
|
||||
fn origin_type(&self) -> OriginType {
|
||||
OriginType::Smb
|
||||
}
|
||||
|
||||
fn display_name(&self) -> &str {
|
||||
&self.share_path
|
||||
}
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>> {
|
||||
self.retry_on_disconnect(|| self.inner.readdir(path)).await
|
||||
}
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat> {
|
||||
self.retry_on_disconnect(|| self.inner.stat(path)).await
|
||||
}
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>> {
|
||||
self.retry_on_disconnect(|| self.inner.read(path, offset, size))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>> {
|
||||
self.retry_on_disconnect(|| self.inner.read_full(path))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool> {
|
||||
self.retry_on_disconnect(|| self.inner.exists(path)).await
|
||||
}
|
||||
|
||||
async fn health(&self) -> HealthStatus {
|
||||
let health_timeout = std::time::Duration::from_secs(5);
|
||||
match tokio::time::timeout(health_timeout, self.is_mounted()).await {
|
||||
Ok(true) => HealthStatus::Healthy,
|
||||
Ok(false) | Err(_) => HealthStatus::Unhealthy,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn tokio::io::AsyncRead + Send + Unpin>> {
|
||||
self.inner.open_read(path).await
|
||||
}
|
||||
|
||||
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle> {
|
||||
warn!("SMB watch using inotify - may be unreliable. Consider polling for remote mounts.");
|
||||
self.inner.watch(path, callback).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_smb_origin_basic() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("test.flac"), b"audio").unwrap();
|
||||
|
||||
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/share");
|
||||
|
||||
let entries = origin.readdir(Path::new("/")).await.unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_smb_origin_type() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/share");
|
||||
|
||||
assert_eq!(origin.origin_type(), OriginType::Smb);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_smb_display_name() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let origin = SmbOrigin::from_mount("smb-test", dir.path(), "//server/music");
|
||||
|
||||
assert_eq!(origin.display_name(), "//server/music");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
use async_trait::async_trait;
|
||||
use musicfs_core::{DirEntry, FileStat, HealthStatus, OriginId, OriginType, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
#[async_trait]
|
||||
pub trait Origin: Send + Sync {
|
||||
fn id(&self) -> &OriginId;
|
||||
|
||||
fn origin_type(&self) -> OriginType;
|
||||
|
||||
fn display_name(&self) -> &str;
|
||||
|
||||
async fn readdir(&self, path: &Path) -> Result<Vec<DirEntry>>;
|
||||
|
||||
async fn stat(&self, path: &Path) -> Result<FileStat>;
|
||||
|
||||
async fn read(&self, path: &Path, offset: u64, size: u32) -> Result<Vec<u8>>;
|
||||
|
||||
/// Read entire file content (for CDC chunking of files <4GB)
|
||||
async fn read_full(&self, path: &Path) -> Result<Vec<u8>>;
|
||||
|
||||
async fn exists(&self, path: &Path) -> Result<bool>;
|
||||
|
||||
async fn health(&self) -> HealthStatus;
|
||||
|
||||
async fn open_read(&self, path: &Path) -> Result<Box<dyn AsyncRead + Send + Unpin>>;
|
||||
|
||||
async fn watch(&self, path: &Path, callback: WatchCallback) -> Result<WatchHandle>;
|
||||
}
|
||||
|
||||
pub type WatchCallback = Box<dyn Fn(WatchEvent) + Send + Sync>;
|
||||
|
||||
pub struct WatchHandle {
|
||||
_cancel: tokio::sync::oneshot::Sender<()>,
|
||||
}
|
||||
|
||||
impl WatchHandle {
|
||||
pub fn new(cancel: tokio::sync::oneshot::Sender<()>) -> Self {
|
||||
Self { _cancel: cancel }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum WatchEvent {
|
||||
Created(PathBuf),
|
||||
Modified(PathBuf),
|
||||
Deleted(PathBuf),
|
||||
}
|
||||
Reference in New Issue
Block a user