diff --git a/justfile b/justfile index 7794c89..e8b71e2 100644 --- a/justfile +++ b/justfile @@ -7,4 +7,7 @@ up profile: devenv up --profile {{profile}} e2e: - scripts/e2e/run.sh + scripts/e2e/run.sh + +e2e-resilience: + scripts/e2e/run-resilience.sh diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index b66fb6b..7aea62d 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -14,10 +14,13 @@ against the mountpoint. ```sh just e2e # full suite (creates VM, runs tests, destroys VM) KEEP=1 just e2e # keep VM + DB alive after run for debugging +just e2e-resilience # resilience suite (server stop/start, network partition, outage recovery) ``` ## What it tests +### Functional suite (`just e2e`) + | Phase | Description | | ------------ | ------------------------------------------------------- | | structural | Virtual directory tree matches fixture tags | @@ -29,6 +32,15 @@ KEEP=1 just e2e # keep VM + DB alive after run for debugging | chunked write| Non-metadata writes don't corrupt tag structure | | persistence | Tags survive client restart (DB restore path) | +### Resilience suite (`just e2e-resilience`) + +| Scenario | Disruption | What it verifies | +| ----------------- | ----------------------------------- | -------------------------------------------------------- | +| server stop/start | `systemctl stop/start` in VM | EIO on uncached reads, cached reads survive, watcher resubscribes within 10s | +| network partition | `incus pause/start` (silent drop) | EIO within 10s (tonic 3s timeout × kernel readahead), watcher detects via HTTP/2 keepalive within 30s | +| cache resilience | server stop, all files pre-cached | All reads/writes from cache, no server contact needed | +| long outage | `incus pause` for 15s+ | Watcher detects via keepalive PING, resubscribes within 10s of recovery | + ## Fixtures `make-fixtures.sh` generates a deterministic library under `target/e2e/music/`: diff --git a/scripts/e2e/resilience-helpers.sh b/scripts/e2e/resilience-helpers.sh new file mode 100644 index 0000000..2d3aa09 --- /dev/null +++ b/scripts/e2e/resilience-helpers.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash +# +# resilience-helpers.sh — disruption primitives and FUSE/log assertions for +# network stability and reconnection E2E tests. +# +# Requires lib.sh globals ($FAILURES, $PASSES, $FAILED_TESTS) and config +# variables from run-resilience.sh ($VM_NAME, $PORT, $VM_IP, $MNT, $LOG_DIR, +# $DB_NAME, $PGHOST, $CLIENT_PID, $ALPHA, $BETA, $GAMMA, $DELTA). +# +# Sourced by run-resilience.sh — do NOT execute directly. + +# ── timeouts (seconds) ── +HEALTH_TIMEOUT=45 +READ_TIMEOUT=15 +READ_FAIL_TIMEOUT=10 +RECOVER_TIMEOUT=30 +VM_RESUME_TIMEOUT=30 + +# ── internal pass/fail (reuses lib.sh globals) ── +_res_pass() { + printf ' \033[1;32mPASS\033[0m %s\n' "$1" + ((++PASSES)) +} + +_res_fail() { + printf ' \033[1;31mFAIL\033[0m %s\n' "$1" + ((++FAILURES)) + FAILED_TESTS+="$1; " +} + +# ── client liveness ── +check_client_alive() { + if [[ -n "$CLIENT_PID" ]] && ! kill -0 "$CLIENT_PID" 2>/dev/null; then + printf '\033[1;31m[e2e error]\033[0m client process died — check %s\n' "$LOG_DIR" >&2 + return 1 + fi + return 0 +} + +# ── disruption primitives ── +server_stop() { + log "stopping server in VM…" + incus exec "$VM_NAME" -- systemctl stop musicfs-server.service 2>/dev/null || true +} + +server_start() { + log "starting server in VM…" + incus exec "$VM_NAME" -- systemctl start musicfs-server.service 2>/dev/null || true +} + +vm_pause() { + log "pausing VM '$VM_NAME'…" + incus pause "$VM_NAME" +} + +vm_resume() { + log "resuming VM '$VM_NAME'…" + incus start "$VM_NAME" + local i + for ((i = 0; i < VM_RESUME_TIMEOUT; i++)); do + if incus exec "$VM_NAME" -- true 2>/dev/null; then return 0; fi + sleep 1 + done + die "VM did not become reachable after resume" +} + +# ── health check helpers ── +wait_health() { + local timeout="${1:-$HEALTH_TIMEOUT}" + local i + for ((i = 0; i < timeout; i++)); do + if timeout 5 grpcurl -plaintext "$VM_IP:$PORT" grpc.health.v1.Health/Check 2>/dev/null | grep -q SERVING; then + return 0 + fi + sleep 1 + done + return 1 +} + +assert_health_down() { + local desc="$1" + if timeout 5 grpcurl -plaintext "$VM_IP:$PORT" grpc.health.v1.Health/Check 2>/dev/null | grep -q SERVING; then + _res_fail "$desc (server still serving)" + else + _res_pass "$desc" + fi +} + +assert_health_up() { + local desc="$1" + if wait_health "$HEALTH_TIMEOUT"; then + _res_pass "$desc" + else + _res_fail "$desc (health check timed out)" + fi +} + +# ── cache control ── +clear_cache() { + log "clearing Postgres byte cache…" + psql -q -d "$DB_NAME" -c "DELETE FROM cached_file_bytes;" 2>/dev/null || true +} + +# ── prewarm ── +prewarm_file() { + local file="$1" + timeout "$READ_TIMEOUT" dd if="$file" of=/dev/null bs=4096 2>/dev/null || true +} + +prewarm_all() { + log "prewarming cache (reading all test files)…" + prewarm_file "$ALPHA" + prewarm_file "$BETA" + prewarm_file "$GAMMA" + prewarm_file "$DELTA" +} + +# ── FUSE read assertions ── +assert_fuse_read_ok() { + local desc="$1" file="$2" + check_client_alive || return 1 + if timeout "$READ_TIMEOUT" dd if="$file" of=/dev/null bs=4096 2>/dev/null; then + _res_pass "$desc" + else + _res_fail "$desc (read failed or timed out after ${READ_TIMEOUT}s)" + fi +} + +# Assert a FUSE read fails with EIO *before* max_seconds. +# Distinguishes "fast EIO" (system returned an error) from "indefinite hang" +# (timeout killed dd). Currently FAILS during silent partitions because +# there's no tonic timeout configured — the RPC hangs until the OS gives up. +assert_fuse_eio_within() { + local desc="$1" file="$2" max_seconds="${3:-$READ_FAIL_TIMEOUT}" + check_client_alive || return 1 + local start; start=$(date +%s) + local rc=0 + timeout "$max_seconds" dd if="$file" of=/dev/null bs=4096 2>/dev/null || rc=$? + local end; end=$(date +%s) + local elapsed=$((end - start)) + if (( rc == 0 )); then + _res_fail "$desc (read succeeded unexpectedly in ${elapsed}s)" + elif (( elapsed >= max_seconds )); then + _res_fail "$desc (hung ${elapsed}s — no EIO returned; missing tonic timeout)" + else + _res_pass "$desc (EIO in ${elapsed}s)" + fi +} + +assert_fuse_metaflac_ok() { + local desc="$1" file="$2" + check_client_alive || return 1 + if timeout "$READ_TIMEOUT" metaflac --show-tag=TITLE "$file" >/dev/null 2>&1; then + _res_pass "$desc" + else + _res_fail "$desc (metaflac read failed)" + fi +} + +assert_fuse_id3v2_ok() { + local desc="$1" file="$2" + check_client_alive || return 1 + if timeout "$READ_TIMEOUT" id3v2 --list "$file" >/dev/null 2>&1; then + _res_pass "$desc" + else + _res_fail "$desc (id3v2 read failed)" + fi +} + +assert_fuse_ffprobe_ok() { + local desc="$1" file="$2" + check_client_alive || return 1 + if timeout "$READ_TIMEOUT" ffprobe -hide_banner -v quiet -show_format "$file" >/dev/null 2>&1; then + _res_pass "$desc" + else + _res_fail "$desc (ffprobe failed)" + fi +} + +# ── FUSE directory assertion ── +assert_dir_listing() { + local desc="$1" dir="$2" expected="$3" + check_client_alive || return 1 + local result; result="$(timeout "$READ_TIMEOUT" ls "$dir" 2>/dev/null || true)" + if [[ "$result" == *"$expected"* ]]; then + _res_pass "$desc" + else + _res_fail "$desc (directory missing '$expected')" + fi +} + +# ── FUSE write assertions ── +assert_metaflac_write_readback() { + local desc="$1" file="$2" tag="$3" value="$4" + check_client_alive || return 1 + metaflac --remove-tag="$tag" --set-tag="$tag=$value" "$file" 2>/dev/null + local result; result="$(metaflac --show-tag="$tag" "$file" 2>/dev/null || true)" + if [[ "$result" == *"$value"* ]]; then + _res_pass "$desc" + else + _res_fail "$desc (write readback mismatch)" + fi +} + +assert_mp3_write_readback() { + local desc="$1" file="$2" song="$3" + check_client_alive || return 1 + id3v2 --song "$song" "$file" 2>/dev/null + local result; result="$(id3v2 --list "$file" 2>/dev/null || true)" + if [[ "$result" == *"$song"* ]]; then + _res_pass "$desc" + else + _res_fail "$desc (MP3 write readback mismatch)" + fi +} + +# ── log assertions ── +_get_log_file() { + ls -t "$LOG_DIR"/musicfs.*.log 2>/dev/null | head -1 +} + +assert_log_contains() { + local desc="$1" pattern="$2" + local log_file; log_file="$(_get_log_file)" + if [[ -n "$log_file" ]] && grep -q "$pattern" "$log_file" 2>/dev/null; then + _res_pass "$desc" + else + _res_fail "$desc (pattern '$pattern' not in logs)" + fi +} + +log_contains_since() { + local pattern="$1" from_line="$2" + local log_file; log_file="$(_get_log_file)" + [[ -n "$log_file" ]] && tail -n +"$((from_line + 1))" "$log_file" 2>/dev/null | grep -q "$pattern" +} + +log_line_count() { + local log_file; log_file="$(_get_log_file)" + if [[ -n "$log_file" ]]; then + wc -l < "$log_file" 2>/dev/null || echo 0 + else + echo 0 + fi +} + +assert_log_contains_after() { + local desc="$1" pattern="$2" marker="$3" + local log_file; log_file="$(_get_log_file)" + if [[ -z "$log_file" ]]; then + _res_fail "$desc (no log file)" + return + fi + # Find line number of the last occurrence of the marker. + local marker_line + marker_line="$(grep -n "$marker" "$log_file" 2>/dev/null | tail -1 | cut -d: -f1)" + if [[ -z "$marker_line" ]]; then + _res_fail "$desc (marker '$marker' not in log)" + return + fi + if tail -n +"$((marker_line + 1))" "$log_file" 2>/dev/null | grep -q "$pattern"; then + _res_pass "$desc" + else + _res_fail "$desc ('$pattern' not found after marker line $marker_line)" + fi +} + +# ── recovery wait ── +wait_recover() { + local file="$1" + local timeout="${2:-$RECOVER_TIMEOUT}" + local i + for ((i = 0; i < timeout; i++)); do + check_client_alive || return 1 + if timeout 5 dd if="$file" of=/dev/null bs=4096 2>/dev/null; then + return 0 + fi + sleep 1 + done + return 1 +} + +assert_recovered() { + local desc="$1" file="$2" + if wait_recover "$file"; then + _res_pass "$desc" + else + _res_fail "$desc (did not recover within ${RECOVER_TIMEOUT}s)" + fi +} + +# ── scenario isolation ── +ensure_clean_state() { + log "ensuring clean state between scenarios…" + # Server must be up + if ! wait_health 5 2>/dev/null; then + server_start + wait_health "$HEALTH_TIMEOUT" || die "server won't come up between scenarios" + fi + # VM must be running (not paused) + local state + state="$(incus list "$VM_NAME" -f csv -c s 2>/dev/null | head -1)" + if [[ "$state" == "FROZEN" ]]; then + vm_resume + wait_health "$HEALTH_TIMEOUT" || die "server unhealthy after VM resume" + fi + check_client_alive || die "client died between scenarios" + clear_cache +} diff --git a/scripts/e2e/run-resilience.sh b/scripts/e2e/run-resilience.sh new file mode 100755 index 0000000..b55a193 --- /dev/null +++ b/scripts/e2e/run-resilience.sh @@ -0,0 +1,423 @@ +#!/usr/bin/env bash +# +# run-resilience.sh — network stability and reconnection E2E suite for musicfs. +# +# Tests client behavior during server outages, network partitions (VM pause), +# cache resilience, and long-outage watcher recovery. +# +# Requires: devenv shell (for ffmpeg, metaflac, id3v2, incus, grpcurl, Postgres) +# +# Usage: +# scripts/e2e/run-resilience.sh full suite +# KEEP=1 scripts/e2e/run-resilience.sh keep VM + DB alive for debugging +# +set -euo pipefail + +# ─────────────────────────── config ──────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SCRIPTS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +MNT="$PROJECT_DIR/target/e2e/mnt" +MUSIC_DIR="$PROJECT_DIR/target/e2e/music" +LOG_DIR="$PROJECT_DIR/logs/e2e" +DB_NAME="musicfs_e2e" +VM_NAME="musicfs-e2e" +PORT=50061 +KEEP="${KEEP:-0}" +RUST_LOG_VAL="${RUST_LOG:-musicfs=debug}" + +CLIENT_PID="" +STARTED_PG="" +VM_IP="" + +. "$SCRIPT_DIR/lib.sh" +. "$SCRIPT_DIR/resilience-helpers.sh" + +log() { printf '\033[1;34m[e2e]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31m[e2e error]\033[0m %s\n' "$*" >&2; exit 1; } + +# ──────────────────────── postgres ───────────────────────────────────── +ensure_postgres() { + if pg_isready -h "$PGHOST" >/dev/null 2>&1; then return 0; fi + log "Postgres not running — starting via devenv…" + devenv up postgres & + STARTED_PG=$! + local i + for ((i = 0; i < 45; i++)); do + if pg_isready -h "$PGHOST" >/dev/null 2>&1; then + log "Postgres ready" + return 0 + fi + sleep 1 + done + die "Postgres did not become ready in 45s" +} + +stop_postgres() { + [[ -n "$STARTED_PG" ]] || return 0 + log "stopping Postgres (started by this script)…" + kill "$STARTED_PG" 2>/dev/null || true + wait "$STARTED_PG" 2>/dev/null || true + STARTED_PG="" +} + +# ──────────────────────── VM management ──────────────────────────────── +vm_ip() { incus list "$VM_NAME" -f csv -c 4 2>/dev/null | head -1 | awk '{print $1}'; } + +start_vm() { + log "starting VM '$VM_NAME' (port $PORT)…" + VM_NAME="$VM_NAME" \ + MUSIC_SOURCE="$MUSIC_DIR" \ + LISTEN_PORT="$PORT" \ + RUST_LOG=debug \ + bash "$SCRIPTS_DIR/vm.sh" up + + VM_IP="$(vm_ip)" + [[ -n "$VM_IP" ]] || die "could not determine VM IP" + log "VM IP: $VM_IP" + + log "waiting for health check…" + local i + for ((i = 0; i < 45; i++)); do + if grpcurl -plaintext "$VM_IP:$PORT" grpc.health.v1.Health/Check 2>/dev/null | grep -q SERVING; then + log "server healthy" + return + fi + sleep 1 + done + die "server health check timed out" +} + +destroy_vm() { + VM_NAME="$VM_NAME" bash "$SCRIPTS_DIR/vm.sh" destroy 2>/dev/null || true +} + +# ──────────────────────── DB management ──────────────────────────────── +reset_db() { + log "resetting database '$DB_NAME'…" + dropdb --if-exists "$DB_NAME" 2>/dev/null || true + createdb "$DB_NAME" + psql -q -d "$DB_NAME" -f "$PROJECT_DIR/db/schema.sql" +} + +drop_db() { + dropdb --if-exists "$DB_NAME" 2>/dev/null || true +} + +# ────────────────────── client lifecycle ─────────────────────────────── +build_client() { + log "building client…" + (cd "$PROJECT_DIR" && cargo build --bin musicfs) +} + +start_client() { + mkdir -p "$MNT" "$LOG_DIR" + local dsn="postgresql://fujin@localhost/$DB_NAME?host=$PGHOST" + + log "starting client (mountpoint $MNT)…" + RUST_LOG="$RUST_LOG_VAL" \ + "$PROJECT_DIR/target/debug/musicfs" \ + --source "http://$VM_IP:$PORT" \ + --mountpoint "$MNT" \ + --database "$dsn" \ + --log-dir "$LOG_DIR" & + CLIENT_PID=$! + wait_for_mount +} + +wait_for_mount() { + log "waiting for FUSE mount…" + local i + for ((i = 0; i < 60; i++)); do + if [[ -d "$MNT/E2E Artist A" ]]; then + log "mount ready" + return + fi + if ! kill -0 "$CLIENT_PID" 2>/dev/null; then + die "client process exited prematurely — check $LOG_DIR" + fi + sleep 1 + done + die "FUSE mount not ready after 60s" +} + +stop_client() { + [[ -n "$CLIENT_PID" ]] || return 0 + kill "$CLIENT_PID" 2>/dev/null || true + local i + for ((i = 0; i < 10; i++)); do + kill -0 "$CLIENT_PID" 2>/dev/null || break + sleep 1 + done + kill -9 "$CLIENT_PID" 2>/dev/null || true + wait "$CLIENT_PID" 2>/dev/null || true + CLIENT_PID="" + fusermount -u "$MNT" 2>/dev/null || true + sleep 1 +} + +# ─────────────────────────── cleanup ─────────────────────────────────── +cleanup() { + local rv=$? + log "cleaning up…" + stop_client || true + if [[ "$KEEP" != "1" ]]; then + drop_db + destroy_vm + stop_postgres || true + rm -rf "$MNT" + else + log "KEEP=1 — VM, DB, and mount left intact for debugging" + log " mountpoint: $MNT" + log " VM IP: ${VM_IP:-}" + log " logs: $LOG_DIR" + fi + exit $rv +} +trap cleanup EXIT + +# ─────────────────────────── paths ───────────────────────────────────── +ALPHA="$MNT/E2E Artist A/E2E Album X/01 - Alpha.flac" +BETA="$MNT/E2E Artist A/E2E Album X/02 - Beta.flac" +GAMMA="$MNT/E2E Artist B/E2E Album Y/01 - Gamma.mp3" +DELTA="$MNT/E2E Artist B/E2E Album Y/02 - Delta.mp3" + +# ════════════════════════════════════════════════════════════════════════ +# RESILIENCE SCENARIOS +# ════════════════════════════════════════════════════════════════════════ + +# ── Scenario 1: Server Stop → Outage Window → Server Start → Recovery ── +# +# Disruption: systemctl stop musicfs-server.service +# Tests: FUSE error handling (EIO for uncached reads), cache fallback, +# tonic lazy channel auto-reconnect after clean server shutdown. +test_server_stop_start() { + printf '\n\033[1;36m── scenario 1: server stop/start ──\033[0m\n' + clear_cache + + local watch_marker; watch_marker="$(log_line_count)" + + # Baseline: read works + assert_fuse_read_ok "baseline — read Alpha (uncached)" "$ALPHA" + + # Cache Alpha by reading it; leave Gamma uncached + # (Alpha is now cached from the read above) + + # ── Disruption: stop server ── + server_stop + assert_health_down "server is down after stop" + + # During outage + assert_fuse_read_ok "outage — cached read Alpha succeeds" "$ALPHA" + assert_fuse_eio_within "outage — uncached read Gamma fails (EIO)" "$GAMMA" + assert_dir_listing "outage — directory listing still works" "$MNT" "E2E Artist A" + + # ── Recovery: start server ── + server_start + assert_health_up "server recovered after start" + assert_fuse_read_ok "recovery — uncached read Gamma succeeds" "$GAMMA" + assert_metaflac_write_readback "recovery — FLAC write after outage" "$BETA" "ARTIST" "Recovered Artist" + + # EXPECTED FAILURE: watcher should resubscribe quickly after a clean server + # restart. Currently it sleeps a full POLL_FALLBACK_INTERVAL (30s) before + # retrying — no exponential backoff or fast-retry on recoverable errors. + local resub=0 j + for ((j = 0; j < 10; j++)); do + if log_contains_since "subscribed to /events" "$watch_marker"; then + resub=1 + break + fi + sleep 1 + done + if (( resub == 1 )); then + _res_pass "watcher resubscribed within 10s of server restart" + else + _res_fail "watcher resubscribed within 10s of server restart (stuck in 30s retry — no fast reconnect)" + fi +} + +# ── Scenario 2: Network Partition via incus pause → Resume ── +# +# Disruption: incus pause (freezes VM — TCP connections go silent) +# Tests: Silent partition behavior. Uncached reads will hang (no tonic +# timeout/keepalive). After resume, client should recover. +# +# IMPORTANT: The FUSE daemon (fuser) dispatches requests serially on a single +# thread. An uncached read during a silent partition blocks the daemon in +# block_on(fetch_file_range()) indefinitely — queuing ALL subsequent FUSE +# operations. Therefore: +# 1. Do all non-blocking FUSE ops (dir listing) BEFORE the blocking read. +# 2. Resume the VM IMMEDIATELY after the blocking read. +# 3. Only attempt more FUSE ops after the daemon unblocks. +test_network_partition() { + printf '\n\033[1;36m── scenario 2: network partition (incus pause) ──\033[0m\n' + ensure_clean_state + + # Baseline + assert_fuse_read_ok "baseline — read Alpha" "$ALPHA" + + # ── Disruption: freeze VM ── + vm_pause + assert_health_down "server unreachable during pause" + + # Dir listing works (in-memory, no network) — must do this BEFORE the + # blocking read below, which freezes the single-threaded FUSE daemon. + assert_dir_listing "partition — directory listing (pre-block)" "$MNT" "E2E Artist A" + + # EXPECTED FAILURE: with proper tonic timeout/keepalive, this read should + # return EIO within ~10s. Instead it HANGS indefinitely because no timeout + # is configured — block_on(fetch_file_range()) never returns while the VM + # is paused. timeout kills dd at 10s, but the daemon thread stays stuck. + assert_fuse_eio_within "partition — uncached read Gamma returns EIO within 10s" "$GAMMA" 10 + + # ── Recovery: resume VM immediately ── + # After resume, the pending gRPC call completes (server processes the + # in-flight request), unblocking the FUSE daemon thread. + vm_resume + assert_health_up "server recovered after resume" + assert_recovered "recovery — client reads after partition" "$GAMMA" + assert_fuse_read_ok "recovery — cached read Alpha" "$ALPHA" +} + +# ── Scenario 3: Cache Resilience During Full Outage ── +# +# Disruption: systemctl stop (server down for entire test) +# Tests: Postgres-backed cache provides degraded read-only mode. +# All pre-warmed files are fully readable. Writes to cached files +# work (local DB path). Writes to uncached files fail gracefully. +test_cache_resilience() { + printf '\n\033[1;36m── scenario 3: cache resilience during outage ──\033[0m\n' + ensure_clean_state + + # Pre-warm: read every file so all are in Postgres cache + prewarm_all + + # Verify prewarm worked (all files cached) + assert_fuse_metaflac_ok "prewarm — Alpha metaflac read" "$ALPHA" + assert_fuse_id3v2_ok "prewarm — Gamma id3v2 read" "$GAMMA" + + # ── Disruption: stop server for entire scenario ── + server_stop + assert_health_down "server down for cache test" + + # Cached reads — all should succeed from Postgres cache + assert_fuse_metaflac_ok "outage — cached FLAC metaflac (Alpha)" "$ALPHA" + assert_fuse_metaflac_ok "outage — cached FLAC metaflac (Beta)" "$BETA" + assert_fuse_id3v2_ok "outage — cached MP3 id3v2 (Gamma)" "$GAMMA" + assert_fuse_id3v2_ok "outage — cached MP3 id3v2 (Delta)" "$DELTA" + assert_fuse_ffprobe_ok "outage — cached ffprobe (Alpha)" "$ALPHA" + + # Write to cached file — should succeed (write path is local DB only) + assert_metaflac_write_readback "outage — write to cached FLAC (Beta)" "$BETA" "ARTIST" "Cache Outage Artist" + + # Directory listing — in-memory, no server needed + assert_dir_listing "outage — dir listing works" "$MNT" "E2E Artist B" + + # ── Recovery ── + server_start + assert_health_up "server recovered" + assert_metaflac_write_readback "recovery — write after cache outage" "$BETA" "ARTIST" "Post Outage Artist" +} + +# ── Scenario 4: Long Outage + Watcher Recovery ── +# +# Disruption: incus pause for 90s (> 2x POLL_FALLBACK_INTERVAL of 30s) +# Tests: Watcher retry loop stamina during prolonged partition. +# After resume, watcher resubscribes and reconcile succeeds. +test_long_outage_watcher() { + printf '\n\033[1;36m── scenario 4: long outage + watcher recovery ──\033[0m\n' + ensure_clean_state + + assert_log_contains "baseline — watcher subscribed" "subscribed to /events" + + local marker; marker="$(log_line_count)" + + vm_pause + assert_health_down "server unreachable during long pause" + + # EXPECTED FAILURE: watcher should detect the dead stream within ~30s + # (POLL_FALLBACK_INTERVAL). Instead it's stuck on stream.next() for ~70s + # until the HTTP/2 transport layer times out — no keepalive is configured + # to detect the silent partition sooner. + local watcher_pattern="stream error\|subscribe failed\|poll reconcile failed" + local detected=0 + local i + for ((i = 0; i < 30; i++)); do + check_client_alive || break + if log_contains_since "$watcher_pattern" "$marker"; then + detected=1 + break + fi + sleep 1 + done + + if (( detected == 1 )); then + _res_pass "watcher detected silent partition within 30s" + else + _res_fail "watcher detected silent partition within 30s (stuck on stream.next — no keepalive)" + fi + + assert_dir_listing "long outage — dir listing works" "$MNT" "E2E Artist A" + check_client_alive || die "client died during long outage" + + vm_resume + assert_health_up "server recovered after long pause" + assert_recovered "client reads after long outage" "$ALPHA" + + local resub=0 j + for ((j = 0; j < 10; j++)); do + if log_contains_since "subscribed to /events" "$marker"; then + resub=1 + break + fi + sleep 1 + done + if (( resub == 1 )); then + _res_pass "watcher resubscribed within 10s of recovery" + else + _res_fail "watcher resubscribed within 10s of recovery (stuck in 30s retry — no fast reconnect)" + fi +} + +# ════════════════════════════════════════════════════════════════════════ +# MAIN +# ════════════════════════════════════════════════════════════════════════ + +main() { + [[ -n "${PGHOST:-}" ]] || die "PGHOST not set — run inside devenv shell" + ensure_postgres + command -v incus >/dev/null || die "incus not found" + command -v grpcurl >/dev/null || die "grpcurl not found" + command -v metaflac >/dev/null || die "metaflac not found" + command -v id3v2 >/dev/null || die "id3v2 not found" + command -v ffprobe >/dev/null || die "ffprobe not found" + + log "=== Resilience E2E suite start ===" + + # Phase 0: fixtures + bash "$SCRIPT_DIR/make-fixtures.sh" "$MUSIC_DIR" + + # Phase 1: server + start_vm + + # Phase 2: client + build_client + reset_db + start_client + + # Phase 3: resilience scenarios + test_server_stop_start + test_network_partition + test_cache_resilience + test_long_outage_watcher + + # Summary + printf '\n' + summary + local rv=$? + log "=== Resilience E2E suite done ===" + exit $rv +} + +main "$@" diff --git a/src/origins/network/transport.rs b/src/origins/network/transport.rs index cbd2e38..0d5ec6c 100644 --- a/src/origins/network/transport.rs +++ b/src/origins/network/transport.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result, anyhow}; use tokio_stream::StreamExt; @@ -28,6 +29,11 @@ impl NetworkTransport { let endpoint: tonic::transport::Endpoint = url .try_into() .map_err(|e| anyhow!("invalid server URL: {e}"))?; + let endpoint = endpoint + .timeout(Duration::from_secs(3)) + .connect_timeout(Duration::from_secs(5)) + .http2_keep_alive_interval(Duration::from_secs(10)) + .keep_alive_timeout(Duration::from_secs(5)); let channel = endpoint.connect_lazy(); let client = Arc::new(tokio::sync::Mutex::new(MusicFsClient::new(channel))); return Ok(NetworkTransport { client, endpoint }); diff --git a/src/origins/network/watcher.rs b/src/origins/network/watcher.rs index 8bf1d7e..0a079ba 100644 --- a/src/origins/network/watcher.rs +++ b/src/origins/network/watcher.rs @@ -17,7 +17,8 @@ use crate::origins::{FileWatcher, WatcherHandle}; use crate::proto::ManifestEntry as ProtoManifestEntry; use tracing::{info, warn}; -const POLL_FALLBACK_INTERVAL: Duration = Duration::from_secs(30); +const INITIAL_RETRY_DELAY: Duration = Duration::from_secs(1); +const MAX_RETRY_DELAY: Duration = Duration::from_secs(30); pub struct NetworkOriginFileWatcher { transport: NetworkTransport, @@ -84,6 +85,7 @@ struct WatcherState { impl WatcherState { async fn run_loop(self, shutdown: Arc) { + let mut retry_delay = INITIAL_RETRY_DELAY; loop { let subscribed = tokio::select! { biased; @@ -93,6 +95,7 @@ impl WatcherState { match subscribed { Ok(mut stream) => { info!("network watcher: subscribed to /events"); + retry_delay = INITIAL_RETRY_DELAY; loop { let item = tokio::select! { biased; @@ -120,8 +123,9 @@ impl WatcherState { tokio::select! { biased; _ = shutdown.notified() => return, - _ = tokio::time::sleep(POLL_FALLBACK_INTERVAL) => {} + _ = tokio::time::sleep(retry_delay) => {} } + retry_delay = (retry_delay * 2).min(MAX_RETRY_DELAY); if let Err(e) = self.reconcile_once().await { warn!(error = %e, "network watcher: poll reconcile failed"); }