MusicFS/scripts/e2e/run-resilience.sh

#!/usr/bin/env bash
#
# run-resilience.sh — network stability and reconnection E2E suite for musicfs.
#
# Tests client behavior during server outages, network partitions (VM pause),
# cache resilience, and long-outage watcher recovery.
#
# Requires: devenv shell (for ffmpeg, metaflac, id3v2, incus, grpcurl, Postgres)
#
# Usage:
#   scripts/e2e/run-resilience.sh          full suite
#   KEEP=1 scripts/e2e/run-resilience.sh   keep VM + DB alive for debugging
#
set -euo pipefail

# ─────────────────────────── config ────────────────────────────────────
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SCRIPTS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"

MNT="$PROJECT_DIR/target/e2e/mnt"
MUSIC_DIR="$PROJECT_DIR/target/e2e/music"
LOG_DIR="$PROJECT_DIR/logs/e2e"
DB_NAME="musicfs_e2e"
VM_NAME="musicfs-e2e"
PORT=50061
KEEP="${KEEP:-0}"
RUST_LOG_VAL="${RUST_LOG:-musicfs=debug}"

CLIENT_PID=""
STARTED_PG=""
VM_IP=""

. "$SCRIPT_DIR/lib.sh"
. "$SCRIPT_DIR/resilience-helpers.sh"

log()  { printf '\033[1;34m[e2e]\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1;31m[e2e error]\033[0m %s\n' "$*" >&2; exit 1; }

# ──────────────────────── postgres ─────────────────────────────────────
ensure_postgres() {
  if pg_isready -h "$PGHOST" >/dev/null 2>&1; then return 0; fi
  log "Postgres not running — starting via devenv…"
  devenv up postgres &
  STARTED_PG=$!
  local i
  for ((i = 0; i < 45; i++)); do
    if pg_isready -h "$PGHOST" >/dev/null 2>&1; then
      log "Postgres ready"
      return 0
    fi
    sleep 1
  done
  die "Postgres did not become ready in 45s"
}

stop_postgres() {
  [[ -n "$STARTED_PG" ]] || return 0
  log "stopping Postgres (started by this script)…"
  kill "$STARTED_PG" 2>/dev/null || true
  wait "$STARTED_PG" 2>/dev/null || true
  STARTED_PG=""
}

# ──────────────────────── VM management ────────────────────────────────
vm_ip() { incus list "$VM_NAME" -f csv -c 4 2>/dev/null | head -1 | awk '{print $1}'; }

detect_e2e_ip() {
  local bridge_cidr
  bridge_cidr="$(incus network get incusbr0 ipv4.address 2>/dev/null || true)"
  if [[ -z "$bridge_cidr" ]]; then echo ""; return; fi
  local gateway="${bridge_cidr%%/*}"
  echo "${gateway%.*}.146"
}

start_vm() {
  log "starting VM '$VM_NAME' (port $PORT)…"
  local e2e_ip; e2e_ip="$(detect_e2e_ip)"
  VM_NAME="$VM_NAME" \
  VM_IP="$e2e_ip" \
  MUSIC_SOURCE="$MUSIC_DIR" \
  LISTEN_PORT="$PORT" \
  RUST_LOG=debug \
  bash "$SCRIPTS_DIR/vm.sh" up

  VM_IP="$(vm_ip)"
  [[ -n "$VM_IP" ]] || die "could not determine VM IP"
  log "VM IP: $VM_IP"

  log "waiting for health check…"
  local i
  for ((i = 0; i < 45; i++)); do
    if grpcurl -plaintext "$VM_IP:$PORT" grpc.health.v1.Health/Check 2>/dev/null | grep -q SERVING; then
      log "server healthy"
      return
    fi
    sleep 1
  done
  die "server health check timed out"
}

destroy_vm() {
  VM_NAME="$VM_NAME" bash "$SCRIPTS_DIR/vm.sh" destroy 2>/dev/null || true
}

# ──────────────────────── DB management ────────────────────────────────
reset_db() {
  log "resetting database '$DB_NAME'…"
  dropdb --if-exists "$DB_NAME" 2>/dev/null || true
  createdb "$DB_NAME"
  psql -q -d "$DB_NAME" -f "$PROJECT_DIR/db/schema.sql"
}

drop_db() {
  dropdb --if-exists "$DB_NAME" 2>/dev/null || true
}

# ────────────────────── client lifecycle ───────────────────────────────
build_client() {
  log "building client…"
  (cd "$PROJECT_DIR" && cargo build --bin musicfs)
}

start_client() {
  mkdir -p "$MNT" "$LOG_DIR"
  local dsn="postgresql://fujin@localhost/$DB_NAME?host=$PGHOST"

  log "starting client (mountpoint $MNT)…"
  RUST_LOG="$RUST_LOG_VAL" \
  "$PROJECT_DIR/target/debug/musicfs" \
    --source "http://$VM_IP:$PORT" \
    --mountpoint "$MNT" \
    --database "$dsn" \
    --log-dir "$LOG_DIR" &
  CLIENT_PID=$!
  wait_for_mount
}

wait_for_mount() {
  log "waiting for FUSE mount…"
  local i
  for ((i = 0; i < 60; i++)); do
    if [[ -d "$MNT/E2E Artist A" ]]; then
      log "mount ready"
      return
    fi
    if ! kill -0 "$CLIENT_PID" 2>/dev/null; then
      die "client process exited prematurely — check $LOG_DIR"
    fi
    sleep 1
  done
  die "FUSE mount not ready after 60s"
}

stop_client() {
  [[ -n "$CLIENT_PID" ]] || return 0
  kill "$CLIENT_PID" 2>/dev/null || true
  local i
  for ((i = 0; i < 10; i++)); do
    kill -0 "$CLIENT_PID" 2>/dev/null || break
    sleep 1
  done
  kill -9 "$CLIENT_PID" 2>/dev/null || true
  wait "$CLIENT_PID" 2>/dev/null || true
  CLIENT_PID=""
  fusermount -u "$MNT" 2>/dev/null || true
  sleep 1
}

# ─────────────────────────── cleanup ───────────────────────────────────
cleanup() {
  local rv=$?
  log "cleaning up…"
  stop_client || true
  if [[ "$KEEP" != "1" ]]; then
    drop_db
    destroy_vm
    stop_postgres || true
    rm -rf "$MNT"
  else
    log "KEEP=1 — VM, DB, and mount left intact for debugging"
    log "  mountpoint: $MNT"
    log "  VM IP:      ${VM_IP:-<unknown>}"
    log "  logs:       $LOG_DIR"
  fi
  exit $rv
}
trap cleanup EXIT

# ─────────────────────────── paths ─────────────────────────────────────
ALPHA="$MNT/E2E Artist A/E2E Album X/01 - Alpha.flac"
BETA="$MNT/E2E Artist A/E2E Album X/02 - Beta.flac"
GAMMA="$MNT/E2E Artist B/E2E Album Y/01 - Gamma.mp3"
DELTA="$MNT/E2E Artist B/E2E Album Y/02 - Delta.mp3"

# ════════════════════════════════════════════════════════════════════════
# RESILIENCE SCENARIOS
# ════════════════════════════════════════════════════════════════════════

# ── Scenario 1: Server Stop → Outage Window → Server Start → Recovery ──
#
# Disruption: systemctl stop musicfs-server.service
# Tests: FUSE error handling (EIO for uncached reads), cache fallback,
#        tonic lazy channel auto-reconnect after clean server shutdown.
test_server_stop_start() {
  printf '\n\033[1;36m── scenario 1: server stop/start ──\033[0m\n'
  clear_cache

  local watch_marker; watch_marker="$(log_line_count)"

  # Baseline: read works
  assert_fuse_read_ok "baseline — read Alpha (uncached)" "$ALPHA"

  # Cache Alpha by reading it; leave Gamma uncached
  # (Alpha is now cached from the read above)

  # ── Disruption: stop server ──
  server_stop
  assert_health_down "server is down after stop"

  # During outage
  assert_fuse_read_ok "outage — cached read Alpha succeeds" "$ALPHA"
  assert_fuse_eio_within "outage — uncached read Gamma fails (EIO)" "$GAMMA"
  assert_dir_listing "outage — directory listing still works" "$MNT" "E2E Artist A"

  # ── Recovery: start server ──
  server_start
  assert_health_up "server recovered after start"
  assert_fuse_read_ok "recovery — uncached read Gamma succeeds" "$GAMMA"
  assert_metaflac_write_readback "recovery — FLAC write after outage" "$BETA" "ARTIST" "Recovered Artist"

  # EXPECTED FAILURE: watcher should resubscribe quickly after a clean server
  # restart. Currently it sleeps a full POLL_FALLBACK_INTERVAL (30s) before
  # retrying — no exponential backoff or fast-retry on recoverable errors.
  local resub=0 j
  for ((j = 0; j < 10; j++)); do
    if log_contains_since "subscribed to /events" "$watch_marker"; then
      resub=1
      break
    fi
    sleep 1
  done
  if (( resub == 1 )); then
    _res_pass "watcher resubscribed within 10s of server restart"
  else
    _res_fail "watcher resubscribed within 10s of server restart (stuck in 30s retry — no fast reconnect)"
  fi
}

# ── Scenario 2: Network Partition via incus pause → Resume ──
#
# Disruption: incus pause (freezes VM — TCP connections go silent)
# Tests: Silent partition behavior. Uncached reads will hang (no tonic
#        timeout/keepalive). After resume, client should recover.
#
# IMPORTANT: The FUSE daemon (fuser) dispatches requests serially on a single
# thread. An uncached read during a silent partition blocks the daemon in
# block_on(fetch_file_range()) indefinitely — queuing ALL subsequent FUSE
# operations. Therefore:
#   1. Do all non-blocking FUSE ops (dir listing) BEFORE the blocking read.
#   2. Resume the VM IMMEDIATELY after the blocking read.
#   3. Only attempt more FUSE ops after the daemon unblocks.
test_network_partition() {
  printf '\n\033[1;36m── scenario 2: network partition (incus pause) ──\033[0m\n'
  ensure_clean_state

  # Baseline
  assert_fuse_read_ok "baseline — read Alpha" "$ALPHA"

  # ── Disruption: freeze VM ──
  vm_pause
  assert_health_down "server unreachable during pause"

  # Dir listing works (in-memory, no network) — must do this BEFORE the
  # blocking read below, which freezes the single-threaded FUSE daemon.
  assert_dir_listing "partition — directory listing (pre-block)" "$MNT" "E2E Artist A"

  # EXPECTED FAILURE: with proper tonic timeout/keepalive, this read should
  # return EIO within ~10s. Instead it HANGS indefinitely because no timeout
  # is configured — block_on(fetch_file_range()) never returns while the VM
  # is paused. timeout kills dd at 10s, but the daemon thread stays stuck.
  assert_fuse_eio_within "partition — uncached read Gamma returns EIO within 10s" "$GAMMA" 10

  # ── Recovery: resume VM immediately ──
  # After resume, the pending gRPC call completes (server processes the
  # in-flight request), unblocking the FUSE daemon thread.
  vm_resume
  assert_health_up "server recovered after resume"
  assert_recovered "recovery — client reads after partition" "$GAMMA"
  assert_fuse_read_ok "recovery — cached read Alpha" "$ALPHA"
}

# ── Scenario 3: Cache Resilience During Full Outage ──
#
# Disruption: systemctl stop (server down for entire test)
# Tests: Postgres-backed cache provides degraded read-only mode.
#        All pre-warmed files are fully readable. Writes to cached files
#        work (local DB path). Writes to uncached files fail gracefully.
test_cache_resilience() {
  printf '\n\033[1;36m── scenario 3: cache resilience during outage ──\033[0m\n'
  ensure_clean_state

  # Pre-warm: read every file so all are in Postgres cache
  prewarm_all

  # Verify prewarm worked (all files cached)
  assert_fuse_metaflac_ok "prewarm — Alpha metaflac read" "$ALPHA"
  assert_fuse_id3v2_ok "prewarm — Gamma id3v2 read" "$GAMMA"

  # ── Disruption: stop server for entire scenario ──
  server_stop
  assert_health_down "server down for cache test"

  # Cached reads — all should succeed from Postgres cache
  assert_fuse_metaflac_ok "outage — cached FLAC metaflac (Alpha)" "$ALPHA"
  assert_fuse_metaflac_ok "outage — cached FLAC metaflac (Beta)" "$BETA"
  assert_fuse_id3v2_ok "outage — cached MP3 id3v2 (Gamma)" "$GAMMA"
  assert_fuse_id3v2_ok "outage — cached MP3 id3v2 (Delta)" "$DELTA"
  assert_fuse_ffprobe_ok "outage — cached ffprobe (Alpha)" "$ALPHA"

  # Write to cached file — should succeed (write path is local DB only)
  assert_metaflac_write_readback "outage — write to cached FLAC (Beta)" "$BETA" "ARTIST" "Cache Outage Artist"

  # Directory listing — in-memory, no server needed
  assert_dir_listing "outage — dir listing works" "$MNT" "E2E Artist B"

  # ── Recovery ──
  server_start
  assert_health_up "server recovered"
  assert_metaflac_write_readback "recovery — write after cache outage" "$BETA" "ARTIST" "Post Outage Artist"
}

# ── Scenario 4: Long Outage + Watcher Recovery ──
#
# Disruption: incus pause for 90s (> 2x POLL_FALLBACK_INTERVAL of 30s)
# Tests: Watcher retry loop stamina during prolonged partition.
#        After resume, watcher resubscribes and reconcile succeeds.
test_long_outage_watcher() {
  printf '\n\033[1;36m── scenario 4: long outage + watcher recovery ──\033[0m\n'
  ensure_clean_state

  assert_log_contains "baseline — watcher subscribed" "subscribed to /events"

  local marker; marker="$(log_line_count)"

  vm_pause
  assert_health_down "server unreachable during long pause"

  # EXPECTED FAILURE: watcher should detect the dead stream within ~30s
  # (POLL_FALLBACK_INTERVAL). Instead it's stuck on stream.next() for ~70s
  # until the HTTP/2 transport layer times out — no keepalive is configured
  # to detect the silent partition sooner.
  local watcher_pattern="stream error\|subscribe failed\|poll reconcile failed"
  local detected=0
  local i
  for ((i = 0; i < 30; i++)); do
    check_client_alive || break
    if log_contains_since "$watcher_pattern" "$marker"; then
      detected=1
      break
    fi
    sleep 1
  done

  if (( detected == 1 )); then
    _res_pass "watcher detected silent partition within 30s"
  else
    _res_fail "watcher detected silent partition within 30s (stuck on stream.next — no keepalive)"
  fi

  assert_dir_listing "long outage — dir listing works" "$MNT" "E2E Artist A"
  check_client_alive || die "client died during long outage"

  vm_resume
  assert_health_up "server recovered after long pause"
  assert_recovered "client reads after long outage" "$ALPHA"

  local resub=0 j
  for ((j = 0; j < 10; j++)); do
    if log_contains_since "subscribed to /events" "$marker"; then
      resub=1
      break
    fi
    sleep 1
  done
  if (( resub == 1 )); then
    _res_pass "watcher resubscribed within 10s of recovery"
  else
    _res_fail "watcher resubscribed within 10s of recovery (stuck in 30s retry — no fast reconnect)"
  fi
}

# ════════════════════════════════════════════════════════════════════════
# MAIN
# ════════════════════════════════════════════════════════════════════════

main() {
  [[ -n "${PGHOST:-}" ]] || die "PGHOST not set — run inside devenv shell"
  ensure_postgres
  command -v incus    >/dev/null || die "incus not found"
  command -v grpcurl  >/dev/null || die "grpcurl not found"
  command -v metaflac >/dev/null || die "metaflac not found"
  command -v id3v2    >/dev/null || die "id3v2 not found"
  command -v ffprobe  >/dev/null || die "ffprobe not found"

  log "=== Resilience E2E suite start ==="

  # Phase 0: fixtures
  bash "$SCRIPT_DIR/make-fixtures.sh" "$MUSIC_DIR"

  # Phase 1: server
  start_vm

  # Phase 2: client
  build_client
  reset_db
  start_client

  # Phase 3: resilience scenarios
  test_server_stop_start
  test_network_partition
  test_cache_resilience
  test_long_outage_watcher

  # Summary
  printf '\n'
  summary
  local rv=$?
  log "=== Resilience E2E suite done ==="
  exit $rv
}

main "$@"