MusicFS/scripts/e2e/resilience-helpers.sh

#!/usr/bin/env bash
#
# resilience-helpers.sh — disruption primitives and FUSE/log assertions for
# network stability and reconnection E2E tests.
#
# Requires lib.sh globals ($FAILURES, $PASSES, $FAILED_TESTS) and config
# variables from run-resilience.sh ($VM_NAME, $PORT, $VM_IP, $MNT, $LOG_DIR,
# $DB_NAME, $PGHOST, $CLIENT_PID, $ALPHA, $BETA, $GAMMA, $DELTA).
#
# Sourced by run-resilience.sh — do NOT execute directly.

# ── timeouts (seconds) ──
HEALTH_TIMEOUT=45
READ_TIMEOUT=15
READ_FAIL_TIMEOUT=10
RECOVER_TIMEOUT=30
VM_RESUME_TIMEOUT=30

# ── internal pass/fail (reuses lib.sh globals) ──
_res_pass() {
  printf '  \033[1;32mPASS\033[0m %s\n' "$1"
  ((++PASSES))
}

_res_fail() {
  printf '  \033[1;31mFAIL\033[0m %s\n' "$1"
  ((++FAILURES))
  FAILED_TESTS+="$1; "
}

# ── client liveness ──
check_client_alive() {
  if [[ -n "$CLIENT_PID" ]] && ! kill -0 "$CLIENT_PID" 2>/dev/null; then
    printf '\033[1;31m[e2e error]\033[0m client process died — check %s\n' "$LOG_DIR" >&2
    return 1
  fi
  return 0
}

# ── disruption primitives ──
server_stop() {
  log "stopping server in VM…"
  incus exec "$VM_NAME" -- systemctl stop musicfs-server.service 2>/dev/null || true
}

server_start() {
  log "starting server in VM…"
  incus exec "$VM_NAME" -- systemctl start musicfs-server.service 2>/dev/null || true
}

vm_pause() {
  log "pausing VM '$VM_NAME'…"
  incus pause "$VM_NAME"
}

vm_resume() {
  log "resuming VM '$VM_NAME'…"
  incus start "$VM_NAME"
  local i
  for ((i = 0; i < VM_RESUME_TIMEOUT; i++)); do
    if incus exec "$VM_NAME" -- true 2>/dev/null; then return 0; fi
    sleep 1
  done
  die "VM did not become reachable after resume"
}

# ── health check helpers ──
wait_health() {
  local timeout="${1:-$HEALTH_TIMEOUT}"
  local i
  for ((i = 0; i < timeout; i++)); do
    if timeout 5 grpcurl -plaintext "$VM_IP:$PORT" grpc.health.v1.Health/Check 2>/dev/null | grep -q SERVING; then
      return 0
    fi
    sleep 1
  done
  return 1
}

assert_health_down() {
  local desc="$1"
  if timeout 5 grpcurl -plaintext "$VM_IP:$PORT" grpc.health.v1.Health/Check 2>/dev/null | grep -q SERVING; then
    _res_fail "$desc (server still serving)"
  else
    _res_pass "$desc"
  fi
}

assert_health_up() {
  local desc="$1"
  if wait_health "$HEALTH_TIMEOUT"; then
    _res_pass "$desc"
  else
    _res_fail "$desc (health check timed out)"
  fi
}

# ── cache control ──
clear_cache() {
  log "clearing Postgres byte cache…"
  psql -q -d "$DB_NAME" -c "DELETE FROM cached_file_bytes;" 2>/dev/null || true
}

# ── prewarm ──
prewarm_file() {
  local file="$1"
  timeout "$READ_TIMEOUT" dd if="$file" of=/dev/null bs=4096 2>/dev/null || true
}

prewarm_all() {
  log "prewarming cache (reading all test files)…"
  prewarm_file "$ALPHA"
  prewarm_file "$BETA"
  prewarm_file "$GAMMA"
  prewarm_file "$DELTA"
}

# ── FUSE read assertions ──
assert_fuse_read_ok() {
  local desc="$1" file="$2"
  check_client_alive || return 1
  if timeout "$READ_TIMEOUT" dd if="$file" of=/dev/null bs=4096 2>/dev/null; then
    _res_pass "$desc"
  else
    _res_fail "$desc (read failed or timed out after ${READ_TIMEOUT}s)"
  fi
}

# Assert a FUSE read fails with EIO *before* max_seconds.
# Distinguishes "fast EIO" (system returned an error) from "indefinite hang"
# (timeout killed dd). Currently FAILS during silent partitions because
# there's no tonic timeout configured — the RPC hangs until the OS gives up.
assert_fuse_eio_within() {
  local desc="$1" file="$2" max_seconds="${3:-$READ_FAIL_TIMEOUT}"
  check_client_alive || return 1
  local start; start=$(date +%s)
  local rc=0
  timeout "$max_seconds" dd if="$file" of=/dev/null bs=4096 2>/dev/null || rc=$?
  local end; end=$(date +%s)
  local elapsed=$((end - start))
  if (( rc == 0 )); then
    _res_fail "$desc (read succeeded unexpectedly in ${elapsed}s)"
  elif (( elapsed >= max_seconds )); then
    _res_fail "$desc (hung ${elapsed}s — no EIO returned; missing tonic timeout)"
  else
    _res_pass "$desc (EIO in ${elapsed}s)"
  fi
}

assert_fuse_metaflac_ok() {
  local desc="$1" file="$2"
  check_client_alive || return 1
  if timeout "$READ_TIMEOUT" metaflac --show-tag=TITLE "$file" >/dev/null 2>&1; then
    _res_pass "$desc"
  else
    _res_fail "$desc (metaflac read failed)"
  fi
}

assert_fuse_id3v2_ok() {
  local desc="$1" file="$2"
  check_client_alive || return 1
  if timeout "$READ_TIMEOUT" id3v2 --list "$file" >/dev/null 2>&1; then
    _res_pass "$desc"
  else
    _res_fail "$desc (id3v2 read failed)"
  fi
}

assert_fuse_ffprobe_ok() {
  local desc="$1" file="$2"
  check_client_alive || return 1
  if timeout "$READ_TIMEOUT" ffprobe -hide_banner -v quiet -show_format "$file" >/dev/null 2>&1; then
    _res_pass "$desc"
  else
    _res_fail "$desc (ffprobe failed)"
  fi
}

# ── FUSE directory assertion ──
assert_dir_listing() {
  local desc="$1" dir="$2" expected="$3"
  check_client_alive || return 1
  local result; result="$(timeout "$READ_TIMEOUT" ls "$dir" 2>/dev/null || true)"
  if [[ "$result" == *"$expected"* ]]; then
    _res_pass "$desc"
  else
    _res_fail "$desc (directory missing '$expected')"
  fi
}

# ── FUSE write assertions ──
assert_metaflac_write_readback() {
  local desc="$1" file="$2" tag="$3" value="$4"
  check_client_alive || return 1
  metaflac --remove-tag="$tag" --set-tag="$tag=$value" "$file" 2>/dev/null
  local result; result="$(metaflac --show-tag="$tag" "$file" 2>/dev/null || true)"
  if [[ "$result" == *"$value"* ]]; then
    _res_pass "$desc"
  else
    _res_fail "$desc (write readback mismatch)"
  fi
}

assert_mp3_write_readback() {
  local desc="$1" file="$2" song="$3"
  check_client_alive || return 1
  id3v2 --song "$song" "$file" 2>/dev/null
  local result; result="$(id3v2 --list "$file" 2>/dev/null || true)"
  if [[ "$result" == *"$song"* ]]; then
    _res_pass "$desc"
  else
    _res_fail "$desc (MP3 write readback mismatch)"
  fi
}

# ── log assertions ──
_get_log_file() {
  ls -t "$LOG_DIR"/musicfs.*.log 2>/dev/null | head -1
}

assert_log_contains() {
  local desc="$1" pattern="$2"
  local log_file; log_file="$(_get_log_file)"
  if [[ -n "$log_file" ]] && grep -q "$pattern" "$log_file" 2>/dev/null; then
    _res_pass "$desc"
  else
    _res_fail "$desc (pattern '$pattern' not in logs)"
  fi
}

log_contains_since() {
  local pattern="$1" from_line="$2"
  local log_file; log_file="$(_get_log_file)"
  [[ -n "$log_file" ]] && tail -n +"$((from_line + 1))" "$log_file" 2>/dev/null | grep -q "$pattern"
}

log_line_count() {
  local log_file; log_file="$(_get_log_file)"
  if [[ -n "$log_file" ]]; then
    wc -l < "$log_file" 2>/dev/null || echo 0
  else
    echo 0
  fi
}

assert_log_contains_after() {
  local desc="$1" pattern="$2" marker="$3"
  local log_file; log_file="$(_get_log_file)"
  if [[ -z "$log_file" ]]; then
    _res_fail "$desc (no log file)"
    return
  fi
  # Find line number of the last occurrence of the marker.
  local marker_line
  marker_line="$(grep -n "$marker" "$log_file" 2>/dev/null | tail -1 | cut -d: -f1)"
  if [[ -z "$marker_line" ]]; then
    _res_fail "$desc (marker '$marker' not in log)"
    return
  fi
  if tail -n +"$((marker_line + 1))" "$log_file" 2>/dev/null | grep -q "$pattern"; then
    _res_pass "$desc"
  else
    _res_fail "$desc ('$pattern' not found after marker line $marker_line)"
  fi
}

# ── recovery wait ──
wait_recover() {
  local file="$1"
  local timeout="${2:-$RECOVER_TIMEOUT}"
  local i
  for ((i = 0; i < timeout; i++)); do
    check_client_alive || return 1
    if timeout 5 dd if="$file" of=/dev/null bs=4096 2>/dev/null; then
      return 0
    fi
    sleep 1
  done
  return 1
}

assert_recovered() {
  local desc="$1" file="$2"
  if wait_recover "$file"; then
    _res_pass "$desc"
  else
    _res_fail "$desc (did not recover within ${RECOVER_TIMEOUT}s)"
  fi
}

# ── scenario isolation ──
ensure_clean_state() {
  log "ensuring clean state between scenarios…"
  # Server must be up
  if ! wait_health 5 2>/dev/null; then
    server_start
    wait_health "$HEALTH_TIMEOUT" || die "server won't come up between scenarios"
  fi
  # VM must be running (not paused)
  local state
  state="$(incus list "$VM_NAME" -f csv -c s 2>/dev/null | head -1)"
  if [[ "$state" == "FROZEN" ]]; then
    vm_resume
    wait_health "$HEALTH_TIMEOUT" || die "server unhealthy after VM resume"
  fi
  check_client_alive || die "client died between scenarios"
  clear_cache
}