Add resilience tests
This commit is contained in:
Alexander
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# resilience-helpers.sh — disruption primitives and FUSE/log assertions for
|
||||
# network stability and reconnection E2E tests.
|
||||
#
|
||||
# Requires lib.sh globals ($FAILURES, $PASSES, $FAILED_TESTS) and config
|
||||
# variables from run-resilience.sh ($VM_NAME, $PORT, $VM_IP, $MNT, $LOG_DIR,
|
||||
# $DB_NAME, $PGHOST, $CLIENT_PID, $ALPHA, $BETA, $GAMMA, $DELTA).
|
||||
#
|
||||
# Sourced by run-resilience.sh — do NOT execute directly.
|
||||
|
||||
# ── timeouts (seconds) ──
|
||||
HEALTH_TIMEOUT=45
|
||||
READ_TIMEOUT=15
|
||||
READ_FAIL_TIMEOUT=10
|
||||
RECOVER_TIMEOUT=30
|
||||
VM_RESUME_TIMEOUT=30
|
||||
|
||||
# ── internal pass/fail (reuses lib.sh globals) ──
|
||||
_res_pass() {
|
||||
printf ' \033[1;32mPASS\033[0m %s\n' "$1"
|
||||
((++PASSES))
|
||||
}
|
||||
|
||||
_res_fail() {
|
||||
printf ' \033[1;31mFAIL\033[0m %s\n' "$1"
|
||||
((++FAILURES))
|
||||
FAILED_TESTS+="$1; "
|
||||
}
|
||||
|
||||
# ── client liveness ──
|
||||
check_client_alive() {
|
||||
if [[ -n "$CLIENT_PID" ]] && ! kill -0 "$CLIENT_PID" 2>/dev/null; then
|
||||
printf '\033[1;31m[e2e error]\033[0m client process died — check %s\n' "$LOG_DIR" >&2
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── disruption primitives ──
|
||||
server_stop() {
|
||||
log "stopping server in VM…"
|
||||
incus exec "$VM_NAME" -- systemctl stop musicfs-server.service 2>/dev/null || true
|
||||
}
|
||||
|
||||
server_start() {
|
||||
log "starting server in VM…"
|
||||
incus exec "$VM_NAME" -- systemctl start musicfs-server.service 2>/dev/null || true
|
||||
}
|
||||
|
||||
vm_pause() {
|
||||
log "pausing VM '$VM_NAME'…"
|
||||
incus pause "$VM_NAME"
|
||||
}
|
||||
|
||||
vm_resume() {
|
||||
log "resuming VM '$VM_NAME'…"
|
||||
incus start "$VM_NAME"
|
||||
local i
|
||||
for ((i = 0; i < VM_RESUME_TIMEOUT; i++)); do
|
||||
if incus exec "$VM_NAME" -- true 2>/dev/null; then return 0; fi
|
||||
sleep 1
|
||||
done
|
||||
die "VM did not become reachable after resume"
|
||||
}
|
||||
|
||||
# ── health check helpers ──
|
||||
wait_health() {
|
||||
local timeout="${1:-$HEALTH_TIMEOUT}"
|
||||
local i
|
||||
for ((i = 0; i < timeout; i++)); do
|
||||
if timeout 5 grpcurl -plaintext "$VM_IP:$PORT" grpc.health.v1.Health/Check 2>/dev/null | grep -q SERVING; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
assert_health_down() {
|
||||
local desc="$1"
|
||||
if timeout 5 grpcurl -plaintext "$VM_IP:$PORT" grpc.health.v1.Health/Check 2>/dev/null | grep -q SERVING; then
|
||||
_res_fail "$desc (server still serving)"
|
||||
else
|
||||
_res_pass "$desc"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_health_up() {
|
||||
local desc="$1"
|
||||
if wait_health "$HEALTH_TIMEOUT"; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (health check timed out)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── cache control ──
|
||||
clear_cache() {
|
||||
log "clearing Postgres byte cache…"
|
||||
psql -q -d "$DB_NAME" -c "DELETE FROM cached_file_bytes;" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# ── prewarm ──
|
||||
prewarm_file() {
|
||||
local file="$1"
|
||||
timeout "$READ_TIMEOUT" dd if="$file" of=/dev/null bs=4096 2>/dev/null || true
|
||||
}
|
||||
|
||||
prewarm_all() {
|
||||
log "prewarming cache (reading all test files)…"
|
||||
prewarm_file "$ALPHA"
|
||||
prewarm_file "$BETA"
|
||||
prewarm_file "$GAMMA"
|
||||
prewarm_file "$DELTA"
|
||||
}
|
||||
|
||||
# ── FUSE read assertions ──
|
||||
assert_fuse_read_ok() {
|
||||
local desc="$1" file="$2"
|
||||
check_client_alive || return 1
|
||||
if timeout "$READ_TIMEOUT" dd if="$file" of=/dev/null bs=4096 2>/dev/null; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (read failed or timed out after ${READ_TIMEOUT}s)"
|
||||
fi
|
||||
}
|
||||
|
||||
# Assert a FUSE read fails with EIO *before* max_seconds.
|
||||
# Distinguishes "fast EIO" (system returned an error) from "indefinite hang"
|
||||
# (timeout killed dd). Currently FAILS during silent partitions because
|
||||
# there's no tonic timeout configured — the RPC hangs until the OS gives up.
|
||||
assert_fuse_eio_within() {
|
||||
local desc="$1" file="$2" max_seconds="${3:-$READ_FAIL_TIMEOUT}"
|
||||
check_client_alive || return 1
|
||||
local start; start=$(date +%s)
|
||||
local rc=0
|
||||
timeout "$max_seconds" dd if="$file" of=/dev/null bs=4096 2>/dev/null || rc=$?
|
||||
local end; end=$(date +%s)
|
||||
local elapsed=$((end - start))
|
||||
if (( rc == 0 )); then
|
||||
_res_fail "$desc (read succeeded unexpectedly in ${elapsed}s)"
|
||||
elif (( elapsed >= max_seconds )); then
|
||||
_res_fail "$desc (hung ${elapsed}s — no EIO returned; missing tonic timeout)"
|
||||
else
|
||||
_res_pass "$desc (EIO in ${elapsed}s)"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_fuse_metaflac_ok() {
|
||||
local desc="$1" file="$2"
|
||||
check_client_alive || return 1
|
||||
if timeout "$READ_TIMEOUT" metaflac --show-tag=TITLE "$file" >/dev/null 2>&1; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (metaflac read failed)"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_fuse_id3v2_ok() {
|
||||
local desc="$1" file="$2"
|
||||
check_client_alive || return 1
|
||||
if timeout "$READ_TIMEOUT" id3v2 --list "$file" >/dev/null 2>&1; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (id3v2 read failed)"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_fuse_ffprobe_ok() {
|
||||
local desc="$1" file="$2"
|
||||
check_client_alive || return 1
|
||||
if timeout "$READ_TIMEOUT" ffprobe -hide_banner -v quiet -show_format "$file" >/dev/null 2>&1; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (ffprobe failed)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── FUSE directory assertion ──
|
||||
assert_dir_listing() {
|
||||
local desc="$1" dir="$2" expected="$3"
|
||||
check_client_alive || return 1
|
||||
local result; result="$(timeout "$READ_TIMEOUT" ls "$dir" 2>/dev/null || true)"
|
||||
if [[ "$result" == *"$expected"* ]]; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (directory missing '$expected')"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── FUSE write assertions ──
|
||||
assert_metaflac_write_readback() {
|
||||
local desc="$1" file="$2" tag="$3" value="$4"
|
||||
check_client_alive || return 1
|
||||
metaflac --remove-tag="$tag" --set-tag="$tag=$value" "$file" 2>/dev/null
|
||||
local result; result="$(metaflac --show-tag="$tag" "$file" 2>/dev/null || true)"
|
||||
if [[ "$result" == *"$value"* ]]; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (write readback mismatch)"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_mp3_write_readback() {
|
||||
local desc="$1" file="$2" song="$3"
|
||||
check_client_alive || return 1
|
||||
id3v2 --song "$song" "$file" 2>/dev/null
|
||||
local result; result="$(id3v2 --list "$file" 2>/dev/null || true)"
|
||||
if [[ "$result" == *"$song"* ]]; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (MP3 write readback mismatch)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── log assertions ──
|
||||
_get_log_file() {
|
||||
ls -t "$LOG_DIR"/musicfs.*.log 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
assert_log_contains() {
|
||||
local desc="$1" pattern="$2"
|
||||
local log_file; log_file="$(_get_log_file)"
|
||||
if [[ -n "$log_file" ]] && grep -q "$pattern" "$log_file" 2>/dev/null; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (pattern '$pattern' not in logs)"
|
||||
fi
|
||||
}
|
||||
|
||||
log_contains_since() {
|
||||
local pattern="$1" from_line="$2"
|
||||
local log_file; log_file="$(_get_log_file)"
|
||||
[[ -n "$log_file" ]] && tail -n +"$((from_line + 1))" "$log_file" 2>/dev/null | grep -q "$pattern"
|
||||
}
|
||||
|
||||
log_line_count() {
|
||||
local log_file; log_file="$(_get_log_file)"
|
||||
if [[ -n "$log_file" ]]; then
|
||||
wc -l < "$log_file" 2>/dev/null || echo 0
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
}
|
||||
|
||||
assert_log_contains_after() {
|
||||
local desc="$1" pattern="$2" marker="$3"
|
||||
local log_file; log_file="$(_get_log_file)"
|
||||
if [[ -z "$log_file" ]]; then
|
||||
_res_fail "$desc (no log file)"
|
||||
return
|
||||
fi
|
||||
# Find line number of the last occurrence of the marker.
|
||||
local marker_line
|
||||
marker_line="$(grep -n "$marker" "$log_file" 2>/dev/null | tail -1 | cut -d: -f1)"
|
||||
if [[ -z "$marker_line" ]]; then
|
||||
_res_fail "$desc (marker '$marker' not in log)"
|
||||
return
|
||||
fi
|
||||
if tail -n +"$((marker_line + 1))" "$log_file" 2>/dev/null | grep -q "$pattern"; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc ('$pattern' not found after marker line $marker_line)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── recovery wait ──
|
||||
wait_recover() {
|
||||
local file="$1"
|
||||
local timeout="${2:-$RECOVER_TIMEOUT}"
|
||||
local i
|
||||
for ((i = 0; i < timeout; i++)); do
|
||||
check_client_alive || return 1
|
||||
if timeout 5 dd if="$file" of=/dev/null bs=4096 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
assert_recovered() {
|
||||
local desc="$1" file="$2"
|
||||
if wait_recover "$file"; then
|
||||
_res_pass "$desc"
|
||||
else
|
||||
_res_fail "$desc (did not recover within ${RECOVER_TIMEOUT}s)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── scenario isolation ──
|
||||
ensure_clean_state() {
|
||||
log "ensuring clean state between scenarios…"
|
||||
# Server must be up
|
||||
if ! wait_health 5 2>/dev/null; then
|
||||
server_start
|
||||
wait_health "$HEALTH_TIMEOUT" || die "server won't come up between scenarios"
|
||||
fi
|
||||
# VM must be running (not paused)
|
||||
local state
|
||||
state="$(incus list "$VM_NAME" -f csv -c s 2>/dev/null | head -1)"
|
||||
if [[ "$state" == "FROZEN" ]]; then
|
||||
vm_resume
|
||||
wait_health "$HEALTH_TIMEOUT" || die "server unhealthy after VM resume"
|
||||
fi
|
||||
check_client_alive || die "client died between scenarios"
|
||||
clear_cache
|
||||
}
|
||||
Reference in New Issue
Block a user