Implement Phase B: Crash Recovery

Add startup integrity checks, corruption recovery, CAS size limits,
graceful shutdown orchestration, and a task supervisor — turning 5
previously-RED resilience tests GREEN and adding 5 new tests.

- CAS: pre-check size limit in put(), add StoreFull error variant
- CAS: sled corruption recovery in open() (retry then recreate)
- SQLite: open_with_integrity_check() via PRAGMA integrity_check(1)
- tantivy: open_with_recovery() deletes and rebuilds corrupt index
- CLI: CancellationToken-based ordered shutdown sequence
- Core: TaskSupervisor with spawn_supervised/spawn_critical + backoff
- Tests: replace 4 todo!() stubs, add 5 new shutdown/supervisor tests
This commit is contained in:
Alexander
2026-05-13 15:33:23 +02:00
parent 4e394c60ec
commit 5da96ffab2
12 changed files with 485 additions and 14 deletions
+29 -1
View File
@@ -6,7 +6,7 @@ use rusqlite::{params, Connection, OptionalExtension};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tracing::{debug, info};
use tracing::{debug, info, warn};
const SCHEMA: &str = include_str!("schema.sql");
@@ -32,6 +32,34 @@ impl Database {
Ok(db)
}
pub fn open_with_integrity_check(path: &Path) -> Result<Self> {
debug!(?path, "Opening database with integrity check");
let conn = Connection::open(path)
.map_err(|e| Error::Database(format!("open failed: {}", e)))?;
let integrity: String = conn
.query_row("PRAGMA integrity_check(1)", [], |row| row.get(0))
.map_err(|e| Error::Database(format!("integrity check failed: {}", e)))?;
if integrity != "ok" {
warn!(path = ?path, result = %integrity, "Database integrity check failed");
return Err(Error::DatabaseCorrupted(format!(
"integrity check failed: {}", integrity
)));
}
conn.execute_batch(SCHEMA)
.map_err(|e| Error::Database(format!("schema init failed: {}", e)))?;
let db = Self {
conn: Arc::new(Mutex::new(conn)),
};
let count = db.file_count().unwrap_or(0);
info!(path = ?path, file_count = count, "Database opened (integrity verified)");
Ok(db)
}
pub fn open_memory() -> Result<Self> {
let conn = Connection::open_in_memory()
.map_err(|e| Error::Database(format!("open_in_memory failed: {}", e)))?;