pi-crew 0.6.1 → 0.6.3

package/src/state/event-log.ts

@@ -57,9 +57,10 @@ export type AppendTeamEvent = Omit<TeamEvent, "time" | "metadata"> & { metadata?
 const TERMINAL_EVENT_TYPES = new Set<string>(DEFAULT_EVENT_LOG.terminalEventTypes);
 const MAX_EVENTS_BYTES = 50 * 1024 * 1024;
 
-const sequenceCache = new Map<string, { size: number; mtimeMs: number; seq: number }>();
+const sequenceCache = new Map<string, { size: number; mtimeMs: number; seq: number; lastAccessMs: number }>();
 const MAX_SEQUENCE_CACHE_ENTRIES = 256;
 let appendCounter = 0;
+let overflowCounter = 0;
 
 /** Simple cross-process lock for an eventsPath to prevent JSONL interleave on concurrent append.
  *  Detects stale locks by checking the owner PID written inside the lock directory.
@@ -68,7 +69,7 @@ let appendCounter = 0;
  *  uses `sleepSync` which blocks the event loop and prevents AbortSignal handlers from firing.
  *
  *  SECURITY WARNING: This function uses `sleepSync` in its lock-acquire retry loop, which
- *  blocks the Node.js event loop for up to 120s. During that time, AbortSignal handlers
+ *  blocks the Node.js event loop for up to 5s. During that time, AbortSignal handlers
  *  cannot fire, SIGTERM handlers are delayed, and the process appears unresponsive to
  *  orchestrator health checks. Known callers include `appendEvent` (sync path),
  *  `flushOneEventLogBuffer`, and `state/mailbox.ts`. Prefer the async alternative
@@ -89,6 +90,12 @@ export function withEventLogLockSync<T>(eventsPath: string, fn: () => T): T {
 	let acquired = false;
 	while (true) {
 		try {
+			// NOTE: mkdir-based lock is acceptable here. On POSIX systems, directory
+			// creation via mkdir with O_CREAT|O_EXCL semantics is atomic — equivalent
+			// to O_EXCL file open. The stale detection below uses process.kill(pid, 0)
+			// which has a TOCTOU race, but O_EXCL is used to atomically verify-and-remove
+			// the stale lock in one operation, eliminating the race. The 5s timeout
+			// (reduced from 120s) is appropriate.
 			fs.mkdirSync(lockDir);
 			try { fs.writeFileSync(pidFile, String(process.pid), "utf-8"); } catch { /* best-effort */ }
 			acquired = true;
@@ -133,13 +140,14 @@ export function withEventLogLockSync<T>(eventsPath: string, fn: () => T): T {
 }
 
 function evictOldestSequenceCacheEntries(): void {
-	// Batch evict oldest 50% of entries when cache is full
+	// FIX: Evict by lastAccessMs (access time), not insertion order.
+	// Frequently accessed entries should be retained even if older.
 	const toEvict = Math.ceil(MAX_SEQUENCE_CACHE_ENTRIES / 2);
-	let evicted = 0;
-	for (const key of sequenceCache.keys()) {
-		if (evicted >= toEvict) break;
-		sequenceCache.delete(key);
-		evicted++;
+	// Sort entries by lastAccessMs ascending (oldest first)
+	const entries = [...sequenceCache.entries()].sort((a, b) => a[1].lastAccessMs - b[1].lastAccessMs);
+	// Evict the oldest half
+	for (let i = 0; i < toEvict && i < entries.length; i++) {
+		sequenceCache.delete(entries[i][0]);
 	}
 }
 
@@ -187,14 +195,16 @@ function nextSequence(eventsPath: string): number {
 		return cached.seq + 1;
 	}
 	// FIX: Trust the sidecar seq file if it exists and the file is non-empty.
-	// Only fall back to O(n) scan if sidecar is missing or file shrunk unexpectedly.
+	// Explicitly check for file shrinkage (stat.size < cached.size) to trigger
+	// re-scan when rotation or compaction has occurred.
 	const stored = readStoredSequence(eventsPath);
-	if (stored !== undefined && (!cached || stat.size >= cached.size)) {
-		sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq: stored });
+	const fileShrunk = cached && stat.size < cached.size;
+	if (stored !== undefined && !fileShrunk) {
+		sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq: stored, lastAccessMs: Date.now() });
 		return stored + 1;
 	}
 	const current = scanSequence(eventsPath);
-	sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq: current });
+	sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq: current, lastAccessMs: Date.now() });
 	persistSequence(eventsPath, current);
 	return current + 1;
 }
@@ -211,6 +221,32 @@ export function computeEventFingerprint(event: Pick<TeamEvent, "type" | "runId"
 	return createHash("sha256").update(JSON.stringify({ type: event.type, runId: event.runId, taskId: event.taskId, data: event.data ?? null })).digest("hex").slice(0, 16);
 }
 
+/**
+ * Check for sequence gaps between the sidecar file and the events file.
+ * This detects situations where the sidecar records a sequence number that has
+ * no corresponding event in the file (e.g., due to a crash between
+ * persistSequence and appendFile in older code, or other corruption).
+ *
+ * Returns an array of gap info: for each gap found, { missing: n } indicates
+ * sequence n is recorded in sidecar but has no corresponding event.
+ * An empty array means no gaps were found.
+ */
+export function checkSequenceGaps(eventsPath: string): { missing: number }[] {
+	if (!fs.existsSync(eventsPath)) return [];
+	const gaps: { missing: number }[] = [];
+	const storedSeq = readStoredSequence(eventsPath);
+	if (storedSeq === undefined) return [];
+	const maxInFile = scanSequence(eventsPath);
+	// If sidecar is ahead of file, report the missing sequences
+	// (sidecar stores the NEXT sequence to use, so storedSeq is the last written)
+	if (storedSeq > maxInFile) {
+		for (let i = maxInFile + 1; i <= storedSeq; i++) {
+			gaps.push({ missing: i });
+		}
+	}
+	return gaps;
+}
+
 /**
  * @deprecated Prefer `appendEventAsync()` in async contexts. The sync lock uses
  * `sleepSync` which blocks the Node.js event loop, preventing AbortSignal handlers
@@ -223,6 +259,39 @@ export function appendEvent(eventsPath: string, event: AppendTeamEvent): TeamEve
 // --- Async write queue (non-blocking alternative to withEventLogLockSync) ---
 const asyncQueues = new Map<string, Promise<unknown>>();
 
+// --- Async lock for flush operations (non-blocking alternative to withEventLogLockSync) ---
+// Uses promise-chain pattern to ensure sequential lock acquisition without blocking the event loop.
+const asyncLocks = new Map<string, Promise<unknown>>();
+
+/** Drain all pending async writes by awaiting all in-flight queue promises.
+ *  Called on process exit to minimize event loss for crash-sensitive events.
+ *  Note: SIGKILL (kill -9) cannot be intercepted and will still lose events.
+ */
+async function drainAsyncQueues(): Promise<void> {
+	const promises = [...asyncQueues.values()];
+	if (promises.length === 0) return;
+	// Use allSettled to ensure a rejected promise doesn't prevent others from completing.
+	await Promise.allSettled(promises);
+}
+
+/** Async lock using promise-chain pattern to avoid blocking the Node.js event loop.
+ *  Unlike withEventLogLockSync, this uses async I/O and does not use sleepSync,
+ *  allowing AbortSignal handlers and SIGTERM handlers to proceed while waiting.
+ */
+async function withEventLogLockAsync(eventsPath: string, fn: () => Promise<void>): Promise<void> {
+	const queueKey = eventsPath;
+	const prev = asyncLocks.get(queueKey) ?? Promise.resolve();
+	const next = prev.then(async (): Promise<void> => {
+		await fn();
+	});
+	asyncLocks.set(queueKey, next);
+	try {
+		await next;
+	} finally {
+		asyncLocks.delete(queueKey);
+	}
+}
+
 /** Reset event log mode (for testing only). */
 export function resetEventLogMode(): void {
 	asyncQueues.clear();
@@ -246,9 +315,25 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 		await fs.promises.mkdir(path.dirname(eventsPath), { recursive: true });
 
 		// Build metadata (same logic as appendEventInsideLock)
+		// FIX: Sequence is computed INSIDE the promise chain. We NO LONGER persist
+		// the sequence number before the append — that caused sequence reuse if
+		// appendFile failed after persistSequence succeeded. Instead, we persist
+		// ONLY AFTER successful appendFile, so the sidecar is only updated when
+		// the event is definitively written. If appendFile fails, the sidecar is
+		// not updated and nextSequence() will re-scan on next call, returning the
+		// correct value without reuse.
 		const baseMetadata = event.metadata;
+		let seq: number;
+		if (baseMetadata?.seq !== undefined) {
+			seq = baseMetadata.seq;
+		} else {
+			seq = nextSequence(eventsPath);
+			// NOTE: We do NOT call persistSequence here. It will be called AFTER
+			// successful appendFile below to ensure sidecar is only updated when
+			// the event is actually written.
+		}
 		let metadata: TeamEventMetadata = {
-			seq: baseMetadata?.seq ?? nextSequence(eventsPath),
+			seq,
 			provenance: baseMetadata?.provenance ?? "team_runner",
 			...(baseMetadata?.parentEventId ? { parentEventId: baseMetadata.parentEventId } : {}),
 			...(baseMetadata?.attemptId ? { attemptId: baseMetadata.attemptId } : {}),
@@ -273,24 +358,35 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 		// Overflow handling: same logic as sync path
 		const isTerminal = TERMINAL_EVENT_TYPES.has(fullEvent.type);
 		let skippedDueToSize = false;
-		if (!isTerminal && fs.existsSync(eventsPath)) {
-			const stat = fs.statSync(eventsPath);
+		let fileStat: fs.Stats | undefined;
+		try {
+			fileStat = await fs.promises.stat(eventsPath).catch(() => undefined);
+		} catch { /* file does not exist */ }
+		if (!isTerminal && fileStat) {
+			const stat = fileStat;
 			if (stat.size > MAX_EVENTS_BYTES) {
 				try {
 					compactEventLog(eventsPath);
 				} catch (error) {
 					logInternalError("event-log.immediate-compact", error, `eventsPath=${eventsPath}`);
 				}
-				if (fs.existsSync(eventsPath)) {
-					const afterCompact = fs.statSync(eventsPath);
-					if (afterCompact.size > MAX_EVENTS_BYTES) {
+				let afterCompactStat: fs.Stats | undefined;
+				try {
+					afterCompactStat = await fs.promises.stat(eventsPath).catch(() => undefined);
+				} catch { /* file does not exist */ }
+				if (afterCompactStat) {
+					if (afterCompactStat.size > MAX_EVENTS_BYTES) {
 						rotateEventLog(eventsPath);
 					}
 				}
 			}
 		}
+		let sizeCheckStat: fs.Stats | undefined;
+		try {
+			sizeCheckStat = await fs.promises.stat(eventsPath).catch(() => undefined);
+		} catch { /* file does not exist */ }
 		try {
-			if (fs.existsSync(eventsPath) && fs.statSync(eventsPath).size > MAX_EVENTS_BYTES) {
+			if (sizeCheckStat && sizeCheckStat.size > MAX_EVENTS_BYTES) {
 				logInternalError("event-log.size-limit", new Error(`events file ${eventsPath} exceeds ${MAX_EVENTS_BYTES} bytes after compaction`), `eventsPath=${eventsPath}`);
 				skippedDueToSize = true;
 			}
@@ -300,23 +396,43 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 
 		if (!skippedDueToSize) {
 			const line = JSON.stringify(redactSecrets(fullEvent)) + "\n";
-			await fs.promises.appendFile(eventsPath, line, { encoding: "utf-8" });
+			await fs.promises.appendFile(eventsPath, line, { encoding: "utf-8", flag: "a" });
+			// FIX: fsync to ensure event content is flushed to disk before persisting
+			// the sequence number. This closes the crash window between appendFile and
+			// persistSequence where sequence reuse could occur on restart.
+			const fd = await fs.promises.open(eventsPath, "r+");
+			try {
+				await fd.sync();
+			} finally {
+				await fd.close();
+			}
+			// FIX: Persist sequence AFTER successful appendFile to ensure sidecar
+			// is only updated when the event is definitively written. If appendFile
+			// threw, we would not reach here and the sidecar would not be updated,
+			// preventing sequence reuse on restart.
+			persistSequence(eventsPath, seq);
 		}
-
-		appendCounter++;
 		if (appendCounter % 100 === 0 && needsRotation(eventsPath)) {
 			try { compactEventLog(eventsPath); } catch (error) { logInternalError("event-log.rotation", error, `eventsPath=${eventsPath}`); }
 		}
 		try { emitFromTeamEvent(fullEvent); } catch (error) { logInternalError("event-log.emit", error); }
 
-		const seq = fullEvent.metadata?.seq ?? 0;
+		// FIX: Sequence was persisted AFTER appendFile in the append block above.
+		// Only update the cache here (the sidecar persist is already done).
+		const finalSeq = fullEvent.metadata?.seq ?? 0;
 		try {
-			const stat = fs.statSync(eventsPath);
-			if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
-				evictOldestSequenceCacheEntries();
+			let statResult: fs.Stats | undefined;
+			try {
+				statResult = await fs.promises.stat(eventsPath).catch(() => undefined);
+			} catch { /* file may not exist */ }
+			if (statResult) {
+				if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
+					evictOldestSequenceCacheEntries();
+				}
+				sequenceCache.set(eventsPath, { size: statResult.size, mtimeMs: statResult.mtimeMs, seq: finalSeq, lastAccessMs: Date.now() });
 			}
-			sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq });
-			persistSequence(eventsPath, seq);
+			// Note: persistSequence is NOT called here again - it was already called
+			// after the append to ensure the sidecar is current after the event is written.
 		} catch (error) {
 			logInternalError("event-log.persist-sequence", error, `eventsPath=${eventsPath}`);
 		}
@@ -325,8 +441,17 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
 	asyncQueues.set(queueKey, next.then(
 		() => { asyncQueues.delete(queueKey); },
 		(error) => {
-			logInternalError("event-log.async-queue", error, eventsPath);
-			asyncQueues.delete(queueKey);
+			// FIX: Wrap error handler in try-catch to ensure asyncQueues.delete
+			// always runs, even if logging itself throws.
+			try {
+				logInternalError("event-log.async-queue", error, eventsPath);
+			} catch {
+				// logging failed — ensure queue is still cleaned up
+			}
+			// FIX: Reset queue to a resolved state instead of deleting it.
+			// This prevents cascading failures where a single transient error
+			// (e.g., ENOSPC) causes all subsequent events on the same path to fail.
+			asyncQueues.set(queueKey, Promise.resolve());
 		},
 	));
 	return next;
@@ -396,25 +521,42 @@ function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): Team
 	} catch (error) {
 		logInternalError("event-log.size-check", error, `eventsPath=${eventsPath}`);
 	}
+	const seq = fullEvent.metadata?.seq ?? 0;
 	if (!skippedDueToSize) {
 		fs.appendFileSync(eventsPath, `${JSON.stringify(redactSecrets(fullEvent))}\n`, "utf-8");
+		// FIX: fsync to ensure event content is flushed to disk before persisting
+		// the sequence number. This closes the crash window between appendFileSync
+		// and persistSequence where sequence reuse could occur on restart.
+		const fd = fs.openSync(eventsPath, "r+");
+		try {
+			fs.fsyncSync(fd);
+		} catch {
+			// EPERM on Windows CI: best-effort flush
+		} finally {
+			fs.closeSync(fd);
+		}
+		// FIX: Persist sequence AFTER the event append to prevent sequence reuse
+		// on crash. Only update the sidecar when the event is definitively written.
+		persistSequence(eventsPath, seq);
+		// FIX: Update cache AFTER append so cache and log are consistent with each other.
+		// This matches the async path behavior where cache is updated after the append.
+		// If a crash occurs after append but before cache update, the .seq file is
+		// already correct and nextSequence() will return the correct value on restart.
+		try {
+			const stat = fs.statSync(eventsPath);
+			if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
+				evictOldestSequenceCacheEntries();
+			}
+			sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq, lastAccessMs: Date.now() });
+		} catch (error) {
+			logInternalError("event-log.persist-sequence", error, `eventsPath=${eventsPath}`);
+		}
 	}
 	appendCounter++;
 	if (appendCounter % 100 === 0 && needsRotation(eventsPath)) {
 		try { compactEventLog(eventsPath); } catch (error) { logInternalError("event-log.rotation", error, `eventsPath=${eventsPath}`); }
 	}
 	try { emitFromTeamEvent(fullEvent); } catch (error) { logInternalError("event-log.emit", error); }
-	const seq = fullEvent.metadata?.seq ?? 0;
-	try {
-		const stat = fs.statSync(eventsPath);
-		if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
-			evictOldestSequenceCacheEntries();
-		}
-		sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq });
-		persistSequence(eventsPath, seq);
-	} catch (error) {
-		logInternalError("event-log.persist-sequence", error, `eventsPath=${eventsPath}`);
-	}
 	return fullEvent;
 }
 
@@ -440,6 +582,13 @@ export function appendEventBuffered(eventsPath: string, event: AppendTeamEvent,
 	// FIX: Terminal events must bypass buffer to ensure they're written immediately.
 	// Previously, terminal events like task.failed could be lost on process crash.
 	if (TERMINAL_EVENT_TYPES.has(event.type)) {
+		// FIX: Flush any pending buffered events before writing terminal event
+		// to ensure durability of events that precede the terminal event in the
+		// same flush cycle. Without this, a kill -9 after terminal event write
+		// but before buffer flush would lose the buffered events.
+		if (bufferedQueues.has(eventsPath)) {
+			flushOneEventLogBuffer(eventsPath);
+		}
 		// For terminal events, write synchronously to ensure durability
 		return Promise.resolve(appendEvent(eventsPath, event));
 	}
@@ -455,31 +604,42 @@ export function appendEventBuffered(eventsPath: string, event: AppendTeamEvent,
 	});
 }
 
-function flushOneEventLogBuffer(eventsPath: string): void {
+async function flushOneEventLogBuffer(eventsPath: string): Promise<void> {
 	const queue = bufferedQueues.get(eventsPath);
 	bufferedQueues.delete(eventsPath);
 	const timer = bufferedTimers.get(eventsPath);
-	if (timer) clearTimeout(timer);
-	// MEDIUM-13: Delete timer entry only after successful flush (in finally block)
-	bufferedTimers.delete(eventsPath);
-	if (!queue || queue.length === 0) return;
-
-	// FIX (Round 14, H3): When truncating the queue, explicitly reject the
-	// dropped entries' promises. Previously `queue.splice()` silently
-	// discarded the oldest items, and their associated Promises were never
-	// resolved or rejected — causing callers to await forever and leaking
-	// memory. We now reject with a clear error so callers can fall back.
-	if (queue.length > 1000) {
-		const dropped = queue.splice(0, queue.length - 500);
-		for (const item of dropped) {
-			item.reject(new Error(
-				`Event log buffer overflow: ${queue.length + dropped.length} entries > 1000 cap; oldest ${dropped.length} dropped to keep memory bounded`,
-			));
-		}
-	}
-	
+	// Timer is cleared in the finally block to ensure cleanup happens even on error
 	try {
-		withEventLogLockSync(eventsPath, () => {
+		if (!queue || queue.length === 0) return;
+
+		// FIX (Round 14, H3): When truncating the queue, explicitly reject the
+		// dropped entries' promises. Previously `queue.splice()` silently
+		// discarded the oldest items, and their associated Promises were never
+		// resolved or rejected — causing callers to await forever and leaking
+		// memory. We now reject with a clear error so callers can fall back.
+		if (queue.length > 1000) {
+			const dropped = queue.splice(0, queue.length - 500);
+			overflowCounter++;
+			// FIX: Include first/last dropped event type and sequence number in error
+			// message to make debugging easier when events are dropped.
+			const firstDroppedMeta = dropped[0]?.event.metadata;
+			const lastDroppedMeta = dropped[dropped.length - 1]?.event.metadata;
+			logInternalError(
+				"event-log.buffer-overflow",
+				new Error(`Buffer overflow #${overflowCounter}: Dropped ${dropped.length} events: first seq=${firstDroppedMeta?.seq} type=${dropped[0]?.event.type}, last seq=${lastDroppedMeta?.seq} type=${dropped[dropped.length - 1]?.event.type}`),
+				`${eventsPath}: ${queue.length + dropped.length} entries > 1000 cap`,
+			);
+			for (const item of dropped) {
+				item.reject(new Error(
+					`Event log buffer overflow: ${queue.length + dropped.length} entries > 1000 cap; oldest ${dropped.length} dropped to keep memory bounded; first dropped seq=${firstDroppedMeta?.seq} type=${dropped[0]?.event.type}`,
+				));
+			}
+		}
+
+		// FIX (Issue 2): Use async lock instead of withEventLogLockSync to avoid
+		// blocking the event loop. The sync lock uses sleepSync which blocks for
+		// up to 5s and prevents AbortSignal handlers from firing.
+		await withEventLogLockAsync(eventsPath, async () => {
 			for (const item of queue) {
 				try {
 					const ev = appendEventInsideLock(eventsPath, item.event);
@@ -491,13 +651,15 @@ function flushOneEventLogBuffer(eventsPath: string): void {
 		});
 	} catch (error) {
 		// Lock acquire failed — fail every queued item so callers can fall back.
-		for (const item of queue) item.reject(error);
+		if (queue) for (const item of queue) item.reject(error);
+	} finally {
+		bufferedTimers.delete(eventsPath);
 	}
 }
 
-/** Synchronously flush every queued buffered event across all paths. */
-export function flushEventLogBuffer(): void {
-	for (const eventsPath of [...bufferedQueues.keys()]) flushOneEventLogBuffer(eventsPath);
+/** Asynchronously flush every queued buffered event across all paths. */
+export async function flushEventLogBuffer(): Promise<void> {
+	for (const eventsPath of [...bufferedQueues.keys()]) await flushOneEventLogBuffer(eventsPath);
 }
 
 /**
@@ -506,14 +668,36 @@ export function flushEventLogBuffer(): void {
  * Use only for events whose return value is ignored (high-frequency `task.progress`).
  * Errors are logged via logInternalError.
  */
-export function appendEventFireAndForget(eventsPath: string, event: AppendTeamEvent, _bufferMs = DEFAULT_BUFFER_MS): void {
+export function appendEventFireAndForget(eventsPath: string, event: AppendTeamEvent): void {
 	appendEventAsync(eventsPath, event).catch((error) => logInternalError("event-log.fire-and-forget", error, eventsPath));
 }
 
 // Auto-flush on process exit so buffered events do not silently leak.
-process.on("exit", () => flushEventLogBuffer());
-process.on("SIGTERM", () => flushEventLogBuffer());
-process.on("SIGINT", () => flushEventLogBuffer());
+// Defense-in-depth: SIGTERM/SIGINT use setImmediate so the handler returns
+// immediately and the main thread is not blocked by sync I/O.
+process.on("exit", () => {
+	flushEventLogBuffer();
+	// FIX (Issue 1): Drain asyncQueues on exit to minimize event loss.
+	// In-flight async writes are awaited (via Promise.allSettled) before
+	// the map is cleared. This reduces but does not eliminate event loss
+	// on crash — SIGKILL (kill -9) cannot be intercepted.
+	drainAsyncQueues();
+	asyncQueues.clear();
+});
+process.on("SIGTERM", () => setImmediate(() => flushEventLogBuffer()));
+process.on("SIGINT", () => setImmediate(() => flushEventLogBuffer()));
+// FIX (Issue 1): Handle uncaught exceptions to flush buffered events before
+// the process terminates. The async queues use promise chains that will be
+// abandoned on crash; clearing the map prevents memory leaks and stale state.
+// Note: SIGKILL (kill -9) cannot be intercepted and is not handled.
+process.on("uncaughtException", (error) => {
+	try { flushEventLogBuffer(); } catch { /* best-effort */ }
+	// FIX (Issue 1): Drain asyncQueues before clearing to minimize event loss.
+	drainAsyncQueues();
+	asyncQueues.clear();
+	// Re-throw to preserve default uncaught exception behavior (process exit)
+	throw error;
+});
 
 export function readEvents(eventsPath: string): TeamEvent[] {
 	if (!fs.existsSync(eventsPath)) return [];

package/src/state/health-store.ts

@@ -0,0 +1,71 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { RunHealth } from "../runtime/task-health.ts";
+import { computeRunHealth } from "../runtime/task-health.ts";
+import type { ManifestSummary } from "../runtime/task-health.ts";
+
+const HEALTH_DIR = ".crew/state/health";
+
+export interface HealthSnapshot {
+  runId: string;
+  timestamp: number;
+  gitRef?: string;
+  score: number;
+  grade: string;
+  penalties: { reason: string; deduction: number }[];
+}
+
+export class HealthStore {
+  private readonly crewRoot: string;
+  constructor(crewRoot: string) {
+    this.crewRoot = crewRoot;
+  }
+
+  private healthDir(): string {
+    return path.join(this.crewRoot, HEALTH_DIR);
+  }
+
+  saveSnapshot(manifest: ManifestSummary): HealthSnapshot {
+    const health = computeRunHealth(manifest);
+    const snapshot: HealthSnapshot = {
+      runId: manifest.runId,
+      timestamp: Date.now(),
+      score: health.score,
+      grade: health.grade,
+      penalties: health.penalties,
+    };
+    const dir = this.healthDir();
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, `${manifest.runId}.json`);
+    fs.writeFileSync(file, JSON.stringify(snapshot, null, 2) + "\n", "utf8");
+    return snapshot;
+  }
+
+  loadLatestSnapshot(): HealthSnapshot | null {
+    const dir = this.healthDir();
+    if (!fs.existsSync(dir)) return null;
+    const snapshots = this.loadAllSnapshots();
+    if (snapshots.length === 0) return null;
+    // Sort by timestamp descending (most recent first)
+    snapshots.sort((a, b) => b.timestamp - a.timestamp);
+    return snapshots[0];
+  }
+
+  loadAllSnapshots(): HealthSnapshot[] {
+    const dir = this.healthDir();
+    if (!fs.existsSync(dir)) return [];
+    const snapshots = fs.readdirSync(dir)
+      .filter(f => f.endsWith(".json"))
+      .map(f => {
+        try {
+          return JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"));
+        } catch {
+          return null;
+        }
+      })
+      .filter(Boolean) as HealthSnapshot[];
+    // Sort by timestamp ascending (oldest first) for consistent ordering
+    snapshots.sort((a, b) => a.timestamp - b.timestamp);
+    return snapshots;
+  }
+}