pi-crew 0.2.2 → 0.2.4

package/src/runtime/group-join.ts

@@ -1,106 +1,234 @@
-import type { CrewRuntimeConfig } from "../config/config.ts";
-import { writeArtifact } from "../state/artifact-store.ts";
-import { appendEvent } from "../state/event-log.ts";
-import { appendMailboxMessage, findMailboxMessageByRequestId, readDeliveryState } from "../state/mailbox.ts";
-import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../state/types.ts";
-import { aggregateTaskOutputs } from "./task-output-context.ts";
-
-export type CrewGroupJoinMode = "off" | "group" | "smart";
-
-export interface CrewGroupJoinDelivery {
-	batchId: string;
-	mode: CrewGroupJoinMode;
-	partial: boolean;
-	taskIds: string[];
-	completed: string[];
-	failed: string[];
-	skipped: string[];
-	remaining: string[];
-	artifact?: ArtifactDescriptor;
-	messageId?: string;
-	requestId?: string;
-	ackRequired?: boolean;
-	ackStatus?: "pending" | "acknowledged";
-}
-
-export function resolveGroupJoinMode(runtime?: CrewRuntimeConfig): CrewGroupJoinMode {
-	return runtime?.groupJoin ?? "smart";
-}
-
-export function shouldGroupJoin(mode: CrewGroupJoinMode, batch: TeamTaskState[]): boolean {
-	if (mode === "off") return false;
-	if (mode === "group") return batch.length > 0;
-	return batch.length > 1;
-}
-
-function batchIdFor(runId: string, taskIds: string[]): string {
-	return `${runId}_${taskIds.join("+").replace(/[^a-zA-Z0-9_+-]/g, "_")}`;
-}
-
-function requestIdFor(runId: string, batchId: string, partial: boolean): string {
-	return `${runId}:group-join:${partial ? "partial" : "completed"}:${batchId}`;
-}
-
-function statusList(tasks: TeamTaskState[], status: TeamTaskState["status"]): string[] {
-	return tasks.filter((task) => task.status === status).map((task) => task.id);
-}
-
-export function deliverGroupJoin(input: {
-	manifest: TeamRunManifest;
-	mode: CrewGroupJoinMode;
-	batch: TeamTaskState[];
-	allTasks: TeamTaskState[];
-	partial?: boolean;
-}): CrewGroupJoinDelivery | undefined {
-	if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
-	const taskIds = input.batch.map((task) => task.id);
-	const latest = taskIds.map((id) => input.allTasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
-	const completed = statusList(latest, "completed");
-	const failed = statusList(latest, "failed");
-	const skipped = statusList(latest, "skipped");
-	const remaining = latest.filter((task) => task.status === "queued" || task.status === "running").map((task) => task.id);
-	const partial = input.partial ?? remaining.length > 0;
-	const batchId = batchIdFor(input.manifest.runId, taskIds);
-	const summary = aggregateTaskOutputs(latest, input.manifest);
-	const requestId = requestIdFor(input.manifest.runId, batchId, partial);
-	const existingMailbox = findMailboxMessageByRequestId(input.manifest, requestId);
-	const existingStatus = existingMailbox ? readDeliveryState(input.manifest).messages[existingMailbox.id] ?? existingMailbox.status : undefined;
-	const delivery: CrewGroupJoinDelivery = { batchId, mode: input.mode, partial, taskIds, completed, failed, skipped, remaining, requestId, ackRequired: true, ackStatus: existingStatus === "acknowledged" ? "acknowledged" : "pending" };
-	const content = `${JSON.stringify({ ...delivery, createdAt: new Date().toISOString() }, null, 2)}\n`;
-	const artifact = writeArtifact(input.manifest.artifactsRoot, {
-		kind: "metadata",
-		relativePath: `metadata/group-joins/${batchId}.json`,
-		producer: "group-join",
-		content,
-	});
-	const mailbox = existingMailbox ?? appendMailboxMessage(input.manifest, {
-		direction: "outbox",
-		from: "group-join",
-		to: "leader",
-		body: [
-			`Group join ${partial ? "partial" : "completed"}: ${taskIds.join(", ")}`,
-			`Request: ${requestId}`,
-			`Completed: ${completed.join(", ") || "none"}`,
-			`Failed: ${failed.join(", ") || "none"}`,
-			`Skipped: ${skipped.join(", ") || "none"}`,
-			`Remaining: ${remaining.join(", ") || "none"}`,
-			"",
-			summary,
-		].join("\n"),
-		status: "delivered",
-		data: { kind: "group_join", requestId, batchId, partial, ackRequired: true, taskIds, completed, failed, skipped, remaining },
-	});
-	appendEvent(input.manifest.eventsPath, {
-		type: partial ? "agent.group_join.partial" : "agent.group_join.completed",
-		runId: input.manifest.runId,
-		message: `Group join ${partial ? "partial" : "completed"} for ${taskIds.length} task(s).`,
-		data: { ...delivery, artifactPath: artifact.path, messageId: mailbox.id, fallback: "mailbox-delivered", reused: Boolean(existingMailbox) },
-	});
-	if (existingMailbox) appendEvent(input.manifest.eventsPath, {
-		type: "agent.group_join.delivery_reused",
-		runId: input.manifest.runId,
-		message: `Reused group join mailbox delivery for ${taskIds.length} task(s).`,
-		data: { requestId, messageId: mailbox.id, batchId, partial },
-	});
-	return { ...delivery, artifact, messageId: mailbox.id };
-}
+import type { CrewRuntimeConfig } from "../config/config.ts";
+import { writeArtifact } from "../state/artifact-store.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { appendMailboxMessage, findMailboxMessageByRequestId, readDeliveryState } from "../state/mailbox.ts";
+import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { aggregateTaskOutputs } from "./task-output-context.ts";
+
+export type CrewGroupJoinMode = "off" | "group" | "smart";
+
+export interface CrewGroupJoinDelivery {
+	batchId: string;
+	mode: CrewGroupJoinMode;
+	partial: boolean;
+	taskIds: string[];
+	completed: string[];
+	failed: string[];
+	skipped: string[];
+	remaining: string[];
+	artifact?: ArtifactDescriptor;
+	messageId?: string;
+	requestId?: string;
+	ackRequired?: boolean;
+	ackStatus?: "pending" | "acknowledged";
+}
+
+export function resolveGroupJoinMode(runtime?: CrewRuntimeConfig): CrewGroupJoinMode {
+	return runtime?.groupJoin ?? "smart";
+}
+
+export function shouldGroupJoin(mode: CrewGroupJoinMode, batch: TeamTaskState[]): boolean {
+	if (mode === "off") return false;
+	if (mode === "group") return batch.length > 0;
+	return batch.length > 1;
+}
+
+function batchIdFor(runId: string, taskIds: string[]): string {
+	return `${runId}_${taskIds.join("+").replace(/[^a-zA-Z0-9_+-]/g, "_")}`;
+}
+
+function requestIdFor(runId: string, batchId: string, partial: boolean): string {
+	return `${runId}:group-join:${partial ? "partial" : "completed"}:${batchId}`;
+}
+
+function statusList(tasks: TeamTaskState[], status: TeamTaskState["status"]): string[] {
+	return tasks.filter((task) => task.status === status).map((task) => task.id);
+}
+
+export function deliverGroupJoin(input: {
+	manifest: TeamRunManifest;
+	mode: CrewGroupJoinMode;
+	batch: TeamTaskState[];
+	allTasks: TeamTaskState[];
+	partial?: boolean;
+}): CrewGroupJoinDelivery | undefined {
+	if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
+	const taskIds = input.batch.map((task) => task.id);
+	const latest = taskIds.map((id) => input.allTasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
+	const completed = statusList(latest, "completed");
+	const failed = statusList(latest, "failed");
+	const skipped = statusList(latest, "skipped");
+	const remaining = latest.filter((task) => task.status === "queued" || task.status === "running").map((task) => task.id);
+	const partial = input.partial ?? remaining.length > 0;
+	const batchId = batchIdFor(input.manifest.runId, taskIds);
+	const summary = aggregateTaskOutputs(latest, input.manifest);
+	const requestId = requestIdFor(input.manifest.runId, batchId, partial);
+	const existingMailbox = findMailboxMessageByRequestId(input.manifest, requestId);
+	const existingStatus = existingMailbox ? readDeliveryState(input.manifest).messages[existingMailbox.id] ?? existingMailbox.status : undefined;
+	const delivery: CrewGroupJoinDelivery = { batchId, mode: input.mode, partial, taskIds, completed, failed, skipped, remaining, requestId, ackRequired: true, ackStatus: existingStatus === "acknowledged" ? "acknowledged" : "pending" };
+	const content = `${JSON.stringify({ ...delivery, createdAt: new Date().toISOString() }, null, 2)}\n`;
+	const artifact = writeArtifact(input.manifest.artifactsRoot, {
+		kind: "metadata",
+		relativePath: `metadata/group-joins/${batchId}.json`,
+		producer: "group-join",
+		content,
+	});
+	const mailbox = existingMailbox ?? appendMailboxMessage(input.manifest, {
+		direction: "outbox",
+		from: "group-join",
+		to: "leader",
+		body: [
+			`Group join ${partial ? "partial" : "completed"}: ${taskIds.join(", ")}`,
+			`Request: ${requestId}`,
+			`Completed: ${completed.join(", ") || "none"}`,
+			`Failed: ${failed.join(", ") || "none"}`,
+			`Skipped: ${skipped.join(", ") || "none"}`,
+			`Remaining: ${remaining.join(", ") || "none"}`,
+			"",
+			summary,
+		].join("\n"),
+		status: "delivered",
+		data: { kind: "group_join", requestId, batchId, partial, ackRequired: true, taskIds, completed, failed, skipped, remaining },
+	});
+	appendEvent(input.manifest.eventsPath, {
+		type: partial ? "agent.group_join.partial" : "agent.group_join.completed",
+		runId: input.manifest.runId,
+		message: `Group join ${partial ? "partial" : "completed"} for ${taskIds.length} task(s).`,
+		data: { ...delivery, artifactPath: artifact.path, messageId: mailbox.id, fallback: "mailbox-delivered", reused: Boolean(existingMailbox) },
+	});
+	if (existingMailbox) appendEvent(input.manifest.eventsPath, {
+		type: "agent.group_join.delivery_reused",
+		runId: input.manifest.runId,
+		message: `Reused group join mailbox delivery for ${taskIds.length} task(s).`,
+		data: { requestId, messageId: mailbox.id, batchId, partial },
+	});
+	return { ...delivery, artifact, messageId: mailbox.id };
+}
+
+import type { CrewAgentRecord } from "./crew-agent-runtime.ts";
+
+export type JoinMode = "async" | "group" | "smart";
+export type DeliveryCallback = (records: CrewAgentRecord[], partial: boolean) => void;
+
+interface AgentGroup {
+	groupId: string;
+	agentIds: Set<string>;
+	completedRecords: Map<string, CrewAgentRecord>;
+	timeoutHandle?: ReturnType<typeof setTimeout>;
+	delivered: boolean;
+	isStraggler: boolean;
+}
+
+const DEFAULT_TIMEOUT = 30_000;
+const STRAGGLER_TIMEOUT = 15_000;
+
+export class GroupJoinManager {
+	private groups = new Map<string, AgentGroup>();
+	private agentToGroup = new Map<string, string>();
+
+	private deliverCb: DeliveryCallback;
+	private groupTimeout: number;
+
+	constructor(
+		deliverCb: DeliveryCallback,
+		groupTimeout = DEFAULT_TIMEOUT,
+	) {
+		this.deliverCb = deliverCb;
+		this.groupTimeout = groupTimeout;
+	}
+
+	registerGroup(groupId: string, agentIds: string[]): void {
+		const group: AgentGroup = {
+			groupId,
+			agentIds: new Set(agentIds),
+			completedRecords: new Map(),
+			delivered: false,
+			isStraggler: false,
+		};
+		this.groups.set(groupId, group);
+		for (const id of agentIds) {
+			this.agentToGroup.set(id, groupId);
+		}
+	}
+
+	onAgentComplete(record: CrewAgentRecord): "delivered" | "held" | "pass" {
+		const groupId = this.agentToGroup.get(record.taskId);
+		if (!groupId) return "pass";
+
+		const group = this.groups.get(groupId);
+		if (!group || group.delivered) return "pass";
+
+		group.completedRecords.set(record.taskId, record);
+
+		if (group.completedRecords.size >= group.agentIds.size) {
+			this.deliver(group, false);
+			return "delivered";
+		}
+
+		if (!group.timeoutHandle) {
+			const timeout = group.isStraggler ? STRAGGLER_TIMEOUT : this.groupTimeout;
+			group.timeoutHandle = setTimeout(() => {
+				this.onTimeout(group);
+			}, timeout);
+		}
+
+		return "held";
+	}
+
+	private onTimeout(group: AgentGroup): void {
+		if (group.delivered) return;
+		group.timeoutHandle = undefined;
+
+		const remaining = new Set<string>();
+		for (const id of group.agentIds) {
+			if (!group.completedRecords.has(id)) remaining.add(id);
+		}
+
+		for (const id of group.completedRecords.keys()) {
+			this.agentToGroup.delete(id);
+		}
+
+		this.deliverCb([...group.completedRecords.values()], true);
+
+		group.completedRecords.clear();
+		group.agentIds = remaining;
+		group.isStraggler = true;
+
+		// Re-arm timer for remaining stragglers so they aren't silently abandoned
+		if (remaining.size > 0) {
+			group.timeoutHandle = setTimeout(() => this.onTimeout(group), STRAGGLER_TIMEOUT);
+			group.timeoutHandle.unref();
+		}
+	}
+
+	private deliver(group: AgentGroup, partial: boolean): void {
+		if (group.timeoutHandle) {
+			clearTimeout(group.timeoutHandle);
+			group.timeoutHandle = undefined;
+		}
+		group.delivered = true;
+		this.deliverCb([...group.completedRecords.values()], partial);
+		this.cleanupGroup(group.groupId);
+	}
+
+	private cleanupGroup(groupId: string): void {
+		const group = this.groups.get(groupId);
+		if (!group) return;
+		for (const id of group.agentIds) {
+			this.agentToGroup.delete(id);
+		}
+		this.groups.delete(groupId);
+	}
+
+	isGrouped(agentId: string): boolean {
+		return this.agentToGroup.has(agentId);
+	}
+
+	dispose(): void {
+		for (const group of this.groups.values()) {
+			if (group.timeoutHandle) clearTimeout(group.timeoutHandle);
+		}
+		this.groups.clear();
+		this.agentToGroup.clear();
+	}
+}

package/src/runtime/heartbeat-watcher.ts

@@ -1,124 +1,145 @@
-import type { NotificationDescriptor } from "../extension/notification-router.ts";
-import type { MetricRegistry } from "../observability/metric-registry.ts";
-import { appendEvent } from "../state/event-log.ts";
-import { loadRunManifestById } from "../state/state-store.ts";
-import type { TeamRunManifest } from "../state/types.ts";
-import { logInternalError } from "../utils/internal-error.ts";
-import type { ManifestCache } from "./manifest-cache.ts";
-import { classifyHeartbeat, DEFAULT_GRADIENT_THRESHOLDS, heartbeatAgeMs, type GradientThresholds, type HeartbeatLevel } from "./heartbeat-gradient.ts";
-
-export interface HeartbeatWatcherRouter {
-	enqueue(notification: NotificationDescriptor): boolean;
-}
-
-export interface HeartbeatWatcherOptions {
-	cwd: string;
-	pollIntervalMs?: number;
-	thresholds?: GradientThresholds;
-	manifestCache: ManifestCache;
-	registry: MetricRegistry;
-	router: HeartbeatWatcherRouter;
-	deadletterTickThreshold?: number;
-	onDead?: (runId: string, taskId: string, elapsed: number) => void;
-	onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
-}
-
-/**
- * Polls running runs for heartbeat staleness.
- *
- * Uses recursive setTimeout to avoid timer storms.
- * Cleanup is done in the same pass — no second scan over manifests.
- * Keys for runs that disappear from the cache are cleaned via staleness-age policy
- * rather than being leaked forever.
- */
-export class HeartbeatWatcher {
-	private timer?: ReturnType<typeof setTimeout>;
-	private lastLevel = new Map<string, HeartbeatLevel>();
-	private consecutiveDead = new Map<string, number>();
-	private lastSeen = new Map<string, number>(); // key → last time it was active
-	/** Max age (ms) to retain a stale key before garbage-collecting it. */
-	private readonly maxKeyAgeMs = 600_000; // 10 minutes
-	private readonly opts: HeartbeatWatcherOptions;
-
-	constructor(opts: HeartbeatWatcherOptions) {
-		this.opts = opts;
-	}
-
-	start(): void {
-		this.dispose();
-		this.scheduleTick();
-	}
-
-	private scheduleTick(): void {
-		this.timer = setTimeout(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
-		this.timer.unref();
-	}
-
-	tick(now = Date.now()): void {
-		try {
-			this.tickUnsafe(now);
-		} catch (error) {
-			logInternalError("heartbeat-watcher.tick", error);
-		} finally {
-			this.scheduleTick();
-		}
-	}
-
-	private tickUnsafe(now: number): void {
-		const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
-		const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
-		const activeKeys = new Set<string>();
-
-		for (const run of this.opts.manifestCache.list(50)) {
-			if (run.status !== "running") continue;
-			const loaded = loadRunManifestById(this.opts.cwd, run.runId);
-			if (!loaded) continue;
-			for (const task of loaded.tasks) {
-				if (task.status !== "running") continue;
-				const key = `${run.runId}:${task.id}`;
-				activeKeys.add(key);
-				this.lastSeen.set(key, now);
-
-				const elapsed = heartbeatAgeMs(task.heartbeat, now);
-				const level = classifyHeartbeat(task.heartbeat, thresholds, now);
-				this.opts.registry.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
-				this.opts.registry.counter("crew.heartbeat.level_total", "Heartbeat classifications by level").inc({ runId: run.runId, level });
-				const previous = this.lastLevel.get(key);
-				this.lastLevel.set(key, level);
-				if (level === "dead" && previous !== "dead") {
-					this.opts.registry.counter("crew.heartbeat.dead_total", "Dead heartbeat detections").inc({ runId: run.runId });
-					appendEvent(loaded.manifest.eventsPath, { type: "crew.task.heartbeat_dead", runId: run.runId, taskId: task.id, message: `Task ${task.id} heartbeat dead.`, data: { elapsedMs: Number.isFinite(elapsed) ? elapsed : undefined } });
-					this.opts.router.enqueue({ id: `dead_${run.runId}_${task.id}`, severity: "warning", source: "heartbeat-watcher", runId: run.runId, title: `Task ${task.id} heartbeat dead`, body: "Background watcher detected a stuck worker." });
-					this.opts.onDead?.(run.runId, task.id, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
-				}
-				if (level === "dead") {
-					const count = (this.consecutiveDead.get(key) ?? 0) + 1;
-					this.consecutiveDead.set(key, count);
-					if (count === tickThreshold) this.opts.onDeadletterTrigger?.(loaded.manifest, task.id);
-				} else {
-					this.consecutiveDead.delete(key);
-				}
-			}
-		}
-
-		// Cleanup: drop keys that were NOT in this tick's active set AND
-		// haven't been seen for > maxKeyAgeMs.  This covers runs that
-		// completed or fell out of the manifest cache's top-50 window.
-		const cutoff = now - this.maxKeyAgeMs;
-		for (const [key, ts] of this.lastSeen) {
-			if (!activeKeys.has(key) && ts < cutoff) {
-				this.lastLevel.delete(key);
-				this.consecutiveDead.delete(key);
-				this.lastSeen.delete(key);
-			}
-		}
-	}
-
-	dispose(): void {
-		if (this.timer) clearTimeout(this.timer);
-		this.timer = undefined;
-		this.lastLevel.clear();
-		this.consecutiveDead.clear();
-		this.lastSeen.clear();
-	}
-}
+import type { NotificationDescriptor } from "../extension/notification-router.ts";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { loadRunManifestById } from "../state/state-store.ts";
+import type { TeamRunManifest } from "../state/types.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import type { ManifestCache } from "./manifest-cache.ts";
+import { classifyHeartbeat, DEFAULT_GRADIENT_THRESHOLDS, heartbeatAgeMs, type GradientThresholds, type HeartbeatLevel } from "./heartbeat-gradient.ts";
+
+export interface HeartbeatWatcherRouter {
+	enqueue(notification: NotificationDescriptor): boolean;
+}
+
+export interface HeartbeatWatcherOptions {
+	cwd: string;
+	pollIntervalMs?: number;
+	thresholds?: GradientThresholds;
+	manifestCache: ManifestCache;
+	registry: MetricRegistry;
+	router: HeartbeatWatcherRouter;
+	deadletterTickThreshold?: number;
+	/**
+	 * 3.6 — minimum interval between repeated deadletter triggers for the same
+	 * runId+taskId. Without this, a flaky worker (dead → alive → dead) can
+	 * fire deadletter entries faster than the operator can respond. Default
+	 * 60_000 ms.
+	 */
+	deadletterCooldownMs?: number;
+	onDead?: (runId: string, taskId: string, elapsed: number) => void;
+	onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
+}
+
+/**
+ * Polls running runs for heartbeat staleness.
+ *
+ * Uses recursive setTimeout to avoid timer storms.
+ * Cleanup is done in the same pass — no second scan over manifests.
+ * Keys for runs that disappear from the cache are cleaned via staleness-age policy
+ * rather than being leaked forever.
+ */
+export class HeartbeatWatcher {
+	private timer?: ReturnType<typeof setTimeout>;
+	private lastLevel = new Map<string, HeartbeatLevel>();
+	private consecutiveDead = new Map<string, number>();
+	private lastSeen = new Map<string, number>(); // key → last time it was active
+	private lastDeadletterTriggerAt = new Map<string, number>(); // 3.6 cooldown gate
+	/** Max age (ms) to retain a stale key before garbage-collecting it. */
+	private readonly maxKeyAgeMs = 600_000; // 10 minutes
+	private readonly opts: HeartbeatWatcherOptions;
+
+	constructor(opts: HeartbeatWatcherOptions) {
+		this.opts = opts;
+	}
+
+	start(): void {
+		this.dispose();
+		this.scheduleTick();
+	}
+
+	private scheduleTick(): void {
+		// 3.2 — when at least one run has a dead-streak in progress, poll faster
+		// (1s) so operators get notified quickly. Healthy state stays at the
+		// configured interval (default 5s) to keep idle CPU near zero.
+		const baseInterval = this.opts.pollIntervalMs ?? 5000;
+		const interval = this.consecutiveDead.size > 0 ? Math.min(1000, baseInterval) : baseInterval;
+		this.timer = setTimeout(() => this.tick(), interval);
+		this.timer.unref();
+	}
+
+	tick(now = Date.now()): void {
+		try {
+			this.tickUnsafe(now);
+		} catch (error) {
+			logInternalError("heartbeat-watcher.tick", error);
+		} finally {
+			this.scheduleTick();
+		}
+	}
+
+	private tickUnsafe(now: number): void {
+		const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
+		const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
+		const activeKeys = new Set<string>();
+
+		for (const run of this.opts.manifestCache.list(50)) {
+			if (run.status !== "running") continue;
+			const loaded = loadRunManifestById(this.opts.cwd, run.runId);
+			if (!loaded) continue;
+			for (const task of loaded.tasks) {
+				if (task.status !== "running") continue;
+				const key = `${run.runId}:${task.id}`;
+				activeKeys.add(key);
+				this.lastSeen.set(key, now);
+
+				const elapsed = heartbeatAgeMs(task.heartbeat, now);
+				const level = classifyHeartbeat(task.heartbeat, thresholds, now);
+				this.opts.registry.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				this.opts.registry.counter("crew.heartbeat.level_total", "Heartbeat classifications by level").inc({ runId: run.runId, level });
+				const previous = this.lastLevel.get(key);
+				this.lastLevel.set(key, level);
+				if (level === "dead" && previous !== "dead") {
+					this.opts.registry.counter("crew.heartbeat.dead_total", "Dead heartbeat detections").inc({ runId: run.runId });
+					appendEvent(loaded.manifest.eventsPath, { type: "crew.task.heartbeat_dead", runId: run.runId, taskId: task.id, message: `Task ${task.id} heartbeat dead.`, data: { elapsedMs: Number.isFinite(elapsed) ? elapsed : undefined } });
+					this.opts.router.enqueue({ id: `dead_${run.runId}_${task.id}`, severity: "warning", source: "heartbeat-watcher", runId: run.runId, title: `Task ${task.id} heartbeat dead`, body: "Background watcher detected a stuck worker." });
+					this.opts.onDead?.(run.runId, task.id, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				}
+				if (level === "dead") {
+					const count = (this.consecutiveDead.get(key) ?? 0) + 1;
+					this.consecutiveDead.set(key, count);
+					if (count === tickThreshold) {
+						// 3.6 cooldown gate
+						const cooldown = this.opts.deadletterCooldownMs ?? 60_000;
+						const lastTrigger = this.lastDeadletterTriggerAt.get(key) ?? 0;
+						if (now - lastTrigger >= cooldown) {
+							this.lastDeadletterTriggerAt.set(key, now);
+							this.opts.onDeadletterTrigger?.(loaded.manifest, task.id);
+						}
+					}
+				} else {
+					this.consecutiveDead.delete(key);
+				}
+			}
+		}
+
+		// Cleanup: drop keys that were NOT in this tick's active set AND
+		// haven't been seen for > maxKeyAgeMs.  This covers runs that
+		// completed or fell out of the manifest cache's top-50 window.
+		const cutoff = now - this.maxKeyAgeMs;
+		for (const [key, ts] of this.lastSeen) {
+			if (!activeKeys.has(key) && ts < cutoff) {
+				this.lastLevel.delete(key);
+				this.consecutiveDead.delete(key);
+				this.lastSeen.delete(key);
+			}
+		}
+	}
+
+	dispose(): void {
+		if (this.timer) clearTimeout(this.timer);
+		this.timer = undefined;
+		this.lastLevel.clear();
+		this.consecutiveDead.clear();
+		this.lastSeen.clear();
+	}
+}