pi-crew 0.1.51 → 0.2.0

package/src/runtime/green-contract.ts

@@ -1,46 +1,46 @@
-import type { GreenLevel, VerificationContract, VerificationEvidence } from "../state/types.ts";
-
-const GREEN_ORDER: Record<GreenLevel, number> = {
-	none: 0,
-	targeted: 1,
-	package: 2,
-	workspace: 3,
-	merge_ready: 4,
-};
-
-export interface GreenContractOutcome {
-	requiredGreenLevel: GreenLevel;
-	observedGreenLevel: GreenLevel;
-	satisfied: boolean;
-}
-
-export function greenLevelSatisfies(observed: GreenLevel, required: GreenLevel): boolean {
-	return GREEN_ORDER[observed] >= GREEN_ORDER[required];
-}
-
-export function evaluateGreenContract(contract: VerificationContract, evidence?: VerificationEvidence): GreenContractOutcome {
-	const observedGreenLevel = evidence?.observedGreenLevel ?? "none";
-	return {
-		requiredGreenLevel: contract.requiredGreenLevel,
-		observedGreenLevel,
-		satisfied: greenLevelSatisfies(observedGreenLevel, contract.requiredGreenLevel),
-	};
-}
-
-export function inferGreenLevelFromTask(success: boolean, contract: VerificationContract): GreenLevel {
-	if (!success) return "none";
-	if (contract.requiredGreenLevel === "none") return "none";
-	return contract.allowManualEvidence ? contract.requiredGreenLevel : "targeted";
-}
-
-export function createVerificationEvidence(contract: VerificationContract, success: boolean, notes: string): VerificationEvidence {
-	const observedGreenLevel = inferGreenLevelFromTask(success, contract);
-	const outcome = evaluateGreenContract(contract, { requiredGreenLevel: contract.requiredGreenLevel, observedGreenLevel, satisfied: false, commands: [], notes });
-	return {
-		requiredGreenLevel: contract.requiredGreenLevel,
-		observedGreenLevel,
-		satisfied: outcome.satisfied,
-		commands: contract.commands.map((cmd) => ({ cmd, status: "not_run" as const })),
-		notes,
-	};
-}
+import type { GreenLevel, VerificationContract, VerificationEvidence } from "../state/types.ts";
+
+const GREEN_ORDER: Record<GreenLevel, number> = {
+	none: 0,
+	targeted: 1,
+	package: 2,
+	workspace: 3,
+	merge_ready: 4,
+};
+
+export interface GreenContractOutcome {
+	requiredGreenLevel: GreenLevel;
+	observedGreenLevel: GreenLevel;
+	satisfied: boolean;
+}
+
+export function greenLevelSatisfies(observed: GreenLevel, required: GreenLevel): boolean {
+	return GREEN_ORDER[observed] >= GREEN_ORDER[required];
+}
+
+export function evaluateGreenContract(contract: VerificationContract, evidence?: VerificationEvidence): GreenContractOutcome {
+	const observedGreenLevel = evidence?.observedGreenLevel ?? "none";
+	return {
+		requiredGreenLevel: contract.requiredGreenLevel,
+		observedGreenLevel,
+		satisfied: greenLevelSatisfies(observedGreenLevel, contract.requiredGreenLevel),
+	};
+}
+
+export function inferGreenLevelFromTask(success: boolean, contract: VerificationContract): GreenLevel {
+	if (!success) return "none";
+	if (contract.requiredGreenLevel === "none") return "none";
+	return contract.allowManualEvidence ? contract.requiredGreenLevel : "targeted";
+}
+
+export function createVerificationEvidence(contract: VerificationContract, success: boolean, notes: string): VerificationEvidence {
+	const observedGreenLevel = inferGreenLevelFromTask(success, contract);
+	const outcome = evaluateGreenContract(contract, { requiredGreenLevel: contract.requiredGreenLevel, observedGreenLevel, satisfied: false, commands: [], notes });
+	return {
+		requiredGreenLevel: contract.requiredGreenLevel,
+		observedGreenLevel,
+		satisfied: outcome.satisfied,
+		commands: contract.commands.map((cmd) => ({ cmd, status: "not_run" as const })),
+		notes,
+	};
+}

package/src/runtime/group-join.ts

@@ -1,106 +1,106 @@
-import type { CrewRuntimeConfig } from "../config/config.ts";
-import { writeArtifact } from "../state/artifact-store.ts";
-import { appendEvent } from "../state/event-log.ts";
-import { appendMailboxMessage, findMailboxMessageByRequestId, readDeliveryState } from "../state/mailbox.ts";
-import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../state/types.ts";
-import { aggregateTaskOutputs } from "./task-output-context.ts";
-
-export type CrewGroupJoinMode = "off" | "group" | "smart";
-
-export interface CrewGroupJoinDelivery {
-	batchId: string;
-	mode: CrewGroupJoinMode;
-	partial: boolean;
-	taskIds: string[];
-	completed: string[];
-	failed: string[];
-	skipped: string[];
-	remaining: string[];
-	artifact?: ArtifactDescriptor;
-	messageId?: string;
-	requestId?: string;
-	ackRequired?: boolean;
-	ackStatus?: "pending" | "acknowledged";
-}
-
-export function resolveGroupJoinMode(runtime?: CrewRuntimeConfig): CrewGroupJoinMode {
-	return runtime?.groupJoin ?? "smart";
-}
-
-export function shouldGroupJoin(mode: CrewGroupJoinMode, batch: TeamTaskState[]): boolean {
-	if (mode === "off") return false;
-	if (mode === "group") return batch.length > 0;
-	return batch.length > 1;
-}
-
-function batchIdFor(runId: string, taskIds: string[]): string {
-	return `${runId}_${taskIds.join("+").replace(/[^a-zA-Z0-9_+-]/g, "_")}`;
-}
-
-function requestIdFor(runId: string, batchId: string, partial: boolean): string {
-	return `${runId}:group-join:${partial ? "partial" : "completed"}:${batchId}`;
-}
-
-function statusList(tasks: TeamTaskState[], status: TeamTaskState["status"]): string[] {
-	return tasks.filter((task) => task.status === status).map((task) => task.id);
-}
-
-export function deliverGroupJoin(input: {
-	manifest: TeamRunManifest;
-	mode: CrewGroupJoinMode;
-	batch: TeamTaskState[];
-	allTasks: TeamTaskState[];
-	partial?: boolean;
-}): CrewGroupJoinDelivery | undefined {
-	if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
-	const taskIds = input.batch.map((task) => task.id);
-	const latest = taskIds.map((id) => input.allTasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
-	const completed = statusList(latest, "completed");
-	const failed = statusList(latest, "failed");
-	const skipped = statusList(latest, "skipped");
-	const remaining = latest.filter((task) => task.status === "queued" || task.status === "running").map((task) => task.id);
-	const partial = input.partial ?? remaining.length > 0;
-	const batchId = batchIdFor(input.manifest.runId, taskIds);
-	const summary = aggregateTaskOutputs(latest, input.manifest);
-	const requestId = requestIdFor(input.manifest.runId, batchId, partial);
-	const existingMailbox = findMailboxMessageByRequestId(input.manifest, requestId);
-	const existingStatus = existingMailbox ? readDeliveryState(input.manifest).messages[existingMailbox.id] ?? existingMailbox.status : undefined;
-	const delivery: CrewGroupJoinDelivery = { batchId, mode: input.mode, partial, taskIds, completed, failed, skipped, remaining, requestId, ackRequired: true, ackStatus: existingStatus === "acknowledged" ? "acknowledged" : "pending" };
-	const content = `${JSON.stringify({ ...delivery, createdAt: new Date().toISOString() }, null, 2)}\n`;
-	const artifact = writeArtifact(input.manifest.artifactsRoot, {
-		kind: "metadata",
-		relativePath: `metadata/group-joins/${batchId}.json`,
-		producer: "group-join",
-		content,
-	});
-	const mailbox = existingMailbox ?? appendMailboxMessage(input.manifest, {
-		direction: "outbox",
-		from: "group-join",
-		to: "leader",
-		body: [
-			`Group join ${partial ? "partial" : "completed"}: ${taskIds.join(", ")}`,
-			`Request: ${requestId}`,
-			`Completed: ${completed.join(", ") || "none"}`,
-			`Failed: ${failed.join(", ") || "none"}`,
-			`Skipped: ${skipped.join(", ") || "none"}`,
-			`Remaining: ${remaining.join(", ") || "none"}`,
-			"",
-			summary,
-		].join("\n"),
-		status: "delivered",
-		data: { kind: "group_join", requestId, batchId, partial, ackRequired: true, taskIds, completed, failed, skipped, remaining },
-	});
-	appendEvent(input.manifest.eventsPath, {
-		type: partial ? "agent.group_join.partial" : "agent.group_join.completed",
-		runId: input.manifest.runId,
-		message: `Group join ${partial ? "partial" : "completed"} for ${taskIds.length} task(s).`,
-		data: { ...delivery, artifactPath: artifact.path, messageId: mailbox.id, fallback: "mailbox-delivered", reused: Boolean(existingMailbox) },
-	});
-	if (existingMailbox) appendEvent(input.manifest.eventsPath, {
-		type: "agent.group_join.delivery_reused",
-		runId: input.manifest.runId,
-		message: `Reused group join mailbox delivery for ${taskIds.length} task(s).`,
-		data: { requestId, messageId: mailbox.id, batchId, partial },
-	});
-	return { ...delivery, artifact, messageId: mailbox.id };
-}
+import type { CrewRuntimeConfig } from "../config/config.ts";
+import { writeArtifact } from "../state/artifact-store.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { appendMailboxMessage, findMailboxMessageByRequestId, readDeliveryState } from "../state/mailbox.ts";
+import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { aggregateTaskOutputs } from "./task-output-context.ts";
+
+export type CrewGroupJoinMode = "off" | "group" | "smart";
+
+export interface CrewGroupJoinDelivery {
+	batchId: string;
+	mode: CrewGroupJoinMode;
+	partial: boolean;
+	taskIds: string[];
+	completed: string[];
+	failed: string[];
+	skipped: string[];
+	remaining: string[];
+	artifact?: ArtifactDescriptor;
+	messageId?: string;
+	requestId?: string;
+	ackRequired?: boolean;
+	ackStatus?: "pending" | "acknowledged";
+}
+
+export function resolveGroupJoinMode(runtime?: CrewRuntimeConfig): CrewGroupJoinMode {
+	return runtime?.groupJoin ?? "smart";
+}
+
+export function shouldGroupJoin(mode: CrewGroupJoinMode, batch: TeamTaskState[]): boolean {
+	if (mode === "off") return false;
+	if (mode === "group") return batch.length > 0;
+	return batch.length > 1;
+}
+
+function batchIdFor(runId: string, taskIds: string[]): string {
+	return `${runId}_${taskIds.join("+").replace(/[^a-zA-Z0-9_+-]/g, "_")}`;
+}
+
+function requestIdFor(runId: string, batchId: string, partial: boolean): string {
+	return `${runId}:group-join:${partial ? "partial" : "completed"}:${batchId}`;
+}
+
+function statusList(tasks: TeamTaskState[], status: TeamTaskState["status"]): string[] {
+	return tasks.filter((task) => task.status === status).map((task) => task.id);
+}
+
+export function deliverGroupJoin(input: {
+	manifest: TeamRunManifest;
+	mode: CrewGroupJoinMode;
+	batch: TeamTaskState[];
+	allTasks: TeamTaskState[];
+	partial?: boolean;
+}): CrewGroupJoinDelivery | undefined {
+	if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
+	const taskIds = input.batch.map((task) => task.id);
+	const latest = taskIds.map((id) => input.allTasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
+	const completed = statusList(latest, "completed");
+	const failed = statusList(latest, "failed");
+	const skipped = statusList(latest, "skipped");
+	const remaining = latest.filter((task) => task.status === "queued" || task.status === "running").map((task) => task.id);
+	const partial = input.partial ?? remaining.length > 0;
+	const batchId = batchIdFor(input.manifest.runId, taskIds);
+	const summary = aggregateTaskOutputs(latest, input.manifest);
+	const requestId = requestIdFor(input.manifest.runId, batchId, partial);
+	const existingMailbox = findMailboxMessageByRequestId(input.manifest, requestId);
+	const existingStatus = existingMailbox ? readDeliveryState(input.manifest).messages[existingMailbox.id] ?? existingMailbox.status : undefined;
+	const delivery: CrewGroupJoinDelivery = { batchId, mode: input.mode, partial, taskIds, completed, failed, skipped, remaining, requestId, ackRequired: true, ackStatus: existingStatus === "acknowledged" ? "acknowledged" : "pending" };
+	const content = `${JSON.stringify({ ...delivery, createdAt: new Date().toISOString() }, null, 2)}\n`;
+	const artifact = writeArtifact(input.manifest.artifactsRoot, {
+		kind: "metadata",
+		relativePath: `metadata/group-joins/${batchId}.json`,
+		producer: "group-join",
+		content,
+	});
+	const mailbox = existingMailbox ?? appendMailboxMessage(input.manifest, {
+		direction: "outbox",
+		from: "group-join",
+		to: "leader",
+		body: [
+			`Group join ${partial ? "partial" : "completed"}: ${taskIds.join(", ")}`,
+			`Request: ${requestId}`,
+			`Completed: ${completed.join(", ") || "none"}`,
+			`Failed: ${failed.join(", ") || "none"}`,
+			`Skipped: ${skipped.join(", ") || "none"}`,
+			`Remaining: ${remaining.join(", ") || "none"}`,
+			"",
+			summary,
+		].join("\n"),
+		status: "delivered",
+		data: { kind: "group_join", requestId, batchId, partial, ackRequired: true, taskIds, completed, failed, skipped, remaining },
+	});
+	appendEvent(input.manifest.eventsPath, {
+		type: partial ? "agent.group_join.partial" : "agent.group_join.completed",
+		runId: input.manifest.runId,
+		message: `Group join ${partial ? "partial" : "completed"} for ${taskIds.length} task(s).`,
+		data: { ...delivery, artifactPath: artifact.path, messageId: mailbox.id, fallback: "mailbox-delivered", reused: Boolean(existingMailbox) },
+	});
+	if (existingMailbox) appendEvent(input.manifest.eventsPath, {
+		type: "agent.group_join.delivery_reused",
+		runId: input.manifest.runId,
+		message: `Reused group join mailbox delivery for ${taskIds.length} task(s).`,
+		data: { requestId, messageId: mailbox.id, batchId, partial },
+	});
+	return { ...delivery, artifact, messageId: mailbox.id };
+}

package/src/runtime/heartbeat-gradient.ts

@@ -1,28 +1,28 @@
-import type { WorkerHeartbeatState } from "./worker-heartbeat.ts";
-
-export type HeartbeatLevel = "healthy" | "warn" | "stale" | "dead";
-
-export interface GradientThresholds {
-	warnMs: number;
-	staleMs: number;
-	deadMs: number;
-}
-
-export const DEFAULT_GRADIENT_THRESHOLDS: GradientThresholds = { warnMs: 30_000, staleMs: 60_000, deadMs: 300_000 };
-
-export function heartbeatAgeMs(heartbeat: WorkerHeartbeatState | undefined, now = Date.now()): number {
-	if (!heartbeat) return Number.POSITIVE_INFINITY;
-	const lastSeen = Date.parse(heartbeat.lastSeenAt);
-	return Number.isFinite(lastSeen) ? Math.max(0, now - lastSeen) : Number.POSITIVE_INFINITY;
-}
-
-export function classifyHeartbeat(heartbeat: WorkerHeartbeatState | undefined, thresholds: GradientThresholds = DEFAULT_GRADIENT_THRESHOLDS, now = Date.now()): HeartbeatLevel {
-	if (!heartbeat) return "dead";
-	if (heartbeat.alive === false) return "dead";
-	const elapsed = heartbeatAgeMs(heartbeat, now);
-	if (!Number.isFinite(elapsed)) return "dead";
-	if (elapsed > thresholds.deadMs) return "dead";
-	if (elapsed > thresholds.staleMs) return "stale";
-	if (elapsed > thresholds.warnMs) return "warn";
-	return "healthy";
-}
+import type { WorkerHeartbeatState } from "./worker-heartbeat.ts";
+
+export type HeartbeatLevel = "healthy" | "warn" | "stale" | "dead";
+
+export interface GradientThresholds {
+	warnMs: number;
+	staleMs: number;
+	deadMs: number;
+}
+
+export const DEFAULT_GRADIENT_THRESHOLDS: GradientThresholds = { warnMs: 30_000, staleMs: 60_000, deadMs: 300_000 };
+
+export function heartbeatAgeMs(heartbeat: WorkerHeartbeatState | undefined, now = Date.now()): number {
+	if (!heartbeat) return Number.POSITIVE_INFINITY;
+	const lastSeen = Date.parse(heartbeat.lastSeenAt);
+	return Number.isFinite(lastSeen) ? Math.max(0, now - lastSeen) : Number.POSITIVE_INFINITY;
+}
+
+export function classifyHeartbeat(heartbeat: WorkerHeartbeatState | undefined, thresholds: GradientThresholds = DEFAULT_GRADIENT_THRESHOLDS, now = Date.now()): HeartbeatLevel {
+	if (!heartbeat) return "dead";
+	if (heartbeat.alive === false) return "dead";
+	const elapsed = heartbeatAgeMs(heartbeat, now);
+	if (!Number.isFinite(elapsed)) return "dead";
+	if (elapsed > thresholds.deadMs) return "dead";
+	if (elapsed > thresholds.staleMs) return "stale";
+	if (elapsed > thresholds.warnMs) return "warn";
+	return "healthy";
+}

package/src/runtime/heartbeat-watcher.ts

@@ -1,124 +1,124 @@
-import type { NotificationDescriptor } from "../extension/notification-router.ts";
-import type { MetricRegistry } from "../observability/metric-registry.ts";
-import { appendEvent } from "../state/event-log.ts";
-import { loadRunManifestById } from "../state/state-store.ts";
-import type { TeamRunManifest } from "../state/types.ts";
-import { logInternalError } from "../utils/internal-error.ts";
-import type { ManifestCache } from "./manifest-cache.ts";
-import { classifyHeartbeat, DEFAULT_GRADIENT_THRESHOLDS, heartbeatAgeMs, type GradientThresholds, type HeartbeatLevel } from "./heartbeat-gradient.ts";
-
-export interface HeartbeatWatcherRouter {
-	enqueue(notification: NotificationDescriptor): boolean;
-}
-
-export interface HeartbeatWatcherOptions {
-	cwd: string;
-	pollIntervalMs?: number;
-	thresholds?: GradientThresholds;
-	manifestCache: ManifestCache;
-	registry: MetricRegistry;
-	router: HeartbeatWatcherRouter;
-	deadletterTickThreshold?: number;
-	onDead?: (runId: string, taskId: string, elapsed: number) => void;
-	onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
-}
-
-/**
- * Polls running runs for heartbeat staleness.
- *
- * Uses recursive setTimeout to avoid timer storms.
- * Cleanup is done in the same pass — no second scan over manifests.
- * Keys for runs that disappear from the cache are cleaned via staleness-age policy
- * rather than being leaked forever.
- */
-export class HeartbeatWatcher {
-	private timer?: ReturnType<typeof setTimeout>;
-	private lastLevel = new Map<string, HeartbeatLevel>();
-	private consecutiveDead = new Map<string, number>();
-	private lastSeen = new Map<string, number>(); // key → last time it was active
-	/** Max age (ms) to retain a stale key before garbage-collecting it. */
-	private readonly maxKeyAgeMs = 600_000; // 10 minutes
-	private readonly opts: HeartbeatWatcherOptions;
-
-	constructor(opts: HeartbeatWatcherOptions) {
-		this.opts = opts;
-	}
-
-	start(): void {
-		this.dispose();
-		this.scheduleTick();
-	}
-
-	private scheduleTick(): void {
-		this.timer = setTimeout(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
-		this.timer.unref();
-	}
-
-	tick(now = Date.now()): void {
-		try {
-			this.tickUnsafe(now);
-		} catch (error) {
-			logInternalError("heartbeat-watcher.tick", error);
-		} finally {
-			this.scheduleTick();
-		}
-	}
-
-	private tickUnsafe(now: number): void {
-		const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
-		const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
-		const activeKeys = new Set<string>();
-
-		for (const run of this.opts.manifestCache.list(50)) {
-			if (run.status !== "running") continue;
-			const loaded = loadRunManifestById(this.opts.cwd, run.runId);
-			if (!loaded) continue;
-			for (const task of loaded.tasks) {
-				if (task.status !== "running") continue;
-				const key = `${run.runId}:${task.id}`;
-				activeKeys.add(key);
-				this.lastSeen.set(key, now);
-
-				const elapsed = heartbeatAgeMs(task.heartbeat, now);
-				const level = classifyHeartbeat(task.heartbeat, thresholds, now);
-				this.opts.registry.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
-				this.opts.registry.counter("crew.heartbeat.level_total", "Heartbeat classifications by level").inc({ runId: run.runId, level });
-				const previous = this.lastLevel.get(key);
-				this.lastLevel.set(key, level);
-				if (level === "dead" && previous !== "dead") {
-					this.opts.registry.counter("crew.heartbeat.dead_total", "Dead heartbeat detections").inc({ runId: run.runId });
-					appendEvent(loaded.manifest.eventsPath, { type: "crew.task.heartbeat_dead", runId: run.runId, taskId: task.id, message: `Task ${task.id} heartbeat dead.`, data: { elapsedMs: Number.isFinite(elapsed) ? elapsed : undefined } });
-					this.opts.router.enqueue({ id: `dead_${run.runId}_${task.id}`, severity: "warning", source: "heartbeat-watcher", runId: run.runId, title: `Task ${task.id} heartbeat dead`, body: "Background watcher detected a stuck worker." });
-					this.opts.onDead?.(run.runId, task.id, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
-				}
-				if (level === "dead") {
-					const count = (this.consecutiveDead.get(key) ?? 0) + 1;
-					this.consecutiveDead.set(key, count);
-					if (count === tickThreshold) this.opts.onDeadletterTrigger?.(loaded.manifest, task.id);
-				} else {
-					this.consecutiveDead.delete(key);
-				}
-			}
-		}
-
-		// Cleanup: drop keys that were NOT in this tick's active set AND
-		// haven't been seen for > maxKeyAgeMs.  This covers runs that
-		// completed or fell out of the manifest cache's top-50 window.
-		const cutoff = now - this.maxKeyAgeMs;
-		for (const [key, ts] of this.lastSeen) {
-			if (!activeKeys.has(key) && ts < cutoff) {
-				this.lastLevel.delete(key);
-				this.consecutiveDead.delete(key);
-				this.lastSeen.delete(key);
-			}
-		}
-	}
-
-	dispose(): void {
-		if (this.timer) clearTimeout(this.timer);
-		this.timer = undefined;
-		this.lastLevel.clear();
-		this.consecutiveDead.clear();
-		this.lastSeen.clear();
-	}
-}
+import type { NotificationDescriptor } from "../extension/notification-router.ts";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { loadRunManifestById } from "../state/state-store.ts";
+import type { TeamRunManifest } from "../state/types.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import type { ManifestCache } from "./manifest-cache.ts";
+import { classifyHeartbeat, DEFAULT_GRADIENT_THRESHOLDS, heartbeatAgeMs, type GradientThresholds, type HeartbeatLevel } from "./heartbeat-gradient.ts";
+
+export interface HeartbeatWatcherRouter {
+	enqueue(notification: NotificationDescriptor): boolean;
+}
+
+export interface HeartbeatWatcherOptions {
+	cwd: string;
+	pollIntervalMs?: number;
+	thresholds?: GradientThresholds;
+	manifestCache: ManifestCache;
+	registry: MetricRegistry;
+	router: HeartbeatWatcherRouter;
+	deadletterTickThreshold?: number;
+	onDead?: (runId: string, taskId: string, elapsed: number) => void;
+	onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
+}
+
+/**
+ * Polls running runs for heartbeat staleness.
+ *
+ * Uses recursive setTimeout to avoid timer storms.
+ * Cleanup is done in the same pass — no second scan over manifests.
+ * Keys for runs that disappear from the cache are cleaned via staleness-age policy
+ * rather than being leaked forever.
+ */
+export class HeartbeatWatcher {
+	private timer?: ReturnType<typeof setTimeout>;
+	private lastLevel = new Map<string, HeartbeatLevel>();
+	private consecutiveDead = new Map<string, number>();
+	private lastSeen = new Map<string, number>(); // key → last time it was active
+	/** Max age (ms) to retain a stale key before garbage-collecting it. */
+	private readonly maxKeyAgeMs = 600_000; // 10 minutes
+	private readonly opts: HeartbeatWatcherOptions;
+
+	constructor(opts: HeartbeatWatcherOptions) {
+		this.opts = opts;
+	}
+
+	start(): void {
+		this.dispose();
+		this.scheduleTick();
+	}
+
+	private scheduleTick(): void {
+		this.timer = setTimeout(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
+		this.timer.unref();
+	}
+
+	tick(now = Date.now()): void {
+		try {
+			this.tickUnsafe(now);
+		} catch (error) {
+			logInternalError("heartbeat-watcher.tick", error);
+		} finally {
+			this.scheduleTick();
+		}
+	}
+
+	private tickUnsafe(now: number): void {
+		const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
+		const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
+		const activeKeys = new Set<string>();
+
+		for (const run of this.opts.manifestCache.list(50)) {
+			if (run.status !== "running") continue;
+			const loaded = loadRunManifestById(this.opts.cwd, run.runId);
+			if (!loaded) continue;
+			for (const task of loaded.tasks) {
+				if (task.status !== "running") continue;
+				const key = `${run.runId}:${task.id}`;
+				activeKeys.add(key);
+				this.lastSeen.set(key, now);
+
+				const elapsed = heartbeatAgeMs(task.heartbeat, now);
+				const level = classifyHeartbeat(task.heartbeat, thresholds, now);
+				this.opts.registry.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				this.opts.registry.counter("crew.heartbeat.level_total", "Heartbeat classifications by level").inc({ runId: run.runId, level });
+				const previous = this.lastLevel.get(key);
+				this.lastLevel.set(key, level);
+				if (level === "dead" && previous !== "dead") {
+					this.opts.registry.counter("crew.heartbeat.dead_total", "Dead heartbeat detections").inc({ runId: run.runId });
+					appendEvent(loaded.manifest.eventsPath, { type: "crew.task.heartbeat_dead", runId: run.runId, taskId: task.id, message: `Task ${task.id} heartbeat dead.`, data: { elapsedMs: Number.isFinite(elapsed) ? elapsed : undefined } });
+					this.opts.router.enqueue({ id: `dead_${run.runId}_${task.id}`, severity: "warning", source: "heartbeat-watcher", runId: run.runId, title: `Task ${task.id} heartbeat dead`, body: "Background watcher detected a stuck worker." });
+					this.opts.onDead?.(run.runId, task.id, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				}
+				if (level === "dead") {
+					const count = (this.consecutiveDead.get(key) ?? 0) + 1;
+					this.consecutiveDead.set(key, count);
+					if (count === tickThreshold) this.opts.onDeadletterTrigger?.(loaded.manifest, task.id);
+				} else {
+					this.consecutiveDead.delete(key);
+				}
+			}
+		}
+
+		// Cleanup: drop keys that were NOT in this tick's active set AND
+		// haven't been seen for > maxKeyAgeMs.  This covers runs that
+		// completed or fell out of the manifest cache's top-50 window.
+		const cutoff = now - this.maxKeyAgeMs;
+		for (const [key, ts] of this.lastSeen) {
+			if (!activeKeys.has(key) && ts < cutoff) {
+				this.lastLevel.delete(key);
+				this.consecutiveDead.delete(key);
+				this.lastSeen.delete(key);
+			}
+		}
+	}
+
+	dispose(): void {
+		if (this.timer) clearTimeout(this.timer);
+		this.timer = undefined;
+		this.lastLevel.clear();
+		this.consecutiveDead.clear();
+		this.lastSeen.clear();
+	}
+}