pi-crew 0.1.41 → 0.1.43

package/src/runtime/foreground-control.ts

@@ -1,82 +1,82 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { appendEvent } from "../state/event-log.ts";
-import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
-import { checkProcessLiveness, isActiveRunStatus } from "./process-status.ts";
-import { readCrewAgents } from "./crew-agent-records.ts";
-
-export type ForegroundControlRequestType = "interrupt" | "status";
-
-export interface ForegroundControlStatus {
-	runId: string;
-	status: TeamRunManifest["status"];
-	active: boolean;
-	asyncPid?: number;
-	asyncAlive?: boolean;
-	runningTasks: string[];
-	runningAgents: string[];
-	controlPath: string;
-	lastRequest?: ForegroundControlRequest;
-}
-
-export interface ForegroundControlRequest {
-	id: string;
-	type: ForegroundControlRequestType;
-	createdAt: string;
-	reason: string;
-	acknowledged: boolean;
-}
-
-export function foregroundControlPath(manifest: TeamRunManifest): string {
-	return path.join(manifest.stateRoot, "foreground-control.json");
-}
-
-function readLastRequest(controlPath: string): ForegroundControlRequest | undefined {
-	if (!fs.existsSync(controlPath)) return undefined;
-	try {
-		const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
-		return parsed.requests?.at(-1);
-	} catch {
-		return undefined;
-	}
-}
-
-export function readForegroundControlStatus(manifest: TeamRunManifest, tasks: TeamTaskState[]): ForegroundControlStatus {
-	const controlPath = foregroundControlPath(manifest);
-	const asyncAlive = manifest.async?.pid !== undefined ? checkProcessLiveness(manifest.async.pid).alive : undefined;
-	return {
-		runId: manifest.runId,
-		status: manifest.status,
-		active: isActiveRunStatus(manifest.status),
-		asyncPid: manifest.async?.pid,
-		asyncAlive,
-		runningTasks: tasks.filter((task) => task.status === "running").map((task) => task.id),
-		runningAgents: readCrewAgents(manifest).filter((agent) => agent.status === "running").map((agent) => agent.id),
-		controlPath,
-		lastRequest: readLastRequest(controlPath),
-	};
-}
-
-export function writeForegroundInterruptRequest(manifest: TeamRunManifest, reason = "User requested foreground interrupt."): ForegroundControlRequest {
-	const controlPath = foregroundControlPath(manifest);
-	let requests: ForegroundControlRequest[] = [];
-	if (fs.existsSync(controlPath)) {
-		try {
-			const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
-			requests = Array.isArray(parsed.requests) ? parsed.requests : [];
-		} catch {
-			requests = [];
-		}
-	}
-	const request: ForegroundControlRequest = {
-		id: `fg_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}`,
-		type: "interrupt",
-		createdAt: new Date().toISOString(),
-		reason,
-		acknowledged: false,
-	};
-	fs.mkdirSync(path.dirname(controlPath), { recursive: true });
-	fs.writeFileSync(controlPath, `${JSON.stringify({ requests: [...requests, request] }, null, 2)}\n`, "utf-8");
-	appendEvent(manifest.eventsPath, { type: "foreground.interrupt_requested", runId: manifest.runId, message: reason, data: { requestId: request.id, controlPath } });
-	return request;
-}
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { appendEvent } from "../state/event-log.ts";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { checkProcessLiveness, isActiveRunStatus } from "./process-status.ts";
+import { readCrewAgents } from "./crew-agent-records.ts";
+
+export type ForegroundControlRequestType = "interrupt" | "status";
+
+export interface ForegroundControlStatus {
+	runId: string;
+	status: TeamRunManifest["status"];
+	active: boolean;
+	asyncPid?: number;
+	asyncAlive?: boolean;
+	runningTasks: string[];
+	runningAgents: string[];
+	controlPath: string;
+	lastRequest?: ForegroundControlRequest;
+}
+
+export interface ForegroundControlRequest {
+	id: string;
+	type: ForegroundControlRequestType;
+	createdAt: string;
+	reason: string;
+	acknowledged: boolean;
+}
+
+export function foregroundControlPath(manifest: TeamRunManifest): string {
+	return path.join(manifest.stateRoot, "foreground-control.json");
+}
+
+function readLastRequest(controlPath: string): ForegroundControlRequest | undefined {
+	if (!fs.existsSync(controlPath)) return undefined;
+	try {
+		const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
+		return parsed.requests?.at(-1);
+	} catch {
+		return undefined;
+	}
+}
+
+export function readForegroundControlStatus(manifest: TeamRunManifest, tasks: TeamTaskState[]): ForegroundControlStatus {
+	const controlPath = foregroundControlPath(manifest);
+	const asyncAlive = manifest.async?.pid !== undefined ? checkProcessLiveness(manifest.async.pid).alive : undefined;
+	return {
+		runId: manifest.runId,
+		status: manifest.status,
+		active: isActiveRunStatus(manifest.status),
+		asyncPid: manifest.async?.pid,
+		asyncAlive,
+		runningTasks: tasks.filter((task) => task.status === "running").map((task) => task.id),
+		runningAgents: readCrewAgents(manifest).filter((agent) => agent.status === "running").map((agent) => agent.id),
+		controlPath,
+		lastRequest: readLastRequest(controlPath),
+	};
+}
+
+export function writeForegroundInterruptRequest(manifest: TeamRunManifest, reason = "User requested foreground interrupt."): ForegroundControlRequest {
+	const controlPath = foregroundControlPath(manifest);
+	let requests: ForegroundControlRequest[] = [];
+	if (fs.existsSync(controlPath)) {
+		try {
+			const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
+			requests = Array.isArray(parsed.requests) ? parsed.requests : [];
+		} catch {
+			requests = [];
+		}
+	}
+	const request: ForegroundControlRequest = {
+		id: `fg_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}`,
+		type: "interrupt",
+		createdAt: new Date().toISOString(),
+		reason,
+		acknowledged: false,
+	};
+	fs.mkdirSync(path.dirname(controlPath), { recursive: true });
+	fs.writeFileSync(controlPath, `${JSON.stringify({ requests: [...requests, request] }, null, 2)}\n`, "utf-8");
+	appendEvent(manifest.eventsPath, { type: "foreground.interrupt_requested", runId: manifest.runId, message: reason, data: { requestId: request.id, controlPath } });
+	return request;
+}

package/src/runtime/green-contract.ts

@@ -1,46 +1,46 @@
-import type { GreenLevel, VerificationContract, VerificationEvidence } from "../state/types.ts";
-
-const GREEN_ORDER: Record<GreenLevel, number> = {
-	none: 0,
-	targeted: 1,
-	package: 2,
-	workspace: 3,
-	merge_ready: 4,
-};
-
-export interface GreenContractOutcome {
-	requiredGreenLevel: GreenLevel;
-	observedGreenLevel: GreenLevel;
-	satisfied: boolean;
-}
-
-export function greenLevelSatisfies(observed: GreenLevel, required: GreenLevel): boolean {
-	return GREEN_ORDER[observed] >= GREEN_ORDER[required];
-}
-
-export function evaluateGreenContract(contract: VerificationContract, evidence?: VerificationEvidence): GreenContractOutcome {
-	const observedGreenLevel = evidence?.observedGreenLevel ?? "none";
-	return {
-		requiredGreenLevel: contract.requiredGreenLevel,
-		observedGreenLevel,
-		satisfied: greenLevelSatisfies(observedGreenLevel, contract.requiredGreenLevel),
-	};
-}
-
-export function inferGreenLevelFromTask(success: boolean, contract: VerificationContract): GreenLevel {
-	if (!success) return "none";
-	if (contract.requiredGreenLevel === "none") return "none";
-	return contract.allowManualEvidence ? contract.requiredGreenLevel : "targeted";
-}
-
-export function createVerificationEvidence(contract: VerificationContract, success: boolean, notes: string): VerificationEvidence {
-	const observedGreenLevel = inferGreenLevelFromTask(success, contract);
-	const outcome = evaluateGreenContract(contract, { requiredGreenLevel: contract.requiredGreenLevel, observedGreenLevel, satisfied: false, commands: [], notes });
-	return {
-		requiredGreenLevel: contract.requiredGreenLevel,
-		observedGreenLevel,
-		satisfied: outcome.satisfied,
-		commands: contract.commands.map((cmd) => ({ cmd, status: "not_run" as const })),
-		notes,
-	};
-}
+import type { GreenLevel, VerificationContract, VerificationEvidence } from "../state/types.ts";
+
+const GREEN_ORDER: Record<GreenLevel, number> = {
+	none: 0,
+	targeted: 1,
+	package: 2,
+	workspace: 3,
+	merge_ready: 4,
+};
+
+export interface GreenContractOutcome {
+	requiredGreenLevel: GreenLevel;
+	observedGreenLevel: GreenLevel;
+	satisfied: boolean;
+}
+
+export function greenLevelSatisfies(observed: GreenLevel, required: GreenLevel): boolean {
+	return GREEN_ORDER[observed] >= GREEN_ORDER[required];
+}
+
+export function evaluateGreenContract(contract: VerificationContract, evidence?: VerificationEvidence): GreenContractOutcome {
+	const observedGreenLevel = evidence?.observedGreenLevel ?? "none";
+	return {
+		requiredGreenLevel: contract.requiredGreenLevel,
+		observedGreenLevel,
+		satisfied: greenLevelSatisfies(observedGreenLevel, contract.requiredGreenLevel),
+	};
+}
+
+export function inferGreenLevelFromTask(success: boolean, contract: VerificationContract): GreenLevel {
+	if (!success) return "none";
+	if (contract.requiredGreenLevel === "none") return "none";
+	return contract.allowManualEvidence ? contract.requiredGreenLevel : "targeted";
+}
+
+export function createVerificationEvidence(contract: VerificationContract, success: boolean, notes: string): VerificationEvidence {
+	const observedGreenLevel = inferGreenLevelFromTask(success, contract);
+	const outcome = evaluateGreenContract(contract, { requiredGreenLevel: contract.requiredGreenLevel, observedGreenLevel, satisfied: false, commands: [], notes });
+	return {
+		requiredGreenLevel: contract.requiredGreenLevel,
+		observedGreenLevel,
+		satisfied: outcome.satisfied,
+		commands: contract.commands.map((cmd) => ({ cmd, status: "not_run" as const })),
+		notes,
+	};
+}

package/src/runtime/group-join.ts

@@ -1,106 +1,106 @@
-import type { CrewRuntimeConfig } from "../config/config.ts";
-import { writeArtifact } from "../state/artifact-store.ts";
-import { appendEvent } from "../state/event-log.ts";
-import { appendMailboxMessage, findMailboxMessageByRequestId, readDeliveryState } from "../state/mailbox.ts";
-import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../state/types.ts";
-import { aggregateTaskOutputs } from "./task-output-context.ts";
-
-export type CrewGroupJoinMode = "off" | "group" | "smart";
-
-export interface CrewGroupJoinDelivery {
-	batchId: string;
-	mode: CrewGroupJoinMode;
-	partial: boolean;
-	taskIds: string[];
-	completed: string[];
-	failed: string[];
-	skipped: string[];
-	remaining: string[];
-	artifact?: ArtifactDescriptor;
-	messageId?: string;
-	requestId?: string;
-	ackRequired?: boolean;
-	ackStatus?: "pending" | "acknowledged";
-}
-
-export function resolveGroupJoinMode(runtime?: CrewRuntimeConfig): CrewGroupJoinMode {
-	return runtime?.groupJoin ?? "smart";
-}
-
-export function shouldGroupJoin(mode: CrewGroupJoinMode, batch: TeamTaskState[]): boolean {
-	if (mode === "off") return false;
-	if (mode === "group") return batch.length > 0;
-	return batch.length > 1;
-}
-
-function batchIdFor(runId: string, taskIds: string[]): string {
-	return `${runId}_${taskIds.join("+").replace(/[^a-zA-Z0-9_+-]/g, "_")}`;
-}
-
-function requestIdFor(runId: string, batchId: string, partial: boolean): string {
-	return `${runId}:group-join:${partial ? "partial" : "completed"}:${batchId}`;
-}
-
-function statusList(tasks: TeamTaskState[], status: TeamTaskState["status"]): string[] {
-	return tasks.filter((task) => task.status === status).map((task) => task.id);
-}
-
-export function deliverGroupJoin(input: {
-	manifest: TeamRunManifest;
-	mode: CrewGroupJoinMode;
-	batch: TeamTaskState[];
-	allTasks: TeamTaskState[];
-	partial?: boolean;
-}): CrewGroupJoinDelivery | undefined {
-	if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
-	const taskIds = input.batch.map((task) => task.id);
-	const latest = taskIds.map((id) => input.allTasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
-	const completed = statusList(latest, "completed");
-	const failed = statusList(latest, "failed");
-	const skipped = statusList(latest, "skipped");
-	const remaining = latest.filter((task) => task.status === "queued" || task.status === "running").map((task) => task.id);
-	const partial = input.partial ?? remaining.length > 0;
-	const batchId = batchIdFor(input.manifest.runId, taskIds);
-	const summary = aggregateTaskOutputs(latest, input.manifest);
-	const requestId = requestIdFor(input.manifest.runId, batchId, partial);
-	const existingMailbox = findMailboxMessageByRequestId(input.manifest, requestId);
-	const existingStatus = existingMailbox ? readDeliveryState(input.manifest).messages[existingMailbox.id] ?? existingMailbox.status : undefined;
-	const delivery: CrewGroupJoinDelivery = { batchId, mode: input.mode, partial, taskIds, completed, failed, skipped, remaining, requestId, ackRequired: true, ackStatus: existingStatus === "acknowledged" ? "acknowledged" : "pending" };
-	const content = `${JSON.stringify({ ...delivery, createdAt: new Date().toISOString() }, null, 2)}\n`;
-	const artifact = writeArtifact(input.manifest.artifactsRoot, {
-		kind: "metadata",
-		relativePath: `metadata/group-joins/${batchId}.json`,
-		producer: "group-join",
-		content,
-	});
-	const mailbox = existingMailbox ?? appendMailboxMessage(input.manifest, {
-		direction: "outbox",
-		from: "group-join",
-		to: "leader",
-		body: [
-			`Group join ${partial ? "partial" : "completed"}: ${taskIds.join(", ")}`,
-			`Request: ${requestId}`,
-			`Completed: ${completed.join(", ") || "none"}`,
-			`Failed: ${failed.join(", ") || "none"}`,
-			`Skipped: ${skipped.join(", ") || "none"}`,
-			`Remaining: ${remaining.join(", ") || "none"}`,
-			"",
-			summary,
-		].join("\n"),
-		status: "delivered",
-		data: { kind: "group_join", requestId, batchId, partial, ackRequired: true, taskIds, completed, failed, skipped, remaining },
-	});
-	appendEvent(input.manifest.eventsPath, {
-		type: partial ? "agent.group_join.partial" : "agent.group_join.completed",
-		runId: input.manifest.runId,
-		message: `Group join ${partial ? "partial" : "completed"} for ${taskIds.length} task(s).`,
-		data: { ...delivery, artifactPath: artifact.path, messageId: mailbox.id, fallback: "mailbox-delivered", reused: Boolean(existingMailbox) },
-	});
-	if (existingMailbox) appendEvent(input.manifest.eventsPath, {
-		type: "agent.group_join.delivery_reused",
-		runId: input.manifest.runId,
-		message: `Reused group join mailbox delivery for ${taskIds.length} task(s).`,
-		data: { requestId, messageId: mailbox.id, batchId, partial },
-	});
-	return { ...delivery, artifact, messageId: mailbox.id };
-}
+import type { CrewRuntimeConfig } from "../config/config.ts";
+import { writeArtifact } from "../state/artifact-store.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { appendMailboxMessage, findMailboxMessageByRequestId, readDeliveryState } from "../state/mailbox.ts";
+import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { aggregateTaskOutputs } from "./task-output-context.ts";
+
+export type CrewGroupJoinMode = "off" | "group" | "smart";
+
+export interface CrewGroupJoinDelivery {
+	batchId: string;
+	mode: CrewGroupJoinMode;
+	partial: boolean;
+	taskIds: string[];
+	completed: string[];
+	failed: string[];
+	skipped: string[];
+	remaining: string[];
+	artifact?: ArtifactDescriptor;
+	messageId?: string;
+	requestId?: string;
+	ackRequired?: boolean;
+	ackStatus?: "pending" | "acknowledged";
+}
+
+export function resolveGroupJoinMode(runtime?: CrewRuntimeConfig): CrewGroupJoinMode {
+	return runtime?.groupJoin ?? "smart";
+}
+
+export function shouldGroupJoin(mode: CrewGroupJoinMode, batch: TeamTaskState[]): boolean {
+	if (mode === "off") return false;
+	if (mode === "group") return batch.length > 0;
+	return batch.length > 1;
+}
+
+function batchIdFor(runId: string, taskIds: string[]): string {
+	return `${runId}_${taskIds.join("+").replace(/[^a-zA-Z0-9_+-]/g, "_")}`;
+}
+
+function requestIdFor(runId: string, batchId: string, partial: boolean): string {
+	return `${runId}:group-join:${partial ? "partial" : "completed"}:${batchId}`;
+}
+
+function statusList(tasks: TeamTaskState[], status: TeamTaskState["status"]): string[] {
+	return tasks.filter((task) => task.status === status).map((task) => task.id);
+}
+
+export function deliverGroupJoin(input: {
+	manifest: TeamRunManifest;
+	mode: CrewGroupJoinMode;
+	batch: TeamTaskState[];
+	allTasks: TeamTaskState[];
+	partial?: boolean;
+}): CrewGroupJoinDelivery | undefined {
+	if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
+	const taskIds = input.batch.map((task) => task.id);
+	const latest = taskIds.map((id) => input.allTasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
+	const completed = statusList(latest, "completed");
+	const failed = statusList(latest, "failed");
+	const skipped = statusList(latest, "skipped");
+	const remaining = latest.filter((task) => task.status === "queued" || task.status === "running").map((task) => task.id);
+	const partial = input.partial ?? remaining.length > 0;
+	const batchId = batchIdFor(input.manifest.runId, taskIds);
+	const summary = aggregateTaskOutputs(latest, input.manifest);
+	const requestId = requestIdFor(input.manifest.runId, batchId, partial);
+	const existingMailbox = findMailboxMessageByRequestId(input.manifest, requestId);
+	const existingStatus = existingMailbox ? readDeliveryState(input.manifest).messages[existingMailbox.id] ?? existingMailbox.status : undefined;
+	const delivery: CrewGroupJoinDelivery = { batchId, mode: input.mode, partial, taskIds, completed, failed, skipped, remaining, requestId, ackRequired: true, ackStatus: existingStatus === "acknowledged" ? "acknowledged" : "pending" };
+	const content = `${JSON.stringify({ ...delivery, createdAt: new Date().toISOString() }, null, 2)}\n`;
+	const artifact = writeArtifact(input.manifest.artifactsRoot, {
+		kind: "metadata",
+		relativePath: `metadata/group-joins/${batchId}.json`,
+		producer: "group-join",
+		content,
+	});
+	const mailbox = existingMailbox ?? appendMailboxMessage(input.manifest, {
+		direction: "outbox",
+		from: "group-join",
+		to: "leader",
+		body: [
+			`Group join ${partial ? "partial" : "completed"}: ${taskIds.join(", ")}`,
+			`Request: ${requestId}`,
+			`Completed: ${completed.join(", ") || "none"}`,
+			`Failed: ${failed.join(", ") || "none"}`,
+			`Skipped: ${skipped.join(", ") || "none"}`,
+			`Remaining: ${remaining.join(", ") || "none"}`,
+			"",
+			summary,
+		].join("\n"),
+		status: "delivered",
+		data: { kind: "group_join", requestId, batchId, partial, ackRequired: true, taskIds, completed, failed, skipped, remaining },
+	});
+	appendEvent(input.manifest.eventsPath, {
+		type: partial ? "agent.group_join.partial" : "agent.group_join.completed",
+		runId: input.manifest.runId,
+		message: `Group join ${partial ? "partial" : "completed"} for ${taskIds.length} task(s).`,
+		data: { ...delivery, artifactPath: artifact.path, messageId: mailbox.id, fallback: "mailbox-delivered", reused: Boolean(existingMailbox) },
+	});
+	if (existingMailbox) appendEvent(input.manifest.eventsPath, {
+		type: "agent.group_join.delivery_reused",
+		runId: input.manifest.runId,
+		message: `Reused group join mailbox delivery for ${taskIds.length} task(s).`,
+		data: { requestId, messageId: mailbox.id, batchId, partial },
+	});
+	return { ...delivery, artifact, messageId: mailbox.id };
+}

package/src/runtime/heartbeat-gradient.ts

@@ -1,28 +1,28 @@
-import type { WorkerHeartbeatState } from "./worker-heartbeat.ts";
-
-export type HeartbeatLevel = "healthy" | "warn" | "stale" | "dead";
-
-export interface GradientThresholds {
-	warnMs: number;
-	staleMs: number;
-	deadMs: number;
-}
-
-export const DEFAULT_GRADIENT_THRESHOLDS: GradientThresholds = { warnMs: 30_000, staleMs: 60_000, deadMs: 300_000 };
-
-export function heartbeatAgeMs(heartbeat: WorkerHeartbeatState | undefined, now = Date.now()): number {
-	if (!heartbeat) return Number.POSITIVE_INFINITY;
-	const lastSeen = Date.parse(heartbeat.lastSeenAt);
-	return Number.isFinite(lastSeen) ? Math.max(0, now - lastSeen) : Number.POSITIVE_INFINITY;
-}
-
-export function classifyHeartbeat(heartbeat: WorkerHeartbeatState | undefined, thresholds: GradientThresholds = DEFAULT_GRADIENT_THRESHOLDS, now = Date.now()): HeartbeatLevel {
-	if (!heartbeat) return "dead";
-	if (heartbeat.alive === false) return "dead";
-	const elapsed = heartbeatAgeMs(heartbeat, now);
-	if (!Number.isFinite(elapsed)) return "dead";
-	if (elapsed > thresholds.deadMs) return "dead";
-	if (elapsed > thresholds.staleMs) return "stale";
-	if (elapsed > thresholds.warnMs) return "warn";
-	return "healthy";
-}
+import type { WorkerHeartbeatState } from "./worker-heartbeat.ts";
+
+export type HeartbeatLevel = "healthy" | "warn" | "stale" | "dead";
+
+export interface GradientThresholds {
+	warnMs: number;
+	staleMs: number;
+	deadMs: number;
+}
+
+export const DEFAULT_GRADIENT_THRESHOLDS: GradientThresholds = { warnMs: 30_000, staleMs: 60_000, deadMs: 300_000 };
+
+export function heartbeatAgeMs(heartbeat: WorkerHeartbeatState | undefined, now = Date.now()): number {
+	if (!heartbeat) return Number.POSITIVE_INFINITY;
+	const lastSeen = Date.parse(heartbeat.lastSeenAt);
+	return Number.isFinite(lastSeen) ? Math.max(0, now - lastSeen) : Number.POSITIVE_INFINITY;
+}
+
+export function classifyHeartbeat(heartbeat: WorkerHeartbeatState | undefined, thresholds: GradientThresholds = DEFAULT_GRADIENT_THRESHOLDS, now = Date.now()): HeartbeatLevel {
+	if (!heartbeat) return "dead";
+	if (heartbeat.alive === false) return "dead";
+	const elapsed = heartbeatAgeMs(heartbeat, now);
+	if (!Number.isFinite(elapsed)) return "dead";
+	if (elapsed > thresholds.deadMs) return "dead";
+	if (elapsed > thresholds.staleMs) return "stale";
+	if (elapsed > thresholds.warnMs) return "warn";
+	return "healthy";
+}

package/src/runtime/heartbeat-watcher.ts

@@ -3,6 +3,7 @@ import type { MetricRegistry } from "../observability/metric-registry.ts";
 import { appendEvent } from "../state/event-log.ts";
 import { loadRunManifestById } from "../state/state-store.ts";
 import type { TeamRunManifest } from "../state/types.ts";
+import { logInternalError } from "../utils/internal-error.ts";
 import type { ManifestCache } from "./manifest-cache.ts";
 import { classifyHeartbeat, DEFAULT_GRADIENT_THRESHOLDS, heartbeatAgeMs, type GradientThresholds, type HeartbeatLevel } from "./heartbeat-gradient.ts";
 
@@ -22,10 +23,21 @@ export interface HeartbeatWatcherOptions {
 	onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
 }
 
+/**
+ * Polls running runs for heartbeat staleness.
+ *
+ * Uses recursive setTimeout to avoid timer storms.
+ * Cleanup is done in the same pass — no second scan over manifests.
+ * Keys for runs that disappear from the cache are cleaned via staleness-age policy
+ * rather than being leaked forever.
+ */
 export class HeartbeatWatcher {
-	private timer?: ReturnType<typeof setInterval>;
+	private timer?: ReturnType<typeof setTimeout>;
 	private lastLevel = new Map<string, HeartbeatLevel>();
 	private consecutiveDead = new Map<string, number>();
+	private lastSeen = new Map<string, number>(); // key → last time it was active
+	/** Max age (ms) to retain a stale key before garbage-collecting it. */
+	private readonly maxKeyAgeMs = 600_000; // 10 minutes
 	private readonly opts: HeartbeatWatcherOptions;
 
 	constructor(opts: HeartbeatWatcherOptions) {
@@ -34,13 +46,29 @@ export class HeartbeatWatcher {
 
 	start(): void {
 		this.dispose();
-		this.timer = setInterval(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
-		this.timer.unref?.();
+		this.scheduleTick();
+	}
+
+	private scheduleTick(): void {
+		this.timer = setTimeout(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
+		this.timer.unref();
 	}
 
 	tick(now = Date.now()): void {
+		try {
+			this.tickUnsafe(now);
+		} catch (error) {
+			logInternalError("heartbeat-watcher.tick", error);
+		} finally {
+			this.scheduleTick();
+		}
+	}
+
+	private tickUnsafe(now: number): void {
 		const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
 		const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
+		const activeKeys = new Set<string>();
+
 		for (const run of this.opts.manifestCache.list(50)) {
 			if (run.status !== "running") continue;
 			const loaded = loadRunManifestById(this.opts.cwd, run.runId);
@@ -48,6 +76,9 @@ export class HeartbeatWatcher {
 			for (const task of loaded.tasks) {
 				if (task.status !== "running") continue;
 				const key = `${run.runId}:${task.id}`;
+				activeKeys.add(key);
+				this.lastSeen.set(key, now);
+
 				const elapsed = heartbeatAgeMs(task.heartbeat, now);
 				const level = classifyHeartbeat(task.heartbeat, thresholds, now);
 				this.opts.registry.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
@@ -69,12 +100,25 @@ export class HeartbeatWatcher {
 				}
 			}
 		}
+
+		// Cleanup: drop keys that were NOT in this tick's active set AND
+		// haven't been seen for > maxKeyAgeMs.  This covers runs that
+		// completed or fell out of the manifest cache's top-50 window.
+		const cutoff = now - this.maxKeyAgeMs;
+		for (const [key, ts] of this.lastSeen) {
+			if (!activeKeys.has(key) && ts < cutoff) {
+				this.lastLevel.delete(key);
+				this.consecutiveDead.delete(key);
+				this.lastSeen.delete(key);
+			}
+		}
 	}
 
 	dispose(): void {
-		if (this.timer) clearInterval(this.timer);
+		if (this.timer) clearTimeout(this.timer);
 		this.timer = undefined;
 		this.lastLevel.clear();
 		this.consecutiveDead.clear();
+		this.lastSeen.clear();
 	}
 }