pi-crew 0.1.35 → 0.1.37

package/src/runtime/crash-recovery.ts

@@ -0,0 +1,56 @@
+import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { appendEvent, scanSequence } from "../state/event-log.ts";
+import { loadRunManifestById, saveRunTasks, updateRunStatus } from "../state/state-store.ts";
+import type { TeamTaskState } from "../state/types.ts";
+import { isWorkerHeartbeatStale } from "./worker-heartbeat.ts";
+import type { ManifestCache } from "./manifest-cache.ts";
+import { checkProcessLiveness } from "./process-status.ts";
+
+export interface RecoveryPlan {
+	runId: string;
+	resumableTasks: string[];
+	preservedTasks: string[];
+	lastEventSeq: number;
+}
+
+function isTerminalTask(task: TeamTaskState): boolean {
+	return task.status === "completed" || task.status === "failed" || task.status === "cancelled" || task.status === "skipped";
+}
+
+function shouldRecoverTask(task: TeamTaskState, deadMs: number): boolean {
+	if (task.status !== "running") return false;
+	if (!task.heartbeat) return true;
+	return task.heartbeat.alive === false || isWorkerHeartbeatStale(task.heartbeat, deadMs);
+}
+
+export function detectInterruptedRuns(cwd: string, manifestCache: ManifestCache, deadMs = 300_000): RecoveryPlan[] {
+	const plans: RecoveryPlan[] = [];
+	for (const manifest of manifestCache.list(50)) {
+		if (manifest.status !== "running") continue;
+		if (manifest.async?.pid !== undefined && checkProcessLiveness(manifest.async.pid).alive) continue;
+		const loaded = loadRunManifestById(cwd, manifest.runId);
+		if (!loaded) continue;
+		const resumableTasks = loaded.tasks.filter((task) => shouldRecoverTask(task, deadMs)).map((task) => task.id);
+		if (!resumableTasks.length) continue;
+		plans.push({ runId: manifest.runId, resumableTasks, preservedTasks: loaded.tasks.filter(isTerminalTask).map((task) => task.id), lastEventSeq: scanSequence(loaded.manifest.eventsPath) });
+	}
+	return plans;
+}
+
+export async function applyRecoveryPlan(plan: RecoveryPlan, ctx: Pick<ExtensionContext, "cwd">, registry?: MetricRegistry): Promise<void> {
+	const loaded = loadRunManifestById(ctx.cwd, plan.runId);
+	if (!loaded) throw new Error(`Run '${plan.runId}' not found.`);
+	const reset = new Set(plan.resumableTasks);
+	const tasks = loaded.tasks.map((task) => reset.has(task.id) ? { ...task, status: "queued" as const, startedAt: undefined, finishedAt: undefined, error: undefined, heartbeat: undefined } : task);
+	saveRunTasks(loaded.manifest, tasks);
+	appendEvent(loaded.manifest.eventsPath, { type: "crew.run.resumed", runId: plan.runId, message: `Recovered ${plan.resumableTasks.length} interrupted task(s).`, data: { recoveredFromSeq: plan.lastEventSeq, resumableTasks: plan.resumableTasks } });
+	registry?.counter("crew.run.count", "Total runs by status").inc({ status: "resumed" });
+}
+
+export function declineRecoveryPlan(plan: RecoveryPlan, ctx: Pick<ExtensionContext, "cwd">): void {
+	const loaded = loadRunManifestById(ctx.cwd, plan.runId);
+	if (!loaded) throw new Error(`Run '${plan.runId}' not found.`);
+	updateRunStatus(loaded.manifest, "cancelled", "interrupted-not-resumed");
+	appendEvent(loaded.manifest.eventsPath, { type: "crew.run.recovery_declined", runId: plan.runId, message: "Interrupted run was not resumed.", data: { recoveredFromSeq: plan.lastEventSeq } });
+}

package/src/runtime/deadletter.ts

@@ -0,0 +1,36 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { TeamRunManifest } from "../state/types.ts";
+
+export type DeadletterReason = "max-retries" | "heartbeat-dead" | "manual";
+
+export interface DeadletterEntry {
+	taskId: string;
+	runId: string;
+	reason: DeadletterReason;
+	attempts: number;
+	lastError?: string;
+	timestamp: string;
+}
+
+export function deadletterPath(manifest: TeamRunManifest): string {
+	return path.join(manifest.stateRoot, "deadletter.jsonl");
+}
+
+export function appendDeadletter(manifest: TeamRunManifest, entry: DeadletterEntry): void {
+	fs.mkdirSync(manifest.stateRoot, { recursive: true });
+	fs.appendFileSync(deadletterPath(manifest), `${JSON.stringify(entry)}\n`, "utf-8");
+}
+
+export function readDeadletter(manifest: TeamRunManifest): DeadletterEntry[] {
+	const filePath = deadletterPath(manifest);
+	if (!fs.existsSync(filePath)) return [];
+	return fs.readFileSync(filePath, "utf-8").split(/\r?\n/).filter(Boolean).flatMap((line) => {
+		try {
+			const parsed = JSON.parse(line) as DeadletterEntry;
+			return parsed && typeof parsed.taskId === "string" && typeof parsed.runId === "string" ? [parsed] : [];
+		} catch {
+			return [];
+		}
+	});
+}

package/src/runtime/diagnostic-export.ts

@@ -1,4 +1,6 @@
 import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import type { MetricSnapshot } from "../observability/metrics-primitives.ts";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { readCrewAgents } from "./crew-agent-records.ts";
@@ -9,6 +11,7 @@ import { summarizeHeartbeats, type HeartbeatSummary } from "../ui/heartbeat-aggr
 import type { RunUiSnapshot } from "../ui/snapshot-types.ts";
 
 export interface DiagnosticReport {
+	schemaVersion?: number;
 	runId: string;
 	exportedAt: string;
 	manifest: TeamRunManifest;
@@ -17,6 +20,7 @@ export interface DiagnosticReport {
 	heartbeat: HeartbeatSummary;
 	agents: unknown[];
 	envRedacted: Record<string, string>;
+	metricsSnapshot?: MetricSnapshot[];
 }
 
 const SECRET_KEY_PATTERN = /(token|key|password|secret|credential|auth)/i;
@@ -70,13 +74,15 @@ function buildSnapshot(manifest: TeamRunManifest, tasks: TeamTaskState[]): RunUi
 	};
 }
 
-export async function exportDiagnostic(ctx: Pick<ExtensionContext, "cwd">, runId: string): Promise<{ path: string; report: DiagnosticReport }> {
+export async function exportDiagnostic(ctx: Pick<ExtensionContext, "cwd">, runId: string, options: { registry?: MetricRegistry } = {}): Promise<{ path: string; report: DiagnosticReport }> {
 	const loaded = loadRunManifestById(ctx.cwd, runId);
 	if (!loaded) throw new Error(`Run '${runId}' not found.`);
 	const exportedAt = new Date().toISOString();
 	const safeTimestamp = exportedAt.replace(/[:.]/g, "-");
 	const recentEvents = readEvents(loaded.manifest.eventsPath).slice(-200);
+	const metricsSnapshot = options.registry?.snapshot();
 	const report: DiagnosticReport = {
+		...(metricsSnapshot ? { schemaVersion: 2 } : {}),
 		runId,
 		exportedAt,
 		manifest: redactSecrets(loaded.manifest) as TeamRunManifest,
@@ -85,6 +91,7 @@ export async function exportDiagnostic(ctx: Pick<ExtensionContext, "cwd">, runId
 		heartbeat: summarizeHeartbeats(buildSnapshot(loaded.manifest, loaded.tasks)),
 		agents: redactSecrets(readCrewAgents(loaded.manifest)) as unknown[],
 		envRedacted: envRedacted(),
+		...(metricsSnapshot ? { metricsSnapshot: redactSecrets(metricsSnapshot) as MetricSnapshot[] } : {}),
 	};
 	const dir = path.join(loaded.manifest.artifactsRoot, "diagnostic");
 	fs.mkdirSync(dir, { recursive: true });

package/src/runtime/heartbeat-gradient.ts

@@ -0,0 +1,28 @@
+import type { WorkerHeartbeatState } from "./worker-heartbeat.ts";
+
+export type HeartbeatLevel = "healthy" | "warn" | "stale" | "dead";
+
+export interface GradientThresholds {
+	warnMs: number;
+	staleMs: number;
+	deadMs: number;
+}
+
+export const DEFAULT_GRADIENT_THRESHOLDS: GradientThresholds = { warnMs: 30_000, staleMs: 60_000, deadMs: 300_000 };
+
+export function heartbeatAgeMs(heartbeat: WorkerHeartbeatState | undefined, now = Date.now()): number {
+	if (!heartbeat) return Number.POSITIVE_INFINITY;
+	const lastSeen = Date.parse(heartbeat.lastSeenAt);
+	return Number.isFinite(lastSeen) ? Math.max(0, now - lastSeen) : Number.POSITIVE_INFINITY;
+}
+
+export function classifyHeartbeat(heartbeat: WorkerHeartbeatState | undefined, thresholds: GradientThresholds = DEFAULT_GRADIENT_THRESHOLDS, now = Date.now()): HeartbeatLevel {
+	if (!heartbeat) return "dead";
+	if (heartbeat.alive === false) return "dead";
+	const elapsed = heartbeatAgeMs(heartbeat, now);
+	if (!Number.isFinite(elapsed)) return "dead";
+	if (elapsed > thresholds.deadMs) return "dead";
+	if (elapsed > thresholds.staleMs) return "stale";
+	if (elapsed > thresholds.warnMs) return "warn";
+	return "healthy";
+}

package/src/runtime/heartbeat-watcher.ts

@@ -0,0 +1,80 @@
+import type { NotificationDescriptor } from "../extension/notification-router.ts";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { loadRunManifestById } from "../state/state-store.ts";
+import type { TeamRunManifest } from "../state/types.ts";
+import type { ManifestCache } from "./manifest-cache.ts";
+import { classifyHeartbeat, DEFAULT_GRADIENT_THRESHOLDS, heartbeatAgeMs, type GradientThresholds, type HeartbeatLevel } from "./heartbeat-gradient.ts";
+
+export interface HeartbeatWatcherRouter {
+	enqueue(notification: NotificationDescriptor): boolean;
+}
+
+export interface HeartbeatWatcherOptions {
+	cwd: string;
+	pollIntervalMs?: number;
+	thresholds?: GradientThresholds;
+	manifestCache: ManifestCache;
+	registry: MetricRegistry;
+	router: HeartbeatWatcherRouter;
+	deadletterTickThreshold?: number;
+	onDead?: (runId: string, taskId: string, elapsed: number) => void;
+	onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
+}
+
+export class HeartbeatWatcher {
+	private timer?: ReturnType<typeof setInterval>;
+	private lastLevel = new Map<string, HeartbeatLevel>();
+	private consecutiveDead = new Map<string, number>();
+	private readonly opts: HeartbeatWatcherOptions;
+
+	constructor(opts: HeartbeatWatcherOptions) {
+		this.opts = opts;
+	}
+
+	start(): void {
+		this.dispose();
+		this.timer = setInterval(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
+		this.timer.unref?.();
+	}
+
+	tick(now = Date.now()): void {
+		const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
+		const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
+		for (const run of this.opts.manifestCache.list(50)) {
+			if (run.status !== "running") continue;
+			const loaded = loadRunManifestById(this.opts.cwd, run.runId);
+			if (!loaded) continue;
+			for (const task of loaded.tasks) {
+				if (task.status !== "running") continue;
+				const key = `${run.runId}:${task.id}`;
+				const elapsed = heartbeatAgeMs(task.heartbeat, now);
+				const level = classifyHeartbeat(task.heartbeat, thresholds, now);
+				this.opts.registry.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				this.opts.registry.counter("crew.heartbeat.level_total", "Heartbeat classifications by level").inc({ runId: run.runId, level });
+				const previous = this.lastLevel.get(key);
+				this.lastLevel.set(key, level);
+				if (level === "dead" && previous !== "dead") {
+					this.opts.registry.counter("crew.heartbeat.dead_total", "Dead heartbeat detections").inc({ runId: run.runId });
+					appendEvent(loaded.manifest.eventsPath, { type: "crew.task.heartbeat_dead", runId: run.runId, taskId: task.id, message: `Task ${task.id} heartbeat dead.`, data: { elapsedMs: Number.isFinite(elapsed) ? elapsed : undefined } });
+					this.opts.router.enqueue({ id: `dead_${run.runId}_${task.id}`, severity: "warning", source: "heartbeat-watcher", runId: run.runId, title: `Task ${task.id} heartbeat dead`, body: "Background watcher detected a stuck worker." });
+					this.opts.onDead?.(run.runId, task.id, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				}
+				if (level === "dead") {
+					const count = (this.consecutiveDead.get(key) ?? 0) + 1;
+					this.consecutiveDead.set(key, count);
+					if (count === tickThreshold) this.opts.onDeadletterTrigger?.(loaded.manifest, task.id);
+				} else {
+					this.consecutiveDead.delete(key);
+				}
+			}
+		}
+	}
+
+	dispose(): void {
+		if (this.timer) clearInterval(this.timer);
+		this.timer = undefined;
+		this.lastLevel.clear();
+		this.consecutiveDead.clear();
+	}
+}

package/src/runtime/retry-executor.ts

@@ -0,0 +1,59 @@
+import { sleep } from "../utils/sleep.ts";
+
+export interface RetryPolicy {
+	maxAttempts: number;
+	backoffMs: number;
+	jitterRatio: number;
+	exponentialFactor: number;
+	retryableErrors?: string[];
+}
+
+export interface RetryHooks {
+	onAttemptFailed?: (attempt: number, error: Error, nextDelayMs: number) => void;
+	onRetryGivenUp?: (attempts: number, error: Error) => void;
+	signal?: AbortSignal;
+}
+
+export const DEFAULT_RETRY_POLICY: RetryPolicy = { maxAttempts: 3, backoffMs: 1000, jitterRatio: 0.3, exponentialFactor: 2 };
+
+function asError(error: unknown): Error {
+	return error instanceof Error ? error : new Error(String(error));
+}
+
+function globToRegex(pattern: string): RegExp {
+	const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+	return new RegExp(`^${escaped}$`, "i");
+}
+
+function isRetryable(error: Error, policy: RetryPolicy): boolean {
+	const patterns = policy.retryableErrors ?? [];
+	if (!patterns.length) return true;
+	return patterns.some((pattern) => globToRegex(pattern).test(error.message));
+}
+
+export function calculateRetryDelay(attempt: number, policy: RetryPolicy = DEFAULT_RETRY_POLICY, random = Math.random): number {
+	const base = policy.backoffMs * Math.pow(policy.exponentialFactor, Math.max(0, attempt - 1));
+	const jitter = (random() * 2 - 1) * policy.jitterRatio * base;
+	return Math.max(0, base + jitter);
+}
+
+export async function executeWithRetry<T>(fn: (attempt: number) => Promise<T>, policy: RetryPolicy = DEFAULT_RETRY_POLICY, hooks: RetryHooks = {}): Promise<T> {
+	const normalized: RetryPolicy = { ...DEFAULT_RETRY_POLICY, ...policy, maxAttempts: Math.max(1, policy.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts) };
+	let lastError: Error | undefined;
+	for (let attempt = 1; attempt <= normalized.maxAttempts; attempt += 1) {
+		if (hooks.signal?.aborted) throw new Error("Retry aborted.");
+		try {
+			return await fn(attempt);
+		} catch (error) {
+			lastError = asError(error);
+			if (attempt >= normalized.maxAttempts || !isRetryable(lastError, normalized)) {
+				hooks.onRetryGivenUp?.(attempt, lastError);
+				throw lastError;
+			}
+			const delay = calculateRetryDelay(attempt, normalized);
+			hooks.onAttemptFailed?.(attempt, lastError, delay);
+			await sleep(delay, hooks.signal);
+		}
+	}
+	throw lastError ?? new Error("Retry failed without error.");
+}

package/src/runtime/task-runner.ts

@@ -106,8 +106,16 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
 		const transcriptPath = `${manifest.artifactsRoot}/transcripts/${task.id}.jsonl`;
 		let finalCheckpointWritten = false;
 		let lastAgentRecordPersistedAt = 0;
+		let lastHeartbeatPersistedAt = 0;
 		let lastRunProgressPersistedAt = 0;
 		let lastRunProgressSummary: ProgressEventSummary | undefined;
+		const persistHeartbeat = (force = false): void => {
+			const now = Date.now();
+			if (!force && now - lastHeartbeatPersistedAt < 1000) return;
+			lastHeartbeatPersistedAt = now;
+			task = { ...task, heartbeat: touchWorkerHeartbeat(task.heartbeat ?? createWorkerHeartbeat(task.id)) };
+			tasks = persistSingleTaskUpdate(manifest, tasks, task);
+		};
 		const persistChildProgress = (event: unknown, force = false): void => {
 			const now = Date.now();
 			if (force || shouldFlushProgressEvent(event) || now - lastAgentRecordPersistedAt >= 500) {
@@ -140,9 +148,13 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
 				onSpawn: (pid) => {
 					({ task, tasks } = checkpointTask(manifest, tasks, task, "child-spawned", pid));
 				},
-				onStdoutLine: (line) => appendCrewAgentOutput(manifest, task.id, line),
+				onStdoutLine: (line) => {
+					appendCrewAgentOutput(manifest, task.id, line);
+					persistHeartbeat();
+				},
 				onJsonEvent: (event) => {
 					appendCrewAgentEvent(manifest, task.id, event);
+					persistHeartbeat();
 					task = { ...task, agentProgress: applyAgentProgressEvent(task.agentProgress ?? emptyCrewAgentProgress(), event, task.startedAt) };
 					tasks = updateTask(tasks, task);
 					if (!finalCheckpointWritten && isFinalChildEvent(event)) {
@@ -158,6 +170,7 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
 			finalStderr = childResult.stderr;
 			parsedOutput = parsePiJsonOutput(fs.existsSync(transcriptPath) ? fs.readFileSync(transcriptPath, "utf-8") : childResult.stdout);
 			error = childResult.error || (childResult.exitCode && childResult.exitCode !== 0 ? childResult.stderr || `Child Pi exited with ${childResult.exitCode}` : undefined);
+			persistHeartbeat(true);
 			persistChildProgress({ type: "attempt_finished" }, true);
 			const attempt: ModelAttemptSummary = { model: model ?? "default", success: !error, exitCode, error };
 			modelAttempts.push(attempt);

package/src/runtime/team-runner.ts

@@ -1,11 +1,11 @@
 import * as fs from "node:fs";
 import type { AgentConfig } from "../agents/agent-config.ts";
-import type { CrewLimitsConfig, CrewRuntimeConfig } from "../config/config.ts";
+import type { CrewLimitsConfig, CrewRuntimeConfig, CrewReliabilityConfig } from "../config/config.ts";
 import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
 import { writeArtifact } from "../state/artifact-store.ts";
 import { appendEvent } from "../state/event-log.ts";
 import type { TeamConfig } from "../teams/team-config.ts";
-import type { ArtifactDescriptor, PolicyDecision, TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import type { ArtifactDescriptor, PolicyDecision, TeamRunManifest, TaskAttemptState, TeamTaskState } from "../state/types.ts";
 import { saveRunManifest, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
 import { aggregateUsage, formatUsage } from "../state/usage.ts";
 import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
@@ -18,6 +18,10 @@ import { saveCrewAgents } from "./crew-agent-records.ts";
 import { recordsForMaterializedTasks } from "./task-display.ts";
 import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
 import { runTeamTask } from "./task-runner.ts";
+import { executeWithRetry, DEFAULT_RETRY_POLICY, type RetryPolicy } from "./retry-executor.ts";
+import { appendDeadletter } from "./deadletter.ts";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
 import { resolveBatchConcurrency } from "./concurrency.ts";
 import { mapConcurrent } from "./parallel-utils.ts";
 
@@ -36,6 +40,8 @@ export interface ExecuteTeamRunInput {
 	modelRegistry?: unknown;
 	modelOverride?: string;
 	signal?: AbortSignal;
+	reliability?: CrewReliabilityConfig;
+	metricRegistry?: MetricRegistry;
 }
 
 function findReadyTask(tasks: TeamTaskState[]): TeamTaskState | undefined {
@@ -73,7 +79,7 @@ function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState):
 	// contain stale queued/running copies of tasks that another worker already
 	// completed. Never let those stale snapshots regress durable task state.
 	if (!isNonTerminalTaskStatus(current.status) && isNonTerminalTaskStatus(updated.status)) return false;
-	return updated.status !== current.status || updated.finishedAt !== current.finishedAt || updated.startedAt !== current.startedAt || Boolean(updated.resultArtifact) || Boolean(updated.error) || Boolean(updated.modelAttempts?.length) || Boolean(updated.usage);
+	return updated.status !== current.status || updated.finishedAt !== current.finishedAt || updated.startedAt !== current.startedAt || Boolean(updated.resultArtifact) || Boolean(updated.error) || Boolean(updated.modelAttempts?.length) || Boolean(updated.usage) || Boolean(updated.attempts?.length);
 }
 
 export function __test__mergeTaskUpdates(base: TeamTaskState[], results: Array<{ tasks: TeamTaskState[] }>): TeamTaskState[] {
@@ -384,6 +390,14 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
 	return { ...manifest, updatedAt: new Date().toISOString(), policyDecisions: decisions, artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "metadata" && (artifact.path.endsWith("policy-decisions.json") || artifact.path.endsWith("recovery-ledger.json") || artifact.path.endsWith("branch-freshness.json")))), branchArtifact, policyArtifact, recoveryArtifact] };
 }
 
+function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): RetryPolicy {
+	return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
+}
+
+function failedTaskFrom(result: { tasks: TeamTaskState[] }, taskId: string): TeamTaskState | undefined {
+	return result.tasks.find((item) => item.id === taskId && item.status === "failed");
+}
+
 export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
 	let workflow = input.workflow;
 	let manifest = updateRunStatus(input.manifest, "running", input.executeWorkers ? "Executing team workflow." : "Creating workflow prompts and placeholder results.");
@@ -450,10 +464,48 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
 		const results = await mapConcurrent(
 			readyBatch,
 			concurrency.selectedCount,
-			(task) => {
+			async (task) => {
 				const step = findStep(workflow, task);
 				const agent = findAgent(input.agents, task);
-				return runTeamTask({ manifest, tasks, task, step, agent, signal: input.signal, executeWorkers: input.executeWorkers, runtimeKind: input.runtime?.kind, runtimeConfig: input.runtimeConfig, parentContext: input.parentContext, parentModel: input.parentModel, modelRegistry: input.modelRegistry, modelOverride: input.modelOverride, limits: input.limits });
+				const baseInput = { manifest, tasks, task, step, agent, signal: input.signal, executeWorkers: input.executeWorkers, runtimeKind: input.runtime?.kind, runtimeConfig: input.runtimeConfig, parentContext: input.parentContext, parentModel: input.parentModel, modelRegistry: input.modelRegistry, modelOverride: input.modelOverride, limits: input.limits };
+				if (input.reliability?.autoRetry !== true) return withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask(baseInput));
+				let lastFailed: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
+				const attemptsSoFar: TaskAttemptState[] = [...(task.attempts ?? [])];
+				const policy = retryPolicyFromConfig(input.reliability);
+				try {
+					return await executeWithRetry(async (attempt) => {
+						const startedAt = new Date().toISOString();
+						const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { startedAt }];
+						input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({ runId: manifest.runId, taskId: task.id });
+						const taskWithAttempt: TeamTaskState = { ...task, attempts: inFlightAttempts };
+						const result = await withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask({ ...baseInput, task: taskWithAttempt }));
+						const failed = failedTaskFrom(result, task.id);
+						const endedAt = new Date().toISOString();
+						const finishedAttempt: TaskAttemptState = { startedAt, endedAt, ...(failed?.error ? { error: failed.error } : {}) };
+						attemptsSoFar.push(finishedAttempt);
+						const withAttempt = result.tasks.map((item) => item.id === task.id ? { ...item, attempts: [...attemptsSoFar] } : item);
+						const enriched = { manifest: result.manifest, tasks: withAttempt };
+						if (failed) {
+							lastFailed = enriched;
+							throw new Error(failed.error ?? `Task ${task.id} failed.`);
+						}
+						input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempt - 1));
+						return enriched;
+					}, policy, {
+						signal: input.signal,
+						onAttemptFailed: (attempt, error, delayMs) => {
+							appendEvent(manifest.eventsPath, { type: "crew.task.retry_attempt", runId: manifest.runId, taskId: task.id, message: error.message, data: { attempt, delayMs } });
+							input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe({ runId: manifest.runId, taskId: task.id }, delayMs);
+						},
+						onRetryGivenUp: (attempts, error) => {
+							appendDeadletter(manifest, { runId: manifest.runId, taskId: task.id, reason: "max-retries", attempts, lastError: error.message, timestamp: new Date().toISOString() });
+							input.metricRegistry?.counter("crew.task.deadletter_total", "Deadletter triggers by reason").inc({ reason: "max-retries" });
+							input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempts - 1));
+						},
+					});
+				} catch {
+					return lastFailed ?? withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask(baseInput));
+				}
 			},
 		);
 		manifest = { ...results.at(-1)!.manifest, artifacts: mergeArtifacts([manifest.artifacts, ...results.map((item) => item.manifest.artifacts)].flat()) };

package/src/schema/config-schema.ts

@@ -81,6 +81,32 @@ export const PiTeamsNotificationsConfigSchema = Type.Object({
 	sinkRetentionDays: Type.Optional(Type.Integer({ minimum: 1, maximum: 90 })),
 });
 
+export const PiTeamsObservabilityConfigSchema = Type.Object({
+	enabled: Type.Optional(Type.Boolean()),
+	pollIntervalMs: Type.Optional(Type.Integer({ minimum: 1000, maximum: 60000 })),
+	metricRetentionDays: Type.Optional(Type.Integer({ minimum: 1, maximum: 365 })),
+});
+
+export const PiTeamsReliabilityConfigSchema = Type.Object({
+	autoRetry: Type.Optional(Type.Boolean()),
+	retryPolicy: Type.Optional(Type.Object({
+		maxAttempts: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })),
+		backoffMs: Type.Optional(Type.Integer({ minimum: 100, maximum: 60000 })),
+		jitterRatio: Type.Optional(Type.Number({ minimum: 0, maximum: 1 })),
+		exponentialFactor: Type.Optional(Type.Number({ minimum: 1, maximum: 5 })),
+		retryableErrors: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
+	})),
+	autoRecover: Type.Optional(Type.Boolean()),
+	deadletterThreshold: Type.Optional(Type.Integer({ minimum: 1 })),
+});
+
+export const PiTeamsOtlpConfigSchema = Type.Object({
+	enabled: Type.Optional(Type.Boolean()),
+	endpoint: Type.Optional(Type.String({ minLength: 1 })),
+	headers: Type.Optional(Type.Record(Type.String({ minLength: 1 }), Type.String())),
+	intervalMs: Type.Optional(Type.Integer({ minimum: 5000 })),
+});
+
 export const PiTeamsUiConfigSchema = Type.Object({
 	widgetPlacement: Type.Optional(Type.Union([Type.Literal("aboveEditor"), Type.Literal("belowEditor")])),
 	widgetMaxLines: Type.Optional(Type.Integer({ minimum: 1 })),
@@ -112,5 +138,8 @@ export const PiTeamsConfigSchema = Type.Object({
 	tools: Type.Optional(PiTeamsToolsConfigSchema),
 	telemetry: Type.Optional(PiTeamsTelemetryConfigSchema),
 	notifications: Type.Optional(PiTeamsNotificationsConfigSchema),
+	observability: Type.Optional(PiTeamsObservabilityConfigSchema),
+	reliability: Type.Optional(PiTeamsReliabilityConfigSchema),
+	otlp: Type.Optional(PiTeamsOtlpConfigSchema),
 	ui: Type.Optional(PiTeamsUiConfigSchema),
 });

package/src/state/event-log.ts

@@ -48,7 +48,7 @@ const MAX_EVENTS_BYTES = 50 * 1024 * 1024;
 
 const sequenceCache = new Map<string, { size: number; mtimeMs: number; seq: number }>();
 
-function sequencePath(eventsPath: string): string {
+export function sequencePath(eventsPath: string): string {
 	return `${eventsPath}.seq`;
 }
 
@@ -57,7 +57,8 @@ function parseSequence(raw: string): number | undefined {
 	return Number.isInteger(value) && value >= 0 ? value : undefined;
 }
 
-function scanSequence(eventsPath: string): number {
+export function scanSequence(eventsPath: string): number {
+	if (!fs.existsSync(eventsPath)) return 0;
 	let max = 0;
 	for (const line of fs.readFileSync(eventsPath, "utf-8").split("\n")) {
 		if (!line.trim()) continue;

package/src/state/types.ts

@@ -138,6 +138,12 @@ export interface TaskCheckpointState {
 	childPid?: number;
 }
 
+export interface TaskAttemptState {
+	startedAt: string;
+	endedAt?: string;
+	error?: string;
+}
+
 export interface TeamTaskState {
 	id: string;
 	runId: string;
@@ -166,6 +172,7 @@ export interface TeamTaskState {
 	claim?: TaskClaimState;
 	heartbeat?: WorkerHeartbeatState;
 	checkpoint?: TaskCheckpointState;
+	attempts?: TaskAttemptState[];
 	taskPacket?: TaskPacket;
 	verification?: VerificationEvidence;
 	graph?: TaskGraphNode;

package/src/ui/dashboard-panes/metrics-pane.ts

@@ -0,0 +1,34 @@
+import type { MetricRegistry } from "../../observability/metric-registry.ts";
+import type { HistogramPoint, MetricLabels, MetricPoint } from "../../observability/metrics-primitives.ts";
+import type { RunUiSnapshot } from "../snapshot-types.ts";
+
+export interface MetricsPaneOptions {
+	registry?: MetricRegistry;
+	maxCounters?: number;
+}
+
+function labelsText(labels: MetricLabels): string {
+	const entries = Object.entries(labels);
+	return entries.length ? `{${entries.map(([key, value]) => `${key}=${value}`).join(",")}}` : "";
+}
+
+function isHistogramPoint(point: MetricPoint | HistogramPoint): point is HistogramPoint {
+	return "quantiles" in point;
+}
+
+export function renderMetricsPane(_snapshot: RunUiSnapshot | undefined, opts: MetricsPaneOptions = {}): string[] {
+	if (!opts.registry) return ["Metrics pane: registry unavailable"];
+	const snapshots = opts.registry.snapshot();
+	if (!snapshots.length) return ["Metrics pane: no metrics recorded"];
+	const lines = ["Metrics pane: top metrics"];
+	for (const snapshot of snapshots.slice(0, opts.maxCounters ?? 10)) {
+		const first = snapshot.values[0];
+		if (!first) {
+			lines.push(`${snapshot.name}: empty`);
+			continue;
+		}
+		if (isHistogramPoint(first)) lines.push(`${snapshot.name}${labelsText(first.labels)} count=${first.count} p95=${Number.isFinite(first.quantiles.p95) ? Math.round(first.quantiles.p95) : "n/a"}`);
+		else lines.push(`${snapshot.name}${labelsText(first.labels)} ${first.value}`);
+	}
+	return lines;
+}

package/src/ui/heartbeat-aggregator.ts

@@ -1,4 +1,6 @@
 import type { TeamTaskState } from "../state/types.ts";
+import { classifyHeartbeat, heartbeatAgeMs } from "../runtime/heartbeat-gradient.ts";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
 import type { RunUiSnapshot } from "./snapshot-types.ts";
 
 export interface HeartbeatSummary {
@@ -9,12 +11,14 @@ export interface HeartbeatSummary {
 	dead: number;
 	missing: number;
 	worstStaleMs: number;
+	gradient: { healthy: number; warn: number; stale: number; dead: number };
 }
 
 export interface HeartbeatSummaryOptions {
 	staleMs?: number;
 	deadMs?: number;
 	now?: number | Date;
+	registry?: MetricRegistry;
 }
 
 function nowMs(now: number | Date | undefined): number {
@@ -24,29 +28,35 @@ function nowMs(now: number | Date | undefined): number {
 }
 
 function isActiveTask(task: TeamTaskState): boolean {
-	return task.status === "running" || task.status === "queued";
+	return task.status === "running";
 }
 
 export function summarizeHeartbeats(snapshot: RunUiSnapshot, opts: HeartbeatSummaryOptions = {}): HeartbeatSummary {
 	const staleMs = opts.staleMs ?? 60_000;
 	const deadMs = opts.deadMs ?? 5 * 60_000;
 	const current = nowMs(opts.now);
-	const summary: HeartbeatSummary = { runId: snapshot.runId, totalTasks: snapshot.tasks.length, healthy: 0, stale: 0, dead: 0, missing: 0, worstStaleMs: 0 };
+	const summary: HeartbeatSummary = { runId: snapshot.runId, totalTasks: snapshot.tasks.length, healthy: 0, stale: 0, dead: 0, missing: 0, worstStaleMs: 0, gradient: { healthy: 0, warn: 0, stale: 0, dead: 0 } };
 	for (const task of snapshot.tasks) {
 		if (!isActiveTask(task)) continue;
 		const heartbeat = task.heartbeat;
 		if (!heartbeat) {
 			summary.missing += 1;
+			summary.gradient.dead += 1;
 			continue;
 		}
-		const age = Math.max(0, current - Date.parse(heartbeat.lastSeenAt));
+		const age = heartbeatAgeMs(heartbeat, current);
 		if (!Number.isFinite(age)) {
 			summary.missing += 1;
+			summary.gradient.dead += 1;
 			continue;
 		}
 		summary.worstStaleMs = Math.max(summary.worstStaleMs, age);
-		if (heartbeat.alive === false || age > deadMs) summary.dead += 1;
-		else if (age > staleMs) summary.stale += 1;
+		const level = classifyHeartbeat(heartbeat, { warnMs: Math.max(1, Math.floor(staleMs / 2)), staleMs, deadMs }, current);
+		summary.gradient[level] += 1;
+		opts.registry?.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: snapshot.runId, taskId: task.id }, age);
+		opts.registry?.counter("crew.heartbeat.level_total", "Heartbeat classifications by level").inc({ runId: snapshot.runId, level });
+		if (level === "dead") summary.dead += 1;
+		else if (level === "stale") summary.stale += 1;
 		else summary.healthy += 1;
 	}
 	return summary;