pi-crew 0.1.34 → 0.1.36

package/src/observability/metric-sink.ts

@@ -0,0 +1,51 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { redactSecrets } from "../runtime/diagnostic-export.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import type { MetricRegistry } from "./metric-registry.ts";
+import type { MetricSnapshot } from "./metrics-primitives.ts";
+
+export interface MetricSink {
+	writeSnapshot(snapshots: MetricSnapshot[]): void;
+	dispose(): void;
+}
+
+export interface MetricFileSinkOptions {
+	crewRoot: string;
+	registry: MetricRegistry;
+	retentionDays?: number;
+	intervalMs?: number;
+}
+
+function rotateOldFiles(dir: string, retentionDays: number, now = Date.now()): void {
+	if (!fs.existsSync(dir)) return;
+	const maxAge = retentionDays * 24 * 60 * 60 * 1000;
+	for (const file of fs.readdirSync(dir)) {
+		if (!file.endsWith(".jsonl")) continue;
+		const fullPath = path.join(dir, file);
+		try {
+			if (now - fs.statSync(fullPath).mtimeMs > maxAge) fs.unlinkSync(fullPath);
+		} catch (error) {
+			logInternalError("metric-sink.rotate", error, fullPath);
+		}
+	}
+}
+
+export function createMetricFileSink(opts: MetricFileSinkOptions): MetricSink {
+	const dir = path.join(opts.crewRoot, "state", "metrics");
+	const retentionDays = opts.retentionDays ?? 7;
+	const writeSnapshot = (snapshots: MetricSnapshot[]): void => {
+		try {
+			const date = new Date().toISOString().slice(0, 10);
+			fs.mkdirSync(dir, { recursive: true });
+			rotateOldFiles(dir, retentionDays);
+			const redacted = redactSecrets(snapshots) as MetricSnapshot[];
+			fs.appendFileSync(path.join(dir, `${date}.jsonl`), `${JSON.stringify({ exportedAt: new Date().toISOString(), snapshots: redacted })}\n`, "utf-8");
+		} catch (error) {
+			logInternalError("metric-sink.write", error);
+		}
+	};
+	const timer = setInterval(() => writeSnapshot(opts.registry.snapshot()), opts.intervalMs ?? 60_000);
+	timer.unref?.();
+	return { writeSnapshot, dispose: () => clearInterval(timer) };
+}

package/src/observability/metrics-primitives.ts

@@ -0,0 +1,166 @@
+export type MetricLabelValue = string | number;
+export type MetricLabels = Record<string, MetricLabelValue>;
+
+export interface MetricPoint {
+	labels: MetricLabels;
+	value: number;
+}
+
+export interface HistogramPoint {
+	labels: MetricLabels;
+	buckets: number[];
+	counts: number[];
+	sum: number;
+	count: number;
+	quantiles: Record<string, number>;
+}
+
+export interface MetricSnapshot {
+	type: "counter" | "gauge" | "histogram";
+	name: string;
+	description: string;
+	values: MetricPoint[] | HistogramPoint[];
+}
+
+interface StoredValue {
+	labels: MetricLabels;
+	value: number;
+}
+
+interface StoredHistogram {
+	labels: MetricLabels;
+	counts: number[];
+	sum: number;
+	count: number;
+}
+
+export const DEFAULT_HISTOGRAM_BUCKETS = [1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000] as const;
+
+function normalizeLabels(labels: MetricLabels = {}): MetricLabels {
+	const normalized: MetricLabels = {};
+	for (const [key, value] of Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))) normalized[key] = value;
+	return normalized;
+}
+
+export function labelKey(labels: MetricLabels = {}): string {
+	return JSON.stringify(normalizeLabels(labels));
+}
+
+function cloneLabels(labels: MetricLabels): MetricLabels {
+	return { ...labels };
+}
+
+export abstract class Metric {
+	readonly name: string;
+	readonly description: string;
+
+	constructor(name: string, description: string) {
+		this.name = name;
+		this.description = description;
+	}
+
+	abstract snapshot(): MetricSnapshot;
+}
+
+export class Counter extends Metric {
+	private values = new Map<string, StoredValue>();
+
+	inc(labels: MetricLabels = {}, delta = 1): void {
+		if (!Number.isFinite(delta)) return;
+		const key = labelKey(labels);
+		const current = this.values.get(key) ?? { labels: normalizeLabels(labels), value: 0 };
+		this.values.set(key, { labels: current.labels, value: current.value + delta });
+	}
+
+	value(labels: MetricLabels = {}): number {
+		return this.values.get(labelKey(labels))?.value ?? 0;
+	}
+
+	snapshot(): MetricSnapshot {
+		return { type: "counter", name: this.name, description: this.description, values: [...this.values.values()].map((entry) => ({ labels: cloneLabels(entry.labels), value: entry.value })) };
+	}
+}
+
+export class Gauge extends Metric {
+	private values = new Map<string, StoredValue>();
+
+	set(labels: MetricLabels = {}, value: number): void {
+		if (!Number.isFinite(value)) return;
+		this.values.set(labelKey(labels), { labels: normalizeLabels(labels), value });
+	}
+
+	add(labels: MetricLabels = {}, delta: number): void {
+		if (!Number.isFinite(delta)) return;
+		this.set(labels, this.value(labels) + delta);
+	}
+
+	value(labels: MetricLabels = {}): number {
+		return this.values.get(labelKey(labels))?.value ?? 0;
+	}
+
+	snapshot(): MetricSnapshot {
+		return { type: "gauge", name: this.name, description: this.description, values: [...this.values.values()].map((entry) => ({ labels: cloneLabels(entry.labels), value: entry.value })) };
+	}
+}
+
+export class Histogram extends Metric {
+	private readonly buckets: number[];
+	private observations = new Map<string, StoredHistogram>();
+
+	constructor(name: string, description: string, buckets?: number[]) {
+		super(name, description);
+		const source = buckets?.length ? buckets : [...DEFAULT_HISTOGRAM_BUCKETS];
+		this.buckets = [...new Set(source.filter((bucket) => Number.isFinite(bucket)).sort((left, right) => left - right))];
+	}
+
+	observe(labels: MetricLabels = {}, value: number): void {
+		if (!Number.isFinite(value)) return;
+		const key = labelKey(labels);
+		const current = this.observations.get(key) ?? { labels: normalizeLabels(labels), counts: new Array(this.buckets.length + 1).fill(0) as number[], sum: 0, count: 0 };
+		const bucketIndex = this.buckets.findIndex((bucket) => value <= bucket);
+		current.counts[bucketIndex === -1 ? this.buckets.length : bucketIndex] = (current.counts[bucketIndex === -1 ? this.buckets.length : bucketIndex] ?? 0) + 1;
+		current.sum += value;
+		current.count += 1;
+		this.observations.set(key, current);
+	}
+
+	quantile(labels: MetricLabels = {}, q: number): number {
+		const obs = this.observations.get(labelKey(labels));
+		if (!obs || obs.count === 0 || !Number.isFinite(q)) return Number.NaN;
+		const bounded = Math.min(1, Math.max(0, q));
+		const target = Math.max(1, bounded * obs.count);
+		let cumulative = 0;
+		for (let index = 0; index < obs.counts.length; index += 1) {
+			const count = obs.counts[index] ?? 0;
+			cumulative += count;
+			if (cumulative >= target) {
+				const previous = cumulative - count;
+				const lower = index === 0 ? 0 : this.buckets[index - 1] ?? this.buckets.at(-1) ?? 0;
+				const upper = index < this.buckets.length ? this.buckets[index] ?? lower : Math.max(lower, obs.sum / Math.max(1, obs.count));
+				const fraction = count === 0 ? 0 : (target - previous) / Math.max(1, count);
+				return lower + fraction * (upper - lower);
+			}
+		}
+		return this.buckets.at(-1) ?? Number.NaN;
+	}
+
+	count(labels: MetricLabels = {}): number {
+		return this.observations.get(labelKey(labels))?.count ?? 0;
+	}
+
+	snapshot(): MetricSnapshot {
+		return {
+			type: "histogram",
+			name: this.name,
+			description: this.description,
+			values: [...this.observations.values()].map((entry) => ({
+				labels: cloneLabels(entry.labels),
+				buckets: [...this.buckets, Number.POSITIVE_INFINITY],
+				counts: [...entry.counts],
+				sum: entry.sum,
+				count: entry.count,
+				quantiles: { p50: this.quantile(entry.labels, 0.5), p95: this.quantile(entry.labels, 0.95), p99: this.quantile(entry.labels, 0.99) },
+			})),
+		};
+	}
+}

package/src/runtime/child-pi.ts

@@ -430,7 +430,11 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
 					clearHardKillTimer(child.pid);
 				}
 				const timeoutError = responseTimeoutHit && !stderr.trim() ? { error: `Child Pi produced no new output for ${responseTimeoutMs}ms; process was terminated as unresponsive.` } : undefined;
-				settle({ exitCode, stdout, stderr, ...(forcedFinalDrain && !stderr.trim() ? { error: `Child Pi did not exit within ${finalDrainMs}ms after final assistant message; termination was requested.` } : {}), ...(timeoutError ? { error: timeoutError.error } : {}) });
+				// A final assistant event is the child Pi contract for "the worker produced its answer".
+				// Some Pi processes can linger during post-final cleanup/stdio shutdown; finalDrain terminates
+				// that lingering process so the parent can continue, but it must not turn a completed
+				// subagent answer into a failed task. Real pre-final response timeouts still report errors.
+				settle({ exitCode: forcedFinalDrain && !timeoutError ? 0 : exitCode, stdout, stderr, ...(timeoutError ? { error: timeoutError.error } : {}) });
 			});
 		});
 	} finally {

package/src/runtime/crash-recovery.ts

@@ -0,0 +1,56 @@
+import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { appendEvent, scanSequence } from "../state/event-log.ts";
+import { loadRunManifestById, saveRunTasks, updateRunStatus } from "../state/state-store.ts";
+import type { TeamTaskState } from "../state/types.ts";
+import { isWorkerHeartbeatStale } from "./worker-heartbeat.ts";
+import type { ManifestCache } from "./manifest-cache.ts";
+import { checkProcessLiveness } from "./process-status.ts";
+
+export interface RecoveryPlan {
+	runId: string;
+	resumableTasks: string[];
+	preservedTasks: string[];
+	lastEventSeq: number;
+}
+
+function isTerminalTask(task: TeamTaskState): boolean {
+	return task.status === "completed" || task.status === "failed" || task.status === "cancelled" || task.status === "skipped";
+}
+
+function shouldRecoverTask(task: TeamTaskState, deadMs: number): boolean {
+	if (task.status !== "running") return false;
+	if (!task.heartbeat) return true;
+	return task.heartbeat.alive === false || isWorkerHeartbeatStale(task.heartbeat, deadMs);
+}
+
+export function detectInterruptedRuns(cwd: string, manifestCache: ManifestCache, deadMs = 300_000): RecoveryPlan[] {
+	const plans: RecoveryPlan[] = [];
+	for (const manifest of manifestCache.list(50)) {
+		if (manifest.status !== "running") continue;
+		if (manifest.async?.pid !== undefined && checkProcessLiveness(manifest.async.pid).alive) continue;
+		const loaded = loadRunManifestById(cwd, manifest.runId);
+		if (!loaded) continue;
+		const resumableTasks = loaded.tasks.filter((task) => shouldRecoverTask(task, deadMs)).map((task) => task.id);
+		if (!resumableTasks.length) continue;
+		plans.push({ runId: manifest.runId, resumableTasks, preservedTasks: loaded.tasks.filter(isTerminalTask).map((task) => task.id), lastEventSeq: scanSequence(loaded.manifest.eventsPath) });
+	}
+	return plans;
+}
+
+export async function applyRecoveryPlan(plan: RecoveryPlan, ctx: Pick<ExtensionContext, "cwd">, registry?: MetricRegistry): Promise<void> {
+	const loaded = loadRunManifestById(ctx.cwd, plan.runId);
+	if (!loaded) throw new Error(`Run '${plan.runId}' not found.`);
+	const reset = new Set(plan.resumableTasks);
+	const tasks = loaded.tasks.map((task) => reset.has(task.id) ? { ...task, status: "queued" as const, startedAt: undefined, finishedAt: undefined, error: undefined, heartbeat: undefined } : task);
+	saveRunTasks(loaded.manifest, tasks);
+	appendEvent(loaded.manifest.eventsPath, { type: "crew.run.resumed", runId: plan.runId, message: `Recovered ${plan.resumableTasks.length} interrupted task(s).`, data: { recoveredFromSeq: plan.lastEventSeq, resumableTasks: plan.resumableTasks } });
+	registry?.counter("crew.run.count", "Total runs by status").inc({ status: "resumed" });
+}
+
+export function declineRecoveryPlan(plan: RecoveryPlan, ctx: Pick<ExtensionContext, "cwd">): void {
+	const loaded = loadRunManifestById(ctx.cwd, plan.runId);
+	if (!loaded) throw new Error(`Run '${plan.runId}' not found.`);
+	updateRunStatus(loaded.manifest, "cancelled", "interrupted-not-resumed");
+	appendEvent(loaded.manifest.eventsPath, { type: "crew.run.recovery_declined", runId: plan.runId, message: "Interrupted run was not resumed.", data: { recoveredFromSeq: plan.lastEventSeq } });
+}

package/src/runtime/deadletter.ts

@@ -0,0 +1,36 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { TeamRunManifest } from "../state/types.ts";
+
+export type DeadletterReason = "max-retries" | "heartbeat-dead" | "manual";
+
+export interface DeadletterEntry {
+	taskId: string;
+	runId: string;
+	reason: DeadletterReason;
+	attempts: number;
+	lastError?: string;
+	timestamp: string;
+}
+
+export function deadletterPath(manifest: TeamRunManifest): string {
+	return path.join(manifest.stateRoot, "deadletter.jsonl");
+}
+
+export function appendDeadletter(manifest: TeamRunManifest, entry: DeadletterEntry): void {
+	fs.mkdirSync(manifest.stateRoot, { recursive: true });
+	fs.appendFileSync(deadletterPath(manifest), `${JSON.stringify(entry)}\n`, "utf-8");
+}
+
+export function readDeadletter(manifest: TeamRunManifest): DeadletterEntry[] {
+	const filePath = deadletterPath(manifest);
+	if (!fs.existsSync(filePath)) return [];
+	return fs.readFileSync(filePath, "utf-8").split(/\r?\n/).filter(Boolean).flatMap((line) => {
+		try {
+			const parsed = JSON.parse(line) as DeadletterEntry;
+			return parsed && typeof parsed.taskId === "string" && typeof parsed.runId === "string" ? [parsed] : [];
+		} catch {
+			return [];
+		}
+	});
+}

package/src/runtime/diagnostic-export.ts

@@ -1,4 +1,6 @@
 import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import type { MetricSnapshot } from "../observability/metrics-primitives.ts";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { readCrewAgents } from "./crew-agent-records.ts";
@@ -9,6 +11,7 @@ import { summarizeHeartbeats, type HeartbeatSummary } from "../ui/heartbeat-aggr
 import type { RunUiSnapshot } from "../ui/snapshot-types.ts";
 
 export interface DiagnosticReport {
+	schemaVersion?: number;
 	runId: string;
 	exportedAt: string;
 	manifest: TeamRunManifest;
@@ -17,6 +20,7 @@ export interface DiagnosticReport {
 	heartbeat: HeartbeatSummary;
 	agents: unknown[];
 	envRedacted: Record<string, string>;
+	metricsSnapshot?: MetricSnapshot[];
 }
 
 const SECRET_KEY_PATTERN = /(token|key|password|secret|credential|auth)/i;
@@ -70,13 +74,15 @@ function buildSnapshot(manifest: TeamRunManifest, tasks: TeamTaskState[]): RunUi
 	};
 }
 
-export async function exportDiagnostic(ctx: Pick<ExtensionContext, "cwd">, runId: string): Promise<{ path: string; report: DiagnosticReport }> {
+export async function exportDiagnostic(ctx: Pick<ExtensionContext, "cwd">, runId: string, options: { registry?: MetricRegistry } = {}): Promise<{ path: string; report: DiagnosticReport }> {
 	const loaded = loadRunManifestById(ctx.cwd, runId);
 	if (!loaded) throw new Error(`Run '${runId}' not found.`);
 	const exportedAt = new Date().toISOString();
 	const safeTimestamp = exportedAt.replace(/[:.]/g, "-");
 	const recentEvents = readEvents(loaded.manifest.eventsPath).slice(-200);
+	const metricsSnapshot = options.registry?.snapshot();
 	const report: DiagnosticReport = {
+		...(metricsSnapshot ? { schemaVersion: 2 } : {}),
 		runId,
 		exportedAt,
 		manifest: redactSecrets(loaded.manifest) as TeamRunManifest,
@@ -85,6 +91,7 @@ export async function exportDiagnostic(ctx: Pick<ExtensionContext, "cwd">, runId
 		heartbeat: summarizeHeartbeats(buildSnapshot(loaded.manifest, loaded.tasks)),
 		agents: redactSecrets(readCrewAgents(loaded.manifest)) as unknown[],
 		envRedacted: envRedacted(),
+		...(metricsSnapshot ? { metricsSnapshot: redactSecrets(metricsSnapshot) as MetricSnapshot[] } : {}),
 	};
 	const dir = path.join(loaded.manifest.artifactsRoot, "diagnostic");
 	fs.mkdirSync(dir, { recursive: true });

package/src/runtime/heartbeat-gradient.ts

@@ -0,0 +1,28 @@
+import type { WorkerHeartbeatState } from "./worker-heartbeat.ts";
+
+export type HeartbeatLevel = "healthy" | "warn" | "stale" | "dead";
+
+export interface GradientThresholds {
+	warnMs: number;
+	staleMs: number;
+	deadMs: number;
+}
+
+export const DEFAULT_GRADIENT_THRESHOLDS: GradientThresholds = { warnMs: 30_000, staleMs: 60_000, deadMs: 300_000 };
+
+export function heartbeatAgeMs(heartbeat: WorkerHeartbeatState | undefined, now = Date.now()): number {
+	if (!heartbeat) return Number.POSITIVE_INFINITY;
+	const lastSeen = Date.parse(heartbeat.lastSeenAt);
+	return Number.isFinite(lastSeen) ? Math.max(0, now - lastSeen) : Number.POSITIVE_INFINITY;
+}
+
+export function classifyHeartbeat(heartbeat: WorkerHeartbeatState | undefined, thresholds: GradientThresholds = DEFAULT_GRADIENT_THRESHOLDS, now = Date.now()): HeartbeatLevel {
+	if (!heartbeat) return "dead";
+	if (heartbeat.alive === false) return "dead";
+	const elapsed = heartbeatAgeMs(heartbeat, now);
+	if (!Number.isFinite(elapsed)) return "dead";
+	if (elapsed > thresholds.deadMs) return "dead";
+	if (elapsed > thresholds.staleMs) return "stale";
+	if (elapsed > thresholds.warnMs) return "warn";
+	return "healthy";
+}

package/src/runtime/heartbeat-watcher.ts

@@ -0,0 +1,80 @@
+import type { NotificationDescriptor } from "../extension/notification-router.ts";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { loadRunManifestById } from "../state/state-store.ts";
+import type { TeamRunManifest } from "../state/types.ts";
+import type { ManifestCache } from "./manifest-cache.ts";
+import { classifyHeartbeat, DEFAULT_GRADIENT_THRESHOLDS, heartbeatAgeMs, type GradientThresholds, type HeartbeatLevel } from "./heartbeat-gradient.ts";
+
+export interface HeartbeatWatcherRouter {
+	enqueue(notification: NotificationDescriptor): boolean;
+}
+
+export interface HeartbeatWatcherOptions {
+	cwd: string;
+	pollIntervalMs?: number;
+	thresholds?: GradientThresholds;
+	manifestCache: ManifestCache;
+	registry: MetricRegistry;
+	router: HeartbeatWatcherRouter;
+	deadletterTickThreshold?: number;
+	onDead?: (runId: string, taskId: string, elapsed: number) => void;
+	onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
+}
+
+export class HeartbeatWatcher {
+	private timer?: ReturnType<typeof setInterval>;
+	private lastLevel = new Map<string, HeartbeatLevel>();
+	private consecutiveDead = new Map<string, number>();
+	private readonly opts: HeartbeatWatcherOptions;
+
+	constructor(opts: HeartbeatWatcherOptions) {
+		this.opts = opts;
+	}
+
+	start(): void {
+		this.dispose();
+		this.timer = setInterval(() => this.tick(), this.opts.pollIntervalMs ?? 5000);
+		this.timer.unref?.();
+	}
+
+	tick(now = Date.now()): void {
+		const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
+		const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
+		for (const run of this.opts.manifestCache.list(50)) {
+			if (run.status !== "running") continue;
+			const loaded = loadRunManifestById(this.opts.cwd, run.runId);
+			if (!loaded) continue;
+			for (const task of loaded.tasks) {
+				if (task.status !== "running" && task.status !== "queued") continue;
+				const key = `${run.runId}:${task.id}`;
+				const elapsed = heartbeatAgeMs(task.heartbeat, now);
+				const level = classifyHeartbeat(task.heartbeat, thresholds, now);
+				this.opts.registry.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds").set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				this.opts.registry.counter("crew.heartbeat.level_total", "Heartbeat classifications by level").inc({ runId: run.runId, level });
+				const previous = this.lastLevel.get(key);
+				this.lastLevel.set(key, level);
+				if (level === "dead" && previous !== "dead") {
+					this.opts.registry.counter("crew.heartbeat.dead_total", "Dead heartbeat detections").inc({ runId: run.runId });
+					appendEvent(loaded.manifest.eventsPath, { type: "crew.task.heartbeat_dead", runId: run.runId, taskId: task.id, message: `Task ${task.id} heartbeat dead.`, data: { elapsedMs: Number.isFinite(elapsed) ? elapsed : undefined } });
+					this.opts.router.enqueue({ id: `dead_${run.runId}_${task.id}`, severity: "warning", source: "heartbeat-watcher", runId: run.runId, title: `Task ${task.id} heartbeat dead`, body: "Background watcher detected a stuck worker." });
+					this.opts.onDead?.(run.runId, task.id, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				}
+				if (level === "dead") {
+					const count = (this.consecutiveDead.get(key) ?? 0) + 1;
+					this.consecutiveDead.set(key, count);
+					if (count === tickThreshold) this.opts.onDeadletterTrigger?.(loaded.manifest, task.id);
+				} else {
+					this.consecutiveDead.delete(key);
+				}
+			}
+		}
+	}
+
+	dispose(): void {
+		if (this.timer) clearInterval(this.timer);
+		this.timer = undefined;
+		this.lastLevel.clear();
+		this.consecutiveDead.clear();
+	}
+}

package/src/runtime/retry-executor.ts

@@ -0,0 +1,59 @@
+import { sleep } from "../utils/sleep.ts";
+
+export interface RetryPolicy {
+	maxAttempts: number;
+	backoffMs: number;
+	jitterRatio: number;
+	exponentialFactor: number;
+	retryableErrors?: string[];
+}
+
+export interface RetryHooks {
+	onAttemptFailed?: (attempt: number, error: Error, nextDelayMs: number) => void;
+	onRetryGivenUp?: (attempts: number, error: Error) => void;
+	signal?: AbortSignal;
+}
+
+export const DEFAULT_RETRY_POLICY: RetryPolicy = { maxAttempts: 3, backoffMs: 1000, jitterRatio: 0.3, exponentialFactor: 2 };
+
+function asError(error: unknown): Error {
+	return error instanceof Error ? error : new Error(String(error));
+}
+
+function globToRegex(pattern: string): RegExp {
+	const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+	return new RegExp(`^${escaped}$`, "i");
+}
+
+function isRetryable(error: Error, policy: RetryPolicy): boolean {
+	const patterns = policy.retryableErrors ?? [];
+	if (!patterns.length) return true;
+	return patterns.some((pattern) => globToRegex(pattern).test(error.message));
+}
+
+export function calculateRetryDelay(attempt: number, policy: RetryPolicy = DEFAULT_RETRY_POLICY, random = Math.random): number {
+	const base = policy.backoffMs * Math.pow(policy.exponentialFactor, Math.max(0, attempt - 1));
+	const jitter = (random() * 2 - 1) * policy.jitterRatio * base;
+	return Math.max(0, base + jitter);
+}
+
+export async function executeWithRetry<T>(fn: (attempt: number) => Promise<T>, policy: RetryPolicy = DEFAULT_RETRY_POLICY, hooks: RetryHooks = {}): Promise<T> {
+	const normalized: RetryPolicy = { ...DEFAULT_RETRY_POLICY, ...policy, maxAttempts: Math.max(1, policy.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts) };
+	let lastError: Error | undefined;
+	for (let attempt = 1; attempt <= normalized.maxAttempts; attempt += 1) {
+		if (hooks.signal?.aborted) throw new Error("Retry aborted.");
+		try {
+			return await fn(attempt);
+		} catch (error) {
+			lastError = asError(error);
+			if (attempt >= normalized.maxAttempts || !isRetryable(lastError, normalized)) {
+				hooks.onRetryGivenUp?.(attempt, lastError);
+				throw lastError;
+			}
+			const delay = calculateRetryDelay(attempt, normalized);
+			hooks.onAttemptFailed?.(attempt, lastError, delay);
+			await sleep(delay, hooks.signal);
+		}
+	}
+	throw lastError ?? new Error("Retry failed without error.");
+}

package/src/runtime/team-runner.ts

@@ -1,11 +1,11 @@
 import * as fs from "node:fs";
 import type { AgentConfig } from "../agents/agent-config.ts";
-import type { CrewLimitsConfig, CrewRuntimeConfig } from "../config/config.ts";
+import type { CrewLimitsConfig, CrewRuntimeConfig, CrewReliabilityConfig } from "../config/config.ts";
 import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
 import { writeArtifact } from "../state/artifact-store.ts";
 import { appendEvent } from "../state/event-log.ts";
 import type { TeamConfig } from "../teams/team-config.ts";
-import type { ArtifactDescriptor, PolicyDecision, TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import type { ArtifactDescriptor, PolicyDecision, TeamRunManifest, TaskAttemptState, TeamTaskState } from "../state/types.ts";
 import { saveRunManifest, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
 import { aggregateUsage, formatUsage } from "../state/usage.ts";
 import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
@@ -18,6 +18,10 @@ import { saveCrewAgents } from "./crew-agent-records.ts";
 import { recordsForMaterializedTasks } from "./task-display.ts";
 import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
 import { runTeamTask } from "./task-runner.ts";
+import { executeWithRetry, DEFAULT_RETRY_POLICY, type RetryPolicy } from "./retry-executor.ts";
+import { appendDeadletter } from "./deadletter.ts";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
 import { resolveBatchConcurrency } from "./concurrency.ts";
 import { mapConcurrent } from "./parallel-utils.ts";
 
@@ -36,6 +40,8 @@ export interface ExecuteTeamRunInput {
 	modelRegistry?: unknown;
 	modelOverride?: string;
 	signal?: AbortSignal;
+	reliability?: CrewReliabilityConfig;
+	metricRegistry?: MetricRegistry;
 }
 
 function findReadyTask(tasks: TeamTaskState[]): TeamTaskState | undefined {
@@ -73,7 +79,7 @@ function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState):
 	// contain stale queued/running copies of tasks that another worker already
 	// completed. Never let those stale snapshots regress durable task state.
 	if (!isNonTerminalTaskStatus(current.status) && isNonTerminalTaskStatus(updated.status)) return false;
-	return updated.status !== current.status || updated.finishedAt !== current.finishedAt || updated.startedAt !== current.startedAt || Boolean(updated.resultArtifact) || Boolean(updated.error) || Boolean(updated.modelAttempts?.length) || Boolean(updated.usage);
+	return updated.status !== current.status || updated.finishedAt !== current.finishedAt || updated.startedAt !== current.startedAt || Boolean(updated.resultArtifact) || Boolean(updated.error) || Boolean(updated.modelAttempts?.length) || Boolean(updated.usage) || Boolean(updated.attempts?.length);
 }
 
 export function __test__mergeTaskUpdates(base: TeamTaskState[], results: Array<{ tasks: TeamTaskState[] }>): TeamTaskState[] {
@@ -384,6 +390,14 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
 	return { ...manifest, updatedAt: new Date().toISOString(), policyDecisions: decisions, artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "metadata" && (artifact.path.endsWith("policy-decisions.json") || artifact.path.endsWith("recovery-ledger.json") || artifact.path.endsWith("branch-freshness.json")))), branchArtifact, policyArtifact, recoveryArtifact] };
 }
 
+function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): RetryPolicy {
+	return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
+}
+
+function failedTaskFrom(result: { tasks: TeamTaskState[] }, taskId: string): TeamTaskState | undefined {
+	return result.tasks.find((item) => item.id === taskId && item.status === "failed");
+}
+
 export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
 	let workflow = input.workflow;
 	let manifest = updateRunStatus(input.manifest, "running", input.executeWorkers ? "Executing team workflow." : "Creating workflow prompts and placeholder results.");
@@ -450,10 +464,48 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
 		const results = await mapConcurrent(
 			readyBatch,
 			concurrency.selectedCount,
-			(task) => {
+			async (task) => {
 				const step = findStep(workflow, task);
 				const agent = findAgent(input.agents, task);
-				return runTeamTask({ manifest, tasks, task, step, agent, signal: input.signal, executeWorkers: input.executeWorkers, runtimeKind: input.runtime?.kind, runtimeConfig: input.runtimeConfig, parentContext: input.parentContext, parentModel: input.parentModel, modelRegistry: input.modelRegistry, modelOverride: input.modelOverride, limits: input.limits });
+				const baseInput = { manifest, tasks, task, step, agent, signal: input.signal, executeWorkers: input.executeWorkers, runtimeKind: input.runtime?.kind, runtimeConfig: input.runtimeConfig, parentContext: input.parentContext, parentModel: input.parentModel, modelRegistry: input.modelRegistry, modelOverride: input.modelOverride, limits: input.limits };
+				if (input.reliability?.autoRetry !== true) return withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask(baseInput));
+				let lastFailed: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
+				const attemptsSoFar: TaskAttemptState[] = [...(task.attempts ?? [])];
+				const policy = retryPolicyFromConfig(input.reliability);
+				try {
+					return await executeWithRetry(async (attempt) => {
+						const startedAt = new Date().toISOString();
+						const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { startedAt }];
+						input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({ runId: manifest.runId, taskId: task.id });
+						const taskWithAttempt: TeamTaskState = { ...task, attempts: inFlightAttempts };
+						const result = await withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask({ ...baseInput, task: taskWithAttempt }));
+						const failed = failedTaskFrom(result, task.id);
+						const endedAt = new Date().toISOString();
+						const finishedAttempt: TaskAttemptState = { startedAt, endedAt, ...(failed?.error ? { error: failed.error } : {}) };
+						attemptsSoFar.push(finishedAttempt);
+						const withAttempt = result.tasks.map((item) => item.id === task.id ? { ...item, attempts: [...attemptsSoFar] } : item);
+						const enriched = { manifest: result.manifest, tasks: withAttempt };
+						if (failed) {
+							lastFailed = enriched;
+							throw new Error(failed.error ?? `Task ${task.id} failed.`);
+						}
+						input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempt - 1));
+						return enriched;
+					}, policy, {
+						signal: input.signal,
+						onAttemptFailed: (attempt, error, delayMs) => {
+							appendEvent(manifest.eventsPath, { type: "crew.task.retry_attempt", runId: manifest.runId, taskId: task.id, message: error.message, data: { attempt, delayMs } });
+							input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe({ runId: manifest.runId, taskId: task.id }, delayMs);
+						},
+						onRetryGivenUp: (attempts, error) => {
+							appendDeadletter(manifest, { runId: manifest.runId, taskId: task.id, reason: "max-retries", attempts, lastError: error.message, timestamp: new Date().toISOString() });
+							input.metricRegistry?.counter("crew.task.deadletter_total", "Deadletter triggers by reason").inc({ reason: "max-retries" });
+							input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempts - 1));
+						},
+					});
+				} catch {
+					return lastFailed ?? withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask(baseInput));
+				}
 			},
 		);
 		manifest = { ...results.at(-1)!.manifest, artifacts: mergeArtifacts([manifest.artifacts, ...results.map((item) => item.manifest.artifacts)].flat()) };