pi-crew 0.9.53 → 0.9.55

package/src/runtime/heartbeat-watcher.ts

@@ -0,0 +1,239 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { NotificationDescriptor } from "../extension/notification-router.ts";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { loadRunManifestById } from "../state/state-store.ts";
+import type { TeamRunManifest } from "../state/types.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import { DEFAULT_GRADIENT_THRESHOLDS, type GradientThresholds, type HeartbeatLevel, heartbeatAgeMs } from "./heartbeat-gradient.ts";
+import type { ManifestCache } from "./manifest-cache.ts";
+
+export interface HeartbeatWatcherRouter {
+	enqueue(notification: NotificationDescriptor): boolean;
+}
+
+export interface HeartbeatWatcherOptions {
+	cwd: string;
+	pollIntervalMs?: number;
+	thresholds?: GradientThresholds;
+	manifestCache: ManifestCache;
+	registry: MetricRegistry;
+	router: HeartbeatWatcherRouter;
+	deadletterTickThreshold?: number;
+	/**
+	 * 3.6 — minimum interval between repeated deadletter triggers for the same
+	 * runId+taskId. Without this, a flaky worker (dead → alive → dead) can
+	 * fire deadletter entries faster than the operator can respond. Default
+	 * 60_000 ms.
+	 */
+	deadletterCooldownMs?: number;
+	onDead?: (runId: string, taskId: string, elapsed: number) => void;
+	onDeadletterTrigger?: (manifest: TeamRunManifest, taskId: string) => void;
+}
+
+/**
+ * Polls running runs for heartbeat staleness.
+ *
+ * Uses recursive setTimeout to avoid timer storms.
+ * Cleanup is done in the same pass — no second scan over manifests.
+ * Keys for runs that disappear from the cache are cleaned via staleness-age policy
+ * rather than being leaked forever.
+ */
+export class HeartbeatWatcher {
+	private timer?: ReturnType<typeof setTimeout>;
+	private lastLevel = new Map<string, HeartbeatLevel>();
+	private consecutiveDead = new Map<string, number>();
+	private lastSeen = new Map<string, number>(); // key → last time it was active
+	private lastDeadletterTriggerAt = new Map<string, number>(); // 3.6 cooldown gate
+	/** Max age (ms) to retain a stale key before garbage-collecting it. */
+	private readonly maxKeyAgeMs = 600_000; // 10 minutes
+	private readonly opts: HeartbeatWatcherOptions;
+
+	constructor(opts: HeartbeatWatcherOptions) {
+		this.opts = opts;
+	}
+
+	start(): void {
+		this.dispose();
+		this.scheduleTick();
+	}
+
+	private scheduleTick(): void {
+		// 3.2 — when at least one run has a dead-streak in progress, poll faster
+		// (1s) so operators get notified quickly. Healthy state stays at the
+		// configured interval (default 5s) to keep idle CPU near zero.
+		const baseInterval = this.opts.pollIntervalMs ?? 5000;
+		const interval = this.consecutiveDead.size > 0 ? Math.min(1000, baseInterval) : baseInterval;
+		this.timer = setTimeout(() => this.tick(), interval);
+		this.timer.unref();
+	}
+
+	tick(now = Date.now()): void {
+		try {
+			this.tickUnsafe(now);
+		} catch (error) {
+			logInternalError("heartbeat-watcher.tick", error);
+		} finally {
+			this.scheduleTick();
+		}
+	}
+
+	private tickUnsafe(now: number): void {
+		const thresholds = this.opts.thresholds ?? DEFAULT_GRADIENT_THRESHOLDS;
+		const tickThreshold = this.opts.deadletterTickThreshold ?? 3;
+		const activeKeys = new Set<string>();
+
+		for (const run of this.opts.manifestCache.list(50)) {
+			if (run.status !== "running") continue;
+			// Bug #5 fix: if stateRoot doesn't exist, the run was pruned — skip it silently.
+			// This prevents stale "heartbeat dead" notifications for runs that no longer exist.
+			if (!fs.existsSync(run.stateRoot)) continue;
+			const loaded = loadRunManifestById(this.opts.cwd, run.runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency;
+			if (!loaded) continue;
+			// Defensive guard: cache may return stale "running" while disk says terminal.
+			// Re-read manifest from disk to verify before entering the heavy loop.
+			// This closes the false-positive alert loop when a run completes but the
+			// manifestCache.list(50) still serves a pre-completion snapshot.
+			if (loaded.manifest.status !== "running") continue;
+			for (const task of loaded.tasks) {
+				if (task.status !== "running") continue;
+				const key = `${run.runId}:${task.id}`;
+				activeKeys.add(key);
+				this.lastSeen.set(key, now);
+
+				// Check heartbeat staleness with lastActivityAt fallback
+				let elapsed = heartbeatAgeMs(task.heartbeat, now);
+				// PR #6 partial: use lastActivityAt as fallback when heartbeat is stale
+				// If heartbeat is stale but lastActivityAt is fresher, use activity age instead.
+				// This prevents false-positive dead detection for live-session tasks during long operations.
+				if (task.agentProgress?.lastActivityAt) {
+					const activityAt = new Date(task.agentProgress.lastActivityAt).getTime();
+					if (Number.isFinite(activityAt)) {
+						const activityAge = now - activityAt;
+						// Use activity age if it's fresher than heartbeat age
+						// (no upper bound - if agent has recent activity, trust it even if old)
+						if (activityAge < elapsed) {
+							elapsed = activityAge;
+						}
+					}
+				}
+				// PID liveness gate: if the worker process is still alive, downgrade
+				// "dead" to "stale". This prevents false positives when the LLM spends
+				// a long time generating a response (>5 min) without tool calls.
+				// A truly dead process will eventually be detected by exit handlers.
+				let isProcessAlive = false;
+				const workerPid = task.heartbeat?.pid ?? task.checkpoint?.childPid;
+				if (workerPid && workerPid > 0) {
+					try {
+						process.kill(workerPid, 0);
+						isProcessAlive = true;
+					} catch {
+						// Process is dead
+					}
+				}
+				let level: HeartbeatLevel =
+					elapsed > thresholds.deadMs
+						? "dead"
+						: elapsed > thresholds.staleMs
+							? "stale"
+							: elapsed > thresholds.warnMs
+								? "warn"
+								: "healthy";
+				if (level === "dead" && isProcessAlive) {
+					level = "stale";
+				}
+				// W8 fix: completion-artifact check — prevents false-positive "dead"
+				// during the exit-before-manifest-update race. When a worker process
+				// exits normally after completing its task, the result artifact is
+				// already on disk, but the manifest status update may lag by a few
+				// seconds (status + finishedAt are set atomically in task-runner.ts).
+				// If the result file exists, the task completed — downgrade to
+				// "stale" so the watcher doesn't fire a misleading "dead" notification
+				// for a task that already produced its output.
+				// W8-fix-v2 — path-traversal defense-in-depth. task.id is
+				// generated internally (e.g. "ts1", "ts2") but we still
+				// resolve the candidate path and verify it's strictly
+				// contained within <artifactsRoot>/results/. If task.id
+				// contained "../" or absolute path segments, the containment
+				// check fails and we skip the W8 check (fail-closed: don't
+				// accidentally treat a malicious task ID as "completed").
+				if (level === "dead" && !isProcessAlive && loaded.manifest.artifactsRoot) {
+					const resultsDir = path.resolve(loaded.manifest.artifactsRoot, "results");
+					const candidate = path.resolve(resultsDir, `${task.id}.txt`);
+					if (candidate.startsWith(resultsDir + path.sep) && fs.existsSync(candidate)) {
+						level = "stale";
+					}
+				}
+				this.opts.registry
+					.gauge("crew.heartbeat.staleness_ms", "Heartbeat elapsed since last seen, milliseconds")
+					.set({ runId: run.runId, taskId: task.id }, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				this.opts.registry
+					.counter("crew.heartbeat.level_total", "Heartbeat classifications by level")
+					.inc({ runId: run.runId, level });
+				const previous = this.lastLevel.get(key);
+				this.lastLevel.set(key, level);
+				if (level === "dead" && previous !== "dead") {
+					this.opts.registry.counter("crew.heartbeat.dead_total", "Dead heartbeat detections").inc({ runId: run.runId });
+					appendEvent(loaded.manifest.eventsPath, {
+						type: "crew.task.heartbeat_dead",
+						runId: run.runId,
+						taskId: task.id,
+						message: `Task ${task.id} heartbeat dead.`,
+						data: {
+							elapsedMs: Number.isFinite(elapsed) ? elapsed : undefined,
+						},
+					});
+					// W9 fix — prefix title with short run label (first 8 chars of runId)
+					// so ambient notifications are scannable when multiple runs are
+					// in flight. Full runId remains in the notification object.
+					const runLabel = run.runId.slice(0, 8);
+					this.opts.router.enqueue({
+						id: `dead_${run.runId}_${task.id}`,
+						severity: "warning",
+						source: "heartbeat-watcher",
+						runId: run.runId,
+						title: `[${runLabel}] Task ${task.id} heartbeat dead`,
+						body: "Background watcher detected a stuck worker.",
+					});
+					this.opts.onDead?.(run.runId, task.id, Number.isFinite(elapsed) ? elapsed : thresholds.deadMs);
+				}
+				if (level === "dead") {
+					const count = (this.consecutiveDead.get(key) ?? 0) + 1;
+					this.consecutiveDead.set(key, count);
+					if (count === tickThreshold) {
+						// 3.6 cooldown gate
+						const cooldown = this.opts.deadletterCooldownMs ?? 60_000;
+						const lastTrigger = this.lastDeadletterTriggerAt.get(key) ?? 0;
+						if (now - lastTrigger >= cooldown) {
+							this.lastDeadletterTriggerAt.set(key, now);
+							this.opts.onDeadletterTrigger?.(loaded.manifest, task.id);
+						}
+					}
+				} else {
+					this.consecutiveDead.delete(key);
+				}
+			}
+		}
+
+		// Cleanup: drop keys that were NOT in this tick's active set AND
+		// haven't been seen for > maxKeyAgeMs.  This covers runs that
+		// completed or fell out of the manifest cache's top-50 window.
+		const cutoff = now - this.maxKeyAgeMs;
+		for (const [key, ts] of this.lastSeen) {
+			if (!activeKeys.has(key) && ts < cutoff) {
+				this.lastLevel.delete(key);
+				this.consecutiveDead.delete(key);
+				this.lastSeen.delete(key);
+			}
+		}
+	}
+
+	dispose(): void {
+		if (this.timer) clearTimeout(this.timer);
+		this.timer = undefined;
+		this.lastLevel.clear();
+		this.consecutiveDead.clear();
+		this.lastSeen.clear();
+	}
+}

package/src/runtime/hidden-handoff.ts

@@ -0,0 +1,407 @@
+/**
+ * HiddenHandoffService - Sends hidden boomerang-handoff messages to parent agents.
+ *
+ * Based on pi-boomerang's triggerHiddenOrchestratorHandoff() pattern:
+ * - Sends customType: "boomerang-handoff" messages
+ * - Full handoff details in hidden message to parent
+ * - Orchestrator immediately reads summary
+ *
+ * @see docs/pi-boomerang-integration-plan.md
+ */
+
+import { logInternalError } from "../utils/internal-error.ts";
+import type { HandoffSummary } from "./handoff-manager.ts";
+
+/**
+ * Type of hidden handoff message.
+ */
+export type HiddenHandoffType = "boomerang-handoff" | "task-complete" | "context-ready";
+
+/**
+ * Hidden handoff message sent to parent agent.
+ */
+export interface HiddenHandoff {
+	type: HiddenHandoffType;
+	hidden: true;
+	content: HandoffContent;
+	metadata: HiddenHandoffMetadata;
+}
+
+/**
+ * Metadata for hidden handoff message.
+ */
+export interface HiddenHandoffMetadata {
+	taskId: string;
+	runId: string;
+	timestamp: number;
+	priority: HandoffPriority;
+}
+
+/**
+ * Priority level for hidden handoffs.
+ */
+export type HandoffPriority = "low" | "normal" | "high";
+
+/**
+ * Content of a hidden handoff message.
+ */
+export interface HandoffContent {
+	summary: string;
+	files: {
+		created: string[];
+		modified: string[];
+		deleted: string[];
+	};
+	decisions: {
+		rationale: string;
+		outcome: string;
+	}[];
+	nextSteps: string[];
+	metrics: {
+		tokens: number;
+		duration: number;
+	};
+}
+
+/**
+ * Options for HiddenHandoffService.
+ */
+export interface HiddenHandoffServiceOptions {
+	/** Custom mailbox service for sending messages */
+	mailbox?: HiddenHandoffMailbox;
+	/** Event emitter for handoff events */
+	eventEmitter?: HiddenHandoffEventEmitter;
+	/** Get parent agent ID callback */
+	getParentAgentId?: () => string;
+}
+
+/**
+ * Mailbox interface for sending hidden handoffs.
+ */
+export interface HiddenHandoffMailbox {
+	send(recipient: string, message: HiddenHandoff): void;
+}
+
+/**
+ * Event emitter for hidden handoff events.
+ */
+export interface HiddenHandoffEventEmitter {
+	emit(event: string, data: unknown): void;
+}
+
+/**
+ * Event data for hidden handoff sent event.
+ */
+export interface HiddenHandoffSentEventData {
+	summary: HandoffSummary;
+	recipient: string;
+	priority: HandoffPriority;
+}
+
+/**
+ * HiddenHandoffService sends hidden boomerang-handoff messages to parent agents.
+ * This enables agents to communicate progress and context without explicit user-visible output.
+ */
+export class HiddenHandoffService {
+	private mailbox: HiddenHandoffMailbox | null = null;
+	private eventEmitter: HiddenHandoffEventEmitter | null = null;
+	private getParentAgentIdFn: (() => string) | null = null;
+	private enabled = true;
+	// C7: Track rate limiting per recipient
+	private sendTimestamps = new Map<string, number[]>();
+	private readonly RATE_LIMIT_WINDOW_MS = 60000; // 1 minute
+	private readonly RATE_LIMIT_MAX_SENDS = 10; // Max handoffs per window
+
+	constructor(options: HiddenHandoffServiceOptions = {}) {
+		if (options.mailbox) {
+			this.mailbox = options.mailbox;
+		}
+		if (options.eventEmitter) {
+			this.eventEmitter = options.eventEmitter;
+		}
+		if (options.getParentAgentId) {
+			this.getParentAgentIdFn = options.getParentAgentId;
+		}
+	}
+
+	/**
+	 * Check if hidden handoff service is enabled.
+	 */
+	isEnabled(): boolean {
+		return this.enabled;
+	}
+
+	/**
+	 * Enable or disable hidden handoff service.
+	 */
+	setEnabled(enabled: boolean): void {
+		this.enabled = enabled;
+	}
+
+	/**
+	 * Set the mailbox service for sending messages.
+	 */
+	setMailbox(mailbox: HiddenHandoffMailbox): void {
+		this.mailbox = mailbox;
+	}
+
+	/**
+	 * Set the event emitter for events.
+	 */
+	setEventEmitter(eventEmitter: HiddenHandoffEventEmitter): void {
+		this.eventEmitter = eventEmitter;
+	}
+
+	/**
+	 * Set the function to get parent agent ID.
+	 */
+	setGetParentAgentId(fn: () => string): void {
+		this.getParentAgentIdFn = fn;
+	}
+
+	/**
+	 * Send a hidden handoff to the parent agent or specified recipient.
+	 *
+	 * @param summary - The handoff summary to send
+	 * @param options - Send options
+	 */
+	sendHandoff(summary: HandoffSummary, options: SendHandoffOptions = {}): void {
+		if (!this.enabled) {
+			return;
+		}
+
+		const priority = options.priority ?? this.inferPriority(summary);
+		const content = this.buildContent(summary);
+		const recipient = options.to ?? this.getParentAgentId();
+
+		if (!recipient) {
+			// No parent to send to, but we still emit the event
+			this.eventEmitter?.emit("handoff:sent_no_recipient", {
+				summary,
+				priority,
+			});
+			return;
+		}
+
+		// C7: Validate recipient is a reasonable agent ID
+		if (!this.isValidRecipient(recipient)) {
+			this.eventEmitter?.emit("handoff:invalid_recipient", {
+				recipient,
+				summary,
+			});
+			return;
+		}
+
+		// C7: Check rate limit
+		if (this.isRateLimited(recipient)) {
+			this.eventEmitter?.emit("handoff:rate_limited", {
+				recipient,
+				summary,
+			});
+			return;
+		}
+
+		const message: HiddenHandoff = {
+			type: options.customType ?? "boomerang-handoff",
+			hidden: true,
+			content,
+			metadata: {
+				taskId: summary.taskId,
+				runId: summary.runId,
+				timestamp: Date.now(),
+				priority,
+			},
+		};
+
+		if (this.mailbox) {
+			this.mailbox.send(recipient, message);
+		}
+
+		// C7: Record send for rate limiting
+		this.recordSend(recipient);
+
+		this.eventEmitter?.emit("handoff:sent", {
+			summary,
+			recipient,
+			priority,
+		});
+	}
+
+	/**
+	 * Send a hidden handoff immediately (fire and forget).
+	 */
+	sendHandoffAsync(summary: HandoffSummary, options?: SendHandoffOptions): void {
+		// Fire and forget - no await
+		try {
+			this.sendHandoff(summary, options);
+		} catch (error) {
+			// Log but don't throw
+			logInternalError("hidden-handoff.async", error, `taskId=${summary.taskId} runId=${summary.runId}`);
+		}
+	}
+
+	/**
+	 * Infer priority based on summary outcome.
+	 */
+	private inferPriority(summary: HandoffSummary): HandoffPriority {
+		if (summary.outcome === "failure") {
+			return "high";
+		}
+		if (summary.blockers.length > 0) {
+			return "normal";
+		}
+		if (summary.metrics.tokensUsed > 10000) {
+			return "normal";
+		}
+		return "low";
+	}
+
+	/**
+	 * Build handoff content from summary.
+	 */
+	private buildContent(summary: HandoffSummary): HandoffContent {
+		return {
+			summary: this.buildSummaryText(summary),
+			files: {
+				created: summary.filesCreated,
+				modified: summary.filesModified,
+				deleted: summary.filesDeleted,
+			},
+			decisions: summary.decisions.map((d) => ({
+				rationale: d.rationale,
+				outcome: d.outcome,
+			})),
+			nextSteps: summary.nextSteps,
+			metrics: {
+				tokens: summary.metrics.tokensUsed,
+				duration: summary.metrics.duration,
+			},
+		};
+	}
+
+	/**
+	 * Build summary text from summary.
+	 */
+	private buildSummaryText(summary: HandoffSummary): string {
+		const parts: string[] = [`Completed: ${summary.task}`, `Outcome: ${summary.outcome}`];
+
+		if (summary.filesCreated.length > 0) {
+			parts.push(`Files created: ${summary.filesCreated.join(", ")}`);
+		}
+		if (summary.filesModified.length > 0) {
+			parts.push(`Files modified: ${summary.filesModified.join(", ")}`);
+		}
+		if (summary.decisions.length > 0) {
+			parts.push(`Decisions: ${summary.decisions.length}`);
+		}
+		if (summary.blockers.length > 0) {
+			parts.push(`Blockers: ${summary.blockers.join("; ")}`);
+		}
+		if (summary.nextSteps.length > 0) {
+			parts.push(`Next steps: ${summary.nextSteps.join("; ")}`);
+		}
+
+		parts.push(`Tokens: ${summary.metrics.tokensUsed}`, `Duration: ${Math.round(summary.metrics.duration / 1000)}s`);
+
+		return parts.join("\n");
+	}
+
+	/**
+	 * Get parent agent ID from callback or context.
+	 */
+	private getParentAgentId(): string | undefined {
+		if (this.getParentAgentIdFn) {
+			return this.getParentAgentIdFn();
+		}
+		// Fallback: try to get from global context
+		const ctx = (globalThis as Record<string, unknown>).__piCrewContext;
+		if (ctx && typeof ctx === "object") {
+			return (ctx as Record<string, unknown>).parentAgentId as string | undefined;
+		}
+		return undefined;
+	}
+
+	/**
+	 * C7: Validate recipient is a reasonable agent ID.
+	 */
+	private isValidRecipient(recipient: string): boolean {
+		if (!recipient || typeof recipient !== "string") {
+			return false;
+		}
+		// Reasonable length for an agent ID
+		if (recipient.length < 1 || recipient.length > 256) {
+			return false;
+		}
+		// Only allow alphanumeric, hyphen, underscore, colon, and period
+		// This prevents injection in mailbox routing
+		if (!/^[a-zA-Z0-9_:.-]+$/.test(recipient)) {
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * C7: Check if recipient is rate limited.
+	 * Also cleans up empty recipient entries to prevent unbounded Map growth.
+	 */
+	private isRateLimited(recipient: string): boolean {
+		const now = Date.now();
+		const timestamps = this.sendTimestamps.get(recipient) ?? [];
+
+		// Filter out old timestamps outside the window
+		const recentTimestamps = timestamps.filter((t) => now - t < this.RATE_LIMIT_WINDOW_MS);
+
+		// HIGH-11: If no recent timestamps, remove the empty key to prevent unbounded growth
+		if (recentTimestamps.length === 0) {
+			this.sendTimestamps.delete(recipient);
+		} else {
+			this.sendTimestamps.set(recipient, recentTimestamps);
+		}
+
+		return recentTimestamps.length >= this.RATE_LIMIT_MAX_SENDS;
+	}
+
+	/**
+	 * C7: Record a send for rate limiting.
+	 */
+	private recordSend(recipient: string): void {
+		const now = Date.now();
+		const timestamps = this.sendTimestamps.get(recipient) ?? [];
+		timestamps.push(now);
+
+		// MEDIUM-14: Use RATE_LIMIT_WINDOW_MS consistently for both filter and record
+		const recentTimestamps = timestamps.filter((t) => now - t < this.RATE_LIMIT_WINDOW_MS);
+
+		this.sendTimestamps.set(recipient, recentTimestamps);
+	}
+
+	/**
+	 * Dispose of resources.
+	 * Call this when the service is no longer needed.
+	 */
+	dispose(): void {
+		this.mailbox = null;
+		this.eventEmitter = null;
+		this.getParentAgentIdFn = null;
+		this.sendTimestamps.clear();
+	}
+}
+
+/**
+ * Options for sending hidden handoffs.
+ */
+export interface SendHandoffOptions {
+	/** Recipient agent ID (defaults to parent) */
+	to?: string;
+	/** Priority level */
+	priority?: HandoffPriority;
+	/** Custom handoff type */
+	customType?: HiddenHandoffType;
+}
+
+/**
+ * Create a HiddenHandoffService with default options.
+ */
+export function createHiddenHandoffService(options?: HiddenHandoffServiceOptions): HiddenHandoffService {
+	return new HiddenHandoffService(options);
+}