pi-crew 0.1.43 → 0.1.45

package/src/runtime/session-usage.ts

@@ -1,79 +1,79 @@
-import * as fs from "node:fs";
-import type { UsageState } from "../state/types.ts";
-
-function asRecord(value: unknown): Record<string, unknown> | undefined {
-	return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
-}
-
-function numberField(obj: Record<string, unknown>, keys: string[]): number | undefined {
-	for (const key of keys) {
-		const value = obj[key];
-		if (typeof value === "number" && Number.isFinite(value)) return value;
-	}
-	return undefined;
-}
-
-function usageFromValue(value: unknown): UsageState | undefined {
-	const obj = asRecord(value);
-	if (!obj) return undefined;
-	const direct: UsageState = {
-		input: numberField(obj, ["input", "inputTokens", "input_tokens"]),
-		output: numberField(obj, ["output", "outputTokens", "output_tokens"]),
-		cacheRead: numberField(obj, ["cacheRead", "cache_read", "cacheReadTokens", "cache_read_tokens"]),
-		cacheWrite: numberField(obj, ["cacheWrite", "cache_write", "cacheWriteTokens", "cache_write_tokens"]),
-		cost: numberField(obj, ["cost", "costUsd", "cost_usd"]),
-		turns: numberField(obj, ["turns", "turnCount", "turn_count"]),
-	};
-	if (Object.values(direct).some((entry) => entry !== undefined)) return direct;
-	for (const key of ["usage", "tokenUsage", "tokens", "stats"]) {
-		const nested = usageFromValue(obj[key]);
-		if (nested) return nested;
-	}
-	const message = asRecord(obj.message);
-	return message ? usageFromValue(message.usage) : undefined;
-}
-
-function addUsage(total: UsageState, usage: UsageState): UsageState {
-	return {
-		input: (total.input ?? 0) + (usage.input ?? 0),
-		output: (total.output ?? 0) + (usage.output ?? 0),
-		cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0),
-		cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0),
-		cost: (total.cost ?? 0) + (usage.cost ?? 0),
-		turns: (total.turns ?? 0) + (usage.turns ?? 0),
-	};
-}
-
-function compactUsage(total: UsageState, foundKeys: Set<keyof UsageState>): UsageState | undefined {
-	if (foundKeys.size === 0) return undefined;
-	const compact: UsageState = {};
-	for (const key of foundKeys) compact[key] = total[key];
-	return compact;
-}
-
-export function parseSessionUsageFromJsonlText(text: string): UsageState | undefined {
-	let total: UsageState = {};
-	const foundKeys = new Set<keyof UsageState>();
-	for (const line of text.split(/\r?\n/)) {
-		const trimmed = line.trim();
-		if (!trimmed) continue;
-		try {
-			const usage = usageFromValue(JSON.parse(trimmed) as unknown);
-			if (!usage) continue;
-			for (const key of Object.keys(usage) as Array<keyof UsageState>) foundKeys.add(key);
-			total = addUsage(total, usage);
-		} catch {
-			// Session JSONL can contain partial/corrupt lines after interrupted workers.
-		}
-	}
-	return compactUsage(total, foundKeys);
-}
-
-export function parseSessionUsage(filePath: string): UsageState | undefined {
-	try {
-		if (!fs.existsSync(filePath)) return undefined;
-		return parseSessionUsageFromJsonlText(fs.readFileSync(filePath, "utf-8"));
-	} catch {
-		return undefined;
-	}
-}
+import * as fs from "node:fs";
+import type { UsageState } from "../state/types.ts";
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+	return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
+}
+
+function numberField(obj: Record<string, unknown>, keys: string[]): number | undefined {
+	for (const key of keys) {
+		const value = obj[key];
+		if (typeof value === "number" && Number.isFinite(value)) return value;
+	}
+	return undefined;
+}
+
+function usageFromValue(value: unknown): UsageState | undefined {
+	const obj = asRecord(value);
+	if (!obj) return undefined;
+	const direct: UsageState = {
+		input: numberField(obj, ["input", "inputTokens", "input_tokens"]),
+		output: numberField(obj, ["output", "outputTokens", "output_tokens"]),
+		cacheRead: numberField(obj, ["cacheRead", "cache_read", "cacheReadTokens", "cache_read_tokens"]),
+		cacheWrite: numberField(obj, ["cacheWrite", "cache_write", "cacheWriteTokens", "cache_write_tokens"]),
+		cost: numberField(obj, ["cost", "costUsd", "cost_usd"]),
+		turns: numberField(obj, ["turns", "turnCount", "turn_count"]),
+	};
+	if (Object.values(direct).some((entry) => entry !== undefined)) return direct;
+	for (const key of ["usage", "tokenUsage", "tokens", "stats"]) {
+		const nested = usageFromValue(obj[key]);
+		if (nested) return nested;
+	}
+	const message = asRecord(obj.message);
+	return message ? usageFromValue(message.usage) : undefined;
+}
+
+function addUsage(total: UsageState, usage: UsageState): UsageState {
+	return {
+		input: (total.input ?? 0) + (usage.input ?? 0),
+		output: (total.output ?? 0) + (usage.output ?? 0),
+		cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0),
+		cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0),
+		cost: (total.cost ?? 0) + (usage.cost ?? 0),
+		turns: (total.turns ?? 0) + (usage.turns ?? 0),
+	};
+}
+
+function compactUsage(total: UsageState, foundKeys: Set<keyof UsageState>): UsageState | undefined {
+	if (foundKeys.size === 0) return undefined;
+	const compact: UsageState = {};
+	for (const key of foundKeys) compact[key] = total[key];
+	return compact;
+}
+
+export function parseSessionUsageFromJsonlText(text: string): UsageState | undefined {
+	let total: UsageState = {};
+	const foundKeys = new Set<keyof UsageState>();
+	for (const line of text.split(/\r?\n/)) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		try {
+			const usage = usageFromValue(JSON.parse(trimmed) as unknown);
+			if (!usage) continue;
+			for (const key of Object.keys(usage) as Array<keyof UsageState>) foundKeys.add(key);
+			total = addUsage(total, usage);
+		} catch {
+			// Session JSONL can contain partial/corrupt lines after interrupted workers.
+		}
+	}
+	return compactUsage(total, foundKeys);
+}
+
+export function parseSessionUsage(filePath: string): UsageState | undefined {
+	try {
+		if (!fs.existsSync(filePath)) return undefined;
+		return parseSessionUsageFromJsonlText(fs.readFileSync(filePath, "utf-8"));
+	} catch {
+		return undefined;
+	}
+}

package/src/runtime/sidechain-output.ts

@@ -1,29 +1,29 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { redactSecrets } from "../utils/redaction.ts";
-
-export interface SidechainEntry {
-	isSidechain: true;
-	agentId: string;
-	type: string;
-	message: unknown;
-	timestamp: string;
-	cwd: string;
-}
-
-export function writeSidechainEntry(filePath: string, entry: Omit<SidechainEntry, "isSidechain" | "timestamp">): void {
-	fs.mkdirSync(path.dirname(filePath), { recursive: true });
-	fs.appendFileSync(filePath, `${JSON.stringify(redactSecrets({ isSidechain: true, timestamp: new Date().toISOString(), ...entry }))}\n`, "utf-8");
-}
-
-export function sidechainOutputPath(stateRoot: string, taskId: string): string {
-	return path.join(stateRoot, "agents", taskId, "sidechain.output.jsonl");
-}
-
-export function eventToSidechainType(event: unknown): string | undefined {
-	if (!event || typeof event !== "object" || Array.isArray(event)) return undefined;
-	const type = (event as { type?: unknown }).type;
-	if (type === "message_start" || type === "message_update" || type === "message_end") return "message";
-	if (type === "tool_execution_start" || type === "tool_execution_update" || type === "tool_execution_end") return "tool";
-	return typeof type === "string" ? type : undefined;
-}
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { redactSecrets } from "../utils/redaction.ts";
+
+export interface SidechainEntry {
+	isSidechain: true;
+	agentId: string;
+	type: string;
+	message: unknown;
+	timestamp: string;
+	cwd: string;
+}
+
+export function writeSidechainEntry(filePath: string, entry: Omit<SidechainEntry, "isSidechain" | "timestamp">): void {
+	fs.mkdirSync(path.dirname(filePath), { recursive: true });
+	fs.appendFileSync(filePath, `${JSON.stringify(redactSecrets({ isSidechain: true, timestamp: new Date().toISOString(), ...entry }))}\n`, "utf-8");
+}
+
+export function sidechainOutputPath(stateRoot: string, taskId: string): string {
+	return path.join(stateRoot, "agents", taskId, "sidechain.output.jsonl");
+}
+
+export function eventToSidechainType(event: unknown): string | undefined {
+	if (!event || typeof event !== "object" || Array.isArray(event)) return undefined;
+	const type = (event as { type?: unknown }).type;
+	if (type === "message_start" || type === "message_update" || type === "message_end") return "message";
+	if (type === "tool_execution_start" || type === "tool_execution_update" || type === "tool_execution_end") return "tool";
+	return typeof type === "string" ? type : undefined;
+}

package/src/runtime/stale-reconciler.ts

@@ -0,0 +1,199 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { checkProcessLiveness } from "./process-status.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import { writeAtomicJson } from "../utils/atomic-write.ts";
+
+/**
+ * Result of reconciling a single stale run.
+ */
+export interface ReconcileResult {
+	runId: string;
+	/** What was found and what action was taken */
+	verdict: "healthy" | "result_exists" | "pid_dead" | "pid_alive_stale" | "no_status";
+	/** Whether repair was applied */
+	repaired: boolean;
+	/** Human-readable detail */
+	detail: string;
+}
+
+const STALE_ALIVE_PID_MS = 24 * 60 * 60 * 1000; // 24 hours
+const ACTIVE_EVIDENCE_TTL_MS = 5 * 60 * 1000;
+
+/**
+ * Phase 1: Check if a result file already exists for the run.
+ * If so, the run completed but status wasn't updated — repair it.
+ */
+function checkResultFile(
+	manifest: TeamRunManifest,
+	tasks: TeamTaskState[],
+): { found: boolean; repaired: boolean } {
+	// Check if all tasks already have terminal status (result was written but manifest wasn't updated)
+	const allTerminal = tasks.length > 0 && tasks.every(
+		(t) => t.status === "completed" || t.status === "failed" || t.status === "cancelled" || t.status === "skipped",
+	);
+	if (allTerminal) {
+		return { found: true, repaired: false };
+	}
+	return { found: false, repaired: false };
+}
+
+/**
+ * Phase 2: Check PID liveness.
+ */
+function checkPidLiveness(pid: number | undefined): {
+	alive: boolean;
+	detail: string;
+} {
+	if (pid === undefined || !Number.isInteger(pid) || pid <= 0) {
+		return { alive: false, detail: "no pid recorded" };
+	}
+	const liveness = checkProcessLiveness(pid);
+	return { alive: liveness.alive, detail: liveness.detail };
+}
+
+/**
+ * Phase 3: For dead PIDs, repair immediately.
+ * For alive PIDs, only mark stale if status hasn't updated in STALE_ALIVE_PID_MS.
+ */
+function evaluateStaleness(
+	manifest: TeamRunManifest,
+	pidAlive: boolean,
+	now: number,
+): { stale: boolean; reason: string } {
+	if (!pidAlive) {
+		return { stale: true, reason: "pid_dead" };
+	}
+	const updatedAt = new Date(manifest.updatedAt).getTime();
+	if (!Number.isFinite(updatedAt)) {
+		return { stale: false, reason: "updated_at_invalid" };
+	}
+	if (now - updatedAt > STALE_ALIVE_PID_MS) {
+		return { stale: true, reason: `alive_but_stale_${Math.round((now - updatedAt) / 3600_000)}h` };
+	}
+	return { stale: false, reason: "alive_and_recent" };
+}
+
+function hasRecentActiveEvidence(tasks: TeamTaskState[], now: number): boolean {
+	return tasks.some((task) => {
+		if (task.status !== "running" && task.status !== "waiting") return false;
+		const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
+		if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= ACTIVE_EVIDENCE_TTL_MS) return true;
+		const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
+		return Number.isFinite(activityAt) && now - activityAt <= ACTIVE_EVIDENCE_TTL_MS;
+	});
+}
+
+/**
+ * Repair a stale run by marking it as failed and cancelling running tasks.
+ */
+function repairStaleRun(
+	manifest: TeamRunManifest,
+	tasks: TeamTaskState[],
+	reason: string,
+): TeamTaskState[] {
+	const now = new Date().toISOString();
+	const repairedTasks = tasks.map((task) => {
+		if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
+			return {
+				...task,
+				status: "cancelled" as const,
+				finishedAt: now,
+				error: `Stale run reconciled: ${reason}`,
+			};
+		}
+		return task;
+	});
+
+	// Write repaired tasks atomically
+	const tasksPath = manifest.tasksPath;
+	if (tasksPath) {
+		try {
+			writeAtomicJson(tasksPath, repairedTasks);
+		} catch (error) {
+			logInternalError("stale-reconciler.repair-tasks", error, `runId=${manifest.runId}`);
+		}
+	}
+
+	return repairedTasks;
+}
+
+/**
+ * Three-phase stale run reconciliation.
+ *
+ * 1. Check if result already exists → use it
+ * 2. Check PID liveness
+ * 3. Dead PID → repair immediately; alive PID → only fail if stale > 24h
+ */
+export function reconcileStaleRun(
+	manifest: TeamRunManifest,
+	tasks: TeamTaskState[],
+	now = Date.now(),
+): ReconcileResult {
+	const runId = manifest.runId;
+
+	// Phase 1: Check if results already exist
+	const phase1 = checkResultFile(manifest, tasks);
+	if (phase1.found) {
+		return {
+			runId,
+			verdict: "result_exists",
+			repaired: false,
+			detail: "All tasks already terminal — no repair needed",
+		};
+	}
+
+	// Phase 2: Check PID liveness
+	const pid = manifest.async?.pid;
+	const pidStatus = checkPidLiveness(pid);
+
+	if (pidStatus.detail === "no pid recorded") {
+		// No async PID may be a foreground/live run. Preserve it if task heartbeat
+		// or agent progress proves active work even when manifest.updatedAt is old.
+		if (hasRecentActiveEvidence(tasks, now)) {
+			return {
+				runId,
+				verdict: "no_status",
+				repaired: false,
+				detail: "No PID recorded, but recent task heartbeat/progress exists; not repairing",
+			};
+		}
+		const updatedAt = new Date(manifest.updatedAt).getTime();
+		if (Number.isFinite(updatedAt) && now - updatedAt > STALE_ALIVE_PID_MS) {
+			const repaired = repairStaleRun(manifest, tasks, "no_pid_stale");
+			return {
+				runId,
+				verdict: "no_status",
+				repaired: true,
+				detail: `No PID; stale ${Math.round((now - updatedAt) / 3600_000)}h; repaired ${repaired.filter((t) => t.status === "cancelled").length} tasks`,
+			};
+		}
+		return {
+			runId,
+			verdict: "no_status",
+			repaired: false,
+			detail: "No PID recorded; not stale enough to repair",
+		};
+	}
+
+	// Phase 3: Evaluate staleness
+	const staleness = evaluateStaleness(manifest, pidStatus.alive, now);
+	if (!staleness.stale) {
+		return {
+			runId,
+			verdict: "healthy",
+			repaired: false,
+			detail: `PID ${pid}: ${pidStatus.detail}, ${staleness.reason}`,
+		};
+	}
+
+	// Repair
+	const repaired = repairStaleRun(manifest, tasks, staleness.reason);
+	return {
+		runId,
+		verdict: pidStatus.alive ? "pid_alive_stale" : "pid_dead",
+		repaired: true,
+		detail: `PID ${pid}: ${pidStatus.detail}; ${staleness.reason}; repaired ${repaired.filter((t) => t.status === "cancelled").length} tasks`,
+	};
+}

package/src/runtime/supervisor-contact.ts

@@ -0,0 +1,59 @@
+import type { TeamRunManifest } from "../state/types.ts";
+import { appendEvent } from "../state/event-log.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+
+export interface SupervisorContactPayload {
+	runId: string;
+	taskId: string;
+	reason: "decision_needed" | "clarification" | "approval" | "error_escalation" | "custom";
+	message: string;
+	data?: Record<string, unknown>;
+	timestamp: string;
+}
+
+/**
+ * Record a supervisor contact event from a child task.
+ * This represents a child→parent communication where the child needs
+ * a decision, clarification, or approval to continue.
+ */
+export function recordSupervisorContact(manifest: TeamRunManifest, payload: Omit<SupervisorContactPayload, "timestamp">): void {
+	const fullPayload: SupervisorContactPayload = {
+		...payload,
+		timestamp: new Date().toISOString(),
+	};
+	try {
+		appendEvent(manifest.eventsPath, {
+			type: "supervisor.contact",
+			runId: manifest.runId,
+			taskId: payload.taskId,
+			data: fullPayload as unknown as Record<string, unknown>,
+		});
+	} catch (error) {
+		logInternalError("supervisor-contact.record", error, `runId=${manifest.runId} taskId=${payload.taskId}`);
+	}
+}
+
+/**
+ * Parse a supervisor contact request from child Pi stdout.
+ * Detects structured JSON lines with type "supervisor_contact".
+ */
+export function parseSupervisorContactFromLine(line: string): Omit<SupervisorContactPayload, "timestamp" | "runId"> | undefined {
+	if (!line.trim()) return undefined;
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(line);
+	} catch {
+		return undefined;
+	}
+	if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
+	const record = parsed as Record<string, unknown>;
+	if (record.type !== "supervisor_contact" && record.type !== "crew_supervisor_contact") return undefined;
+	return {
+		taskId: typeof record.taskId === "string" ? record.taskId : "",
+		reason: typeof record.reason === "string" && ["decision_needed", "clarification", "approval", "error_escalation", "custom"].includes(record.reason)
+			? record.reason as SupervisorContactPayload["reason"]
+			: "custom",
+		message: typeof record.message === "string" ? record.message : String(record.message ?? ""),
+		data: record.data && typeof record.data === "object" && !Array.isArray(record.data) ? record.data as Record<string, unknown> : undefined,
+	};
+}

package/src/runtime/task-display.ts

@@ -1,38 +1,38 @@
-import type { TeamTaskState } from "../state/types.ts";
-import type { CrewAgentRecord, CrewRuntimeKind } from "./crew-agent-runtime.ts";
-import { recordFromTask } from "./crew-agent-records.ts";
-import type { TeamRunManifest } from "../state/types.ts";
-
-export function shouldMaterializeAgent(task: TeamTaskState): boolean {
-	return task.status !== "queued" && task.status !== "skipped";
-}
-
-export function recordsForMaterializedTasks(manifest: TeamRunManifest, tasks: TeamTaskState[], runtime: CrewRuntimeKind): CrewAgentRecord[] {
-	return tasks.filter(shouldMaterializeAgent).map((task) => recordFromTask(manifest, task, runtime));
-}
-
-export function taskById(tasks: TeamTaskState[]): Map<string, TeamTaskState> {
-	const map = new Map<string, TeamTaskState>();
-	for (const task of tasks) {
-		map.set(task.id, task);
-		if (task.stepId) map.set(task.stepId, task);
-	}
-	return map;
-}
-
-export function waitingReason(task: TeamTaskState, tasks: TeamTaskState[]): string | undefined {
-	if (task.status !== "queued") return undefined;
-	const byId = taskById(tasks);
-	const waiting = task.dependsOn.map((id) => byId.get(id)?.id ?? id).filter((id) => byId.get(id)?.status !== "completed");
-	if (waiting.length === 0) return "ready";
-	return `waiting for ${waiting.join(", ")}`;
-}
-
-export function formatTaskGraphLines(tasks: TeamTaskState[]): string[] {
-	if (tasks.length === 0) return ["- (none)"];
-	return tasks.map((task) => {
-		const icon = task.status === "completed" ? "✓" : task.status === "running" ? "⠋" : task.status === "failed" ? "✗" : task.status === "cancelled" || task.status === "skipped" ? "■" : "◦";
-		const wait = waitingReason(task, tasks);
-		return `- ${icon} ${task.id} [${task.status}] ${task.role}->${task.agent}${wait && wait !== "ready" ? ` (${wait})` : ""}`;
-	});
-}
+import type { TeamTaskState } from "../state/types.ts";
+import type { CrewAgentRecord, CrewRuntimeKind } from "./crew-agent-runtime.ts";
+import { recordFromTask } from "./crew-agent-records.ts";
+import type { TeamRunManifest } from "../state/types.ts";
+
+export function shouldMaterializeAgent(task: TeamTaskState): boolean {
+	return task.status !== "queued" && task.status !== "skipped";
+}
+
+export function recordsForMaterializedTasks(manifest: TeamRunManifest, tasks: TeamTaskState[], runtime: CrewRuntimeKind): CrewAgentRecord[] {
+	return tasks.filter(shouldMaterializeAgent).map((task) => recordFromTask(manifest, task, runtime));
+}
+
+export function taskById(tasks: TeamTaskState[]): Map<string, TeamTaskState> {
+	const map = new Map<string, TeamTaskState>();
+	for (const task of tasks) {
+		map.set(task.id, task);
+		if (task.stepId) map.set(task.stepId, task);
+	}
+	return map;
+}
+
+export function waitingReason(task: TeamTaskState, tasks: TeamTaskState[]): string | undefined {
+	if (task.status !== "queued") return undefined;
+	const byId = taskById(tasks);
+	const waiting = task.dependsOn.map((id) => byId.get(id)?.id ?? id).filter((id) => byId.get(id)?.status !== "completed");
+	if (waiting.length === 0) return "ready";
+	return `waiting for ${waiting.join(", ")}`;
+}
+
+export function formatTaskGraphLines(tasks: TeamTaskState[]): string[] {
+	if (tasks.length === 0) return ["- (none)"];
+	return tasks.map((task) => {
+		const icon = task.status === "completed" ? "✓" : task.status === "running" ? "⠋" : task.status === "failed" ? "✗" : task.status === "cancelled" || task.status === "skipped" ? "■" : "◦";
+		const wait = waitingReason(task, tasks);
+		return `- ${icon} ${task.id} [${task.status}] ${task.role}->${task.agent}${wait && wait !== "ready" ? ` (${wait})` : ""}`;
+	});
+}