pi-crew 0.9.53 → 0.9.55

package/src/runtime/checkpoint.ts

@@ -0,0 +1,291 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { atomicWriteFile } from "../state/atomic-write.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import { projectCrewRoot } from "../utils/paths.ts";
+import { assertSafePathId } from "../utils/safe-paths.ts";
+
+export interface Checkpoint {
+	runId: string;
+	taskId: string;
+	step: number;
+	context: string;
+	progress: string;
+	savedAt: number;
+	agentId: string;
+	agentModel?: string;
+}
+
+export interface CheckpointStore {
+	save(checkpoint: Checkpoint): void;
+	load(runId: string, taskId: string): Checkpoint | null;
+	delete(runId: string, taskId: string): void;
+	list(runId: string): Checkpoint[];
+	hasCheckpoint(runId: string, taskId: string): boolean;
+}
+
+interface CheckpointEntry {
+	checkpoints: Record<string, Checkpoint>;
+}
+
+/**
+ * File-based checkpoint store.
+ * Saves checkpoints as JSON files in .crew/state/runs/<runId>/checkpoints/
+ */
+export class FileCheckpointStore implements CheckpointStore {
+	private readonly stateRoot: string;
+
+	constructor(stateRoot: string) {
+		this.stateRoot = stateRoot;
+	}
+
+	private checkpointDir(): string {
+		return path.join(this.stateRoot, "checkpoints");
+	}
+
+	private checkpointPath(taskId: string): string {
+		return path.join(this.checkpointDir(), `${taskId}.json`);
+	}
+
+	private ensureDir(): void {
+		const dir = this.checkpointDir();
+		if (!fs.existsSync(dir)) {
+			fs.mkdirSync(dir, { recursive: true });
+		}
+	}
+
+	save(checkpoint: Checkpoint): void {
+		// Validate taskId to prevent path traversal: the taskId is used to
+		// build a file path under this.checkpointDir(). Without validation, a
+		// malicious or buggy taskId like "../../../etc/passwd" could escape
+		// the checkpoints directory.
+		assertSafePathId("taskId", checkpoint.taskId);
+		this.ensureDir();
+		const p = this.checkpointPath(checkpoint.taskId);
+		// Atomic write: write to temp file first, then rename, then fsync parent.
+		// This guarantees either the old file or the new file, never a partial
+		// write, even on network filesystems or certain journal modes.
+		//
+		// Round 22 (BUG 1): the temp filename MUST be unique per save call.
+		// Previously a fixed '.tmp.checkpoint' was shared across ALL concurrent
+		// saves; pi-crew's multi-process architecture (main + detached background
+		// workers each checkpointing their own tasks) made this realistic: two
+		// processes writing '.tmp.checkpoint' at once → one's rename picks up the
+		// other's data (silent corruption) and the second rename hits ENOENT
+		// (silent data loss). Including taskId + pid + timestamp guarantees
+		// uniqueness across processes and across tasks.
+		// atomicWriteFile handles temp + write + fsync(data) + rename + fsync(parent)
+		// with a UUID-based unique temp name (cross-process safe). The previous
+		// hand-rolled version missed data-fsync (ST-5), risking empty/stale content
+		// after power loss.
+		atomicWriteFile(p, JSON.stringify(checkpoint, null, 2));
+	}
+
+	load(runId: string, taskId: string): Checkpoint | null {
+		assertSafePathId("taskId", taskId);
+		const p = this.checkpointPath(taskId);
+		if (!fs.existsSync(p)) return null;
+
+		try {
+			const data = JSON.parse(fs.readFileSync(p, "utf-8")) as Checkpoint;
+			// Verify it's for the correct run
+			if (data.runId !== runId) return null;
+			return data;
+		} catch {
+			// File existed but JSON was corrupt — log and rename for later inspection
+			logInternalError("checkpoint-load", new Error("JSON parse failed"), `file=${p}`);
+			try {
+				fs.renameSync(p, `${p}.corrupt.${Date.now()}`);
+			} catch {
+				// Best effort — ignore rename failure
+			}
+			return null;
+		}
+	}
+
+	delete(runId: string, taskId: string): void {
+		assertSafePathId("taskId", taskId);
+		const p = this.checkpointPath(taskId);
+		if (fs.existsSync(p)) {
+			try {
+				const data = JSON.parse(fs.readFileSync(p, "utf-8")) as Checkpoint;
+				if (data.runId === runId) {
+					fs.unlinkSync(p);
+				}
+			} catch {
+				// File existed but couldn't read — delete it anyway
+				try {
+					fs.unlinkSync(p);
+				} catch {
+					/* ignore */
+				}
+			}
+		}
+	}
+
+	list(runId: string): Checkpoint[] {
+		const dir = this.checkpointDir();
+		if (!fs.existsSync(dir)) return [];
+
+		return fs
+			.readdirSync(dir)
+			.filter((f) => f.endsWith(".json"))
+			.map((f) => {
+				try {
+					return JSON.parse(fs.readFileSync(path.join(dir, f), "utf-8")) as Checkpoint;
+				} catch {
+					return null;
+				}
+			})
+			.filter((c): c is Checkpoint => c !== null && c.runId === runId);
+	}
+
+	hasCheckpoint(runId: string, taskId: string): boolean {
+		return this.load(runId, taskId) !== null;
+	}
+}
+
+const MAX_STORES = 100;
+const _stores = new Map<string, FileCheckpointStore>();
+
+/**
+ * Get checkpoint store for a run's state root.
+ * Uses LRU eviction when the store exceeds MAX_STORES entries.
+ */
+export function getCheckpointStore(stateRoot: string): CheckpointStore {
+	if (!_stores.has(stateRoot)) {
+		if (_stores.size >= MAX_STORES) {
+			// Evict the oldest entry (first in insertion order)
+			const oldestKey = _stores.keys().next().value;
+			if (oldestKey !== undefined) {
+				_stores.delete(oldestKey);
+			}
+		}
+		_stores.set(stateRoot, new FileCheckpointStore(stateRoot));
+	}
+	return _stores.get(stateRoot)!;
+}
+
+/**
+ * Clear all checkpoint stores (for testing).
+ */
+export function clearCheckpointStores(): void {
+	_stores.clear();
+}
+
+/**
+ * Save a checkpoint during agent execution.
+ */
+export function saveCheckpoint(
+	runId: string,
+	taskId: string,
+	step: number,
+	context: string,
+	progress: string,
+	agentId: string,
+	agentModel?: string,
+	cwd?: string,
+): void {
+	// Validate both runId and taskId to prevent path traversal: these are
+	// used to build the file path under <crewRoot>/state/runs/<runId>/checkpoints/<taskId>.json.
+	assertSafePathId("runId", runId);
+	assertSafePathId("taskId", taskId);
+	const checkpoint: Checkpoint = {
+		runId,
+		taskId,
+		step,
+		context,
+		progress,
+		savedAt: Date.now(),
+		agentId,
+		agentModel,
+	};
+
+	// State root is parent of checkpoints dir. Use projectCrewRoot() so the
+	// path lands in .pi/teams/state/runs/ for .pi-based projects (issue #29).
+	const stateRoot = path.join(projectCrewRoot(cwd ?? process.cwd()), "state", "runs", runId);
+	const store = getCheckpointStore(stateRoot);
+	store.save(checkpoint);
+}
+
+/**
+ * Load a checkpoint for resuming.
+ */
+export function loadCheckpoint(runId: string, taskId: string, cwd?: string): Checkpoint | null {
+	assertSafePathId("runId", runId);
+	assertSafePathId("taskId", taskId);
+	const stateRoot = path.join(projectCrewRoot(cwd ?? process.cwd()), "state", "runs", runId);
+	const store = getCheckpointStore(stateRoot);
+	return store.load(runId, taskId);
+}
+
+/**
+ * Delete a checkpoint after successful completion.
+ */
+export function clearCheckpoint(runId: string, taskId: string, cwd?: string): void {
+	assertSafePathId("runId", runId);
+	assertSafePathId("taskId", taskId);
+	const stateRoot = path.join(projectCrewRoot(cwd ?? process.cwd()), "state", "runs", runId);
+	const store = getCheckpointStore(stateRoot);
+	store.delete(runId, taskId);
+}
+
+/**
+ * Check if a checkpoint exists for a task.
+ */
+export function hasCheckpoint(runId: string, taskId: string, cwd?: string): boolean {
+	assertSafePathId("runId", runId);
+	assertSafePathId("taskId", taskId);
+	const stateRoot = path.join(projectCrewRoot(cwd ?? process.cwd()), "state", "runs", runId);
+	const store = getCheckpointStore(stateRoot);
+	return store.hasCheckpoint(runId, taskId);
+}
+
+/**
+ * List all checkpoints for a run.
+ */
+export function listCheckpoints(runId: string, cwd?: string): Checkpoint[] {
+	assertSafePathId("runId", runId);
+	const stateRoot = path.join(projectCrewRoot(cwd ?? process.cwd()), "state", "runs", runId);
+	const store = getCheckpointStore(stateRoot);
+	return store.list(runId);
+}
+
+/**
+ * Format a checkpoint for display.
+ */
+export function formatCheckpoint(checkpoint: Checkpoint): string {
+	return [
+		`## Checkpoint: ${checkpoint.taskId}`,
+		"",
+		`**Agent:** ${checkpoint.agentId}`,
+		checkpoint.agentModel ? `**Model:** ${checkpoint.agentModel}` : "",
+		"",
+		`**Progress:** ${checkpoint.progress}`,
+		"",
+		`**Step:** ${checkpoint.step}`,
+		`**Saved:** ${new Date(checkpoint.savedAt).toISOString()}`,
+		"",
+		`**Context:** ${checkpoint.context.slice(0, 300)}${checkpoint.context.length > 300 ? "..." : ""}`,
+	]
+		.filter(Boolean)
+		.join("\n");
+}
+
+/**
+ * Format all checkpoints for a run.
+ */
+export function formatAllCheckpoints(runId: string, cwd?: string): string {
+	const checkpoints = listCheckpoints(runId, cwd);
+	if (checkpoints.length === 0) {
+		return `No checkpoints found for run ${runId}`;
+	}
+
+	return [
+		`# Checkpoints: ${runId}`,
+		"",
+		...checkpoints.map((cp, i) => `${i + 1}. **${cp.taskId}** — ${cp.progress} (${new Date(cp.savedAt).toLocaleString()})`),
+		"",
+		`Use \`team action='resume' runId=${runId} taskId=<taskId>\` to resume.`,
+	].join("\n");
+}

package/src/runtime/child-pi-constants.ts

@@ -0,0 +1,42 @@
+/**
+ * child-pi-constants.ts — Shared timing and capture constants for child-pi runtime.
+ *
+ * Extracted from child-pi.ts (H-7 decomposition, step 3). Zero behavior change.
+ *
+ * These constants are shared between runChildPi (in child-pi.ts) and the kill
+ * helpers (in child-pi-kill.ts). Centralizing them here avoids circular imports
+ * and makes the timing budget configurable from one place.
+ */
+
+import { DEFAULT_CHILD_PI } from "../config/defaults.ts";
+
+/** Post-exit window during which stdio is guarded against late writes. */
+export const POST_EXIT_STDIO_GUARD_MS = DEFAULT_CHILD_PI.postExitStdioGuardMs;
+
+/** Maximum time to wait for a final assistant event after the last stdout byte. */
+export const FINAL_DRAIN_MS = DEFAULT_CHILD_PI.finalDrainMs;
+
+/** Time after SIGTERM to escalate to SIGKILL. */
+export const HARD_KILL_MS = DEFAULT_CHILD_PI.hardKillMs;
+
+/** Maximum time with no output before the child is considered unresponsive. */
+export const RESPONSE_TIMEOUT_MS = DEFAULT_CHILD_PI.responseTimeoutMs;
+
+/**
+ * Maximum size (bytes) for the ChildPiLineObserver's line accumulation buffer.
+ * When exceeded, the buffer is force-flushed to prevent unbounded memory growth
+ * from chatty child processes that produce output without newlines.
+ */
+export const MAX_LINE_BUFFER_BYTES = 1024 * 1024; // 1 MB
+
+/** Maximum characters for assistant text fragments in compacted events. */
+export const MAX_ASSISTANT_TEXT_CHARS = DEFAULT_CHILD_PI.maxAssistantTextChars;
+
+/** Maximum characters for tool-result fragments in compacted events. */
+export const MAX_TOOL_RESULT_CHARS = DEFAULT_CHILD_PI.maxToolResultChars;
+
+/** Maximum characters for tool-input fragments in compacted events. */
+export const MAX_TOOL_INPUT_CHARS = DEFAULT_CHILD_PI.maxToolInputChars;
+
+/** Maximum characters for general compactable content (used by TruncationStage). */
+export const MAX_COMPACT_CONTENT_CHARS = DEFAULT_CHILD_PI.maxCompactContentChars;

package/src/runtime/child-pi-kill.ts

@@ -0,0 +1,180 @@
+/**
+ * child-pi-kill.ts — Kill/escape path for child Pi worker processes.
+ *
+ * Extracted from child-pi.ts (H-7 decomposition, step 2). Zero behavior change.
+ *
+ * Responsibilities:
+ *   - appendBoundedTail(): bounded concatenation with truncation marker.
+ *   - killProcessPid(): send SIGTERM → SIGKILL escalation, with Windows taskkill fallback.
+ *   - killProcessTree(): log+kill a child by pid, optionally attach exit listener to clear timer.
+ *   - terminateActiveChildPiProcesses(): kill all known active children.
+ *   - registerActiveChild()/unregisterActiveChild(): book-keeping for the active-children Map.
+ *   - Periodic zombie reaper (setInterval) that drops dead entries.
+ *
+ * The activeChildrenMap and hardKillTimers Map are kept here (not re-exported)
+ * because they are tightly coupled to the kill lifecycle.
+ */
+
+import type { ChildProcess } from "node:child_process";
+import { spawn } from "node:child_process";
+import { DEFAULT_CHILD_PI } from "../config/defaults.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import { HARD_KILL_MS } from "./child-pi-constants.ts";
+import { TailCaptureStage } from "./compact-stages/tail-capture-stage.ts";
+
+const MAX_CAPTURE_BYTES = DEFAULT_CHILD_PI.maxCaptureBytes;
+
+// Active children bookkeeping. Mutated by registerActiveChild/unregisterActiveChild.
+const activeChildProcesses = new Map<number, ChildProcess>();
+const childHardKillTimers = new Map<number, NodeJS.Timeout>();
+
+// Periodic cleanup of dead child process entries to prevent memory leaks.
+// If a child process never emits exit/close (zombie), the entry would leak.
+setInterval(() => {
+	for (const [pid, child] of activeChildProcesses) {
+		try {
+			process.kill(pid, 0); // Throws ESRCH if dead
+		} catch {
+			activeChildProcesses.delete(pid);
+		}
+	}
+}, 60_000).unref();
+
+/** Register a newly-spawned child so it can be tracked + killed on shutdown. */
+export function registerActiveChild(pid: number, child: ChildProcess): void {
+	activeChildProcesses.set(pid, child);
+}
+
+/** Remove a child from the active set once it has exited (or before kill). */
+export function unregisterActiveChild(pid: number): void {
+	activeChildProcesses.delete(pid);
+}
+
+/** Clear the SIGKILL escalation timer for a pid (called on early exit). */
+export function clearHardKillTimer(pid: number | undefined): void {
+	if (pid === undefined) return;
+	const timer = childHardKillTimers.get(pid);
+	if (timer) {
+		clearTimeout(timer);
+		childHardKillTimers.delete(pid);
+	}
+}
+
+/**
+ * Append `chunk` to `current`, keeping the result within `maxBytes`. When the
+ * cap is exceeded, returns the tail (most recent bytes) prefixed with a marker
+ * showing how much was dropped. Delegates to TailCaptureStage so the logic
+ * stays consistent with the rest of the compaction pipeline.
+ */
+export function appendBoundedTail(current: string, chunk: string, maxBytes = MAX_CAPTURE_BYTES): string {
+	return new TailCaptureStage({
+		maxBytes,
+		marker: `[pi-crew captured output truncated to last ${Math.round(maxBytes / 1024)} KiB]`,
+	}).apply(current + chunk);
+}
+
+function spawnTaskkillSafe(pid: number): void {
+	// B6: spawn taskkill and attach an 'error' listener. spawn() emits ENOENT/EACCES
+	// via the 'error' event — unhandled these would become uncaughtException and
+	// can crash the parent as an uncaught exception. taskkill is a standard Windows
+	// utility and rarely fails to spawn, but be defensive.
+	try {
+		const taskkillChild = spawn("taskkill", ["/pid", String(pid), "/t", "/f"], {
+			stdio: "ignore",
+			detached: false,
+		});
+		taskkillChild.on("error", (err) => {
+			logInternalError("child-pi.taskkill-spawn-error", err instanceof Error ? err : new Error(String(err)), `pid=${pid}`);
+		});
+		taskkillChild.unref();
+	} catch (error) {
+		logInternalError("child-pi.taskkill-sync-error", error instanceof Error ? error : new Error(String(error)), `pid=${pid}`);
+	}
+}
+
+export function killProcessPid(pid: number): void {
+	if (!Number.isInteger(pid) || pid <= 0) return;
+	try {
+		if (process.platform === "win32") {
+			// 3.8: Windows path uses taskkill /T /F (force kill the entire tree).
+			// taskkill itself can silently fail (PID gone, permission denied, etc.)
+			// so verify after 2s and log a warning if the process is still alive.
+			spawnTaskkillSafe(pid);
+			const verifyTimer = setTimeout(() => {
+				try {
+					process.kill(pid, 0); // throws ESRCH when dead
+					// Still alive — log and retry once.
+					logInternalError(
+						"child-pi.taskkill-stuck",
+						new Error(`process ${pid} still alive 2s after taskkill /T /F; retrying`),
+						`pid=${pid}`,
+						"error",
+					);
+					try {
+						spawnTaskkillSafe(pid);
+					} catch {
+						/* best-effort */
+					}
+				} catch {
+					// ESRCH or EPERM — process is gone. OK.
+				}
+			}, 2000);
+			verifyTimer.unref();
+			return;
+		}
+		try {
+			process.kill(-pid, "SIGTERM");
+		} catch (error) {
+			logInternalError("child-pi.sigterm", error, `pid=${pid}`);
+			try {
+				process.kill(pid, "SIGTERM");
+			} catch (fallbackError) {
+				logInternalError("child-pi.sigterm-absolute", fallbackError, `pid=${pid}`);
+			}
+		}
+		clearHardKillTimer(pid);
+		const hardKillTimer = setTimeout(() => {
+			try {
+				process.kill(-pid, "SIGKILL");
+			} catch (error) {
+				logInternalError("child-pi.sigkill", error, `pid=${pid}`);
+				try {
+					process.kill(pid, "SIGKILL");
+				} catch (fallbackError) {
+					logInternalError("child-pi.sigkill-absolute", fallbackError, `pid=${pid}`);
+				}
+			}
+			childHardKillTimers.delete(pid);
+		}, HARD_KILL_MS);
+		hardKillTimer.unref();
+		childHardKillTimers.set(pid, hardKillTimer);
+	} catch (error) {
+		logInternalError("child-pi.kill-process-pid", error, `pid=${pid}`);
+	}
+}
+
+export function killProcessTree(pid: number | undefined, child?: ChildProcess): void {
+	// Phase-0 diagnostic (HB-003a): capture who invoked killProcessTree so the
+	// exit-null race has a provenance trail. .stack is best-effort (may be undefined
+	// under deep async), so we take a snapshot lazily.
+	try {
+		const callerStack = new Error("killProcessTree caller").stack ?? "(no stack)";
+		logInternalError(
+			"child-pi.kill-process-tree-invoked",
+			new Error(`pid=${pid} called from:\n${callerStack.split("\n").slice(0, 8).join("\n")}`),
+			`pid=${pid}`,
+		);
+	} catch {
+		/* diagnostic best-effort */
+	}
+	if (!pid || !Number.isInteger(pid) || pid <= 0) return;
+	if (child && child.exitCode !== null) return;
+	killProcessPid(pid);
+	child?.once("exit", () => clearHardKillTimer(pid));
+}
+
+export function terminateActiveChildPiProcesses(): number {
+	const entries = [...activeChildProcesses.entries()];
+	for (const [pid, child] of entries) killProcessTree(pid, child);
+	return entries.length;
+}

package/src/runtime/child-pi-pool.ts

@@ -0,0 +1,68 @@
+// 2.6 — Child-pi warm pool skeleton (ADR 0008 Proposed).
+//
+// This module exposes the pool interface that child-pi.ts will consume
+// once Pi gains the wait-for-prompt handshake protocol. Until the Pi-side
+// support lands, the pool is permanently disabled (size=0): callers always
+// receive null from `acquirePooledChild` and fall through to the regular
+// spawn path. This keeps the integration point ready without altering
+// production behaviour.
+//
+// Enabling a real pool requires three pieces in order:
+//
+//   1. Pi runtime: respond to a `PI_CREW_POOL_HEALTH=1` ping on stdin and
+//      block in a "wait-for-prompt" state until the parent writes a real
+//      prompt. Pi does not currently implement this.
+//   2. This module: replace the `acquirePooledChild` returning null with
+//      an actual pool that spawn-and-park N processes.
+//   3. child-pi.ts: prefer pooled children on `runChildPi` entry; fall
+//      back to fresh spawn on miss.
+//
+// Disabled by default; opt-in via `runtime.warmPool.size > 0` config or
+// the `PI_CREW_WARM_POOL_SIZE` env var.
+import type { ChildProcess } from "node:child_process";
+
+export interface WarmPoolOptions {
+	/** Number of warm processes to maintain. 0 disables the pool. */
+	size: number;
+	/** Drop pooled processes that have been idle longer than this. */
+	maxIdleMs: number;
+}
+
+export const DEFAULT_WARM_POOL_OPTIONS: WarmPoolOptions = {
+	size: 0,
+	maxIdleMs: 5 * 60_000,
+};
+
+/** Resolve the effective pool size from env / config / defaults. */
+export function resolveWarmPoolSize(env: NodeJS.ProcessEnv = process.env, configured?: number): number {
+	const fromEnv = Number.parseInt(env.PI_CREW_WARM_POOL_SIZE ?? "", 10);
+	if (Number.isFinite(fromEnv) && fromEnv >= 0) return fromEnv;
+	if (typeof configured === "number" && Number.isFinite(configured) && configured >= 0) return configured;
+	return DEFAULT_WARM_POOL_OPTIONS.size;
+}
+
+/**
+ * Try to acquire a parked child from the pool. Returns null when the pool
+ * is disabled or empty; caller should spawn a fresh child instead.
+ *
+ * Skeleton — currently always returns null. See module docstring.
+ */
+export function acquirePooledChild(_options: Partial<WarmPoolOptions> = {}): ChildProcess | null {
+	return null;
+}
+
+/**
+ * Mark a pooled child as done. Pool processes are single-use: this terminates
+ * the child rather than returning it to the pool, because state contamination
+ * across runs would be unsafe (file handles, env mutations, mounted FDs).
+ *
+ * Skeleton — currently a no-op since acquirePooledChild never returns a child.
+ */
+export function releasePooledChild(_child: ChildProcess | null | undefined): void {
+	// no-op while the pool is disabled
+}
+
+/** Drain and terminate every parked child. Call on cleanupRuntime. */
+export function disposeWarmPool(): void {
+	// no-op while the pool is disabled
+}