pi-crew 0.9.53 → 0.9.55

package/src/runtime/deterministic-ast.ts

@@ -0,0 +1,161 @@
+/**
+ * deterministic-ast.ts — AST-based determinism enforcement for dynamic-workflow scripts (round-13 P0-2).
+ *
+ * Rejects `Date.now()`, `Math.random()`, and `new Date()` at workflow-load time
+ * using a true AST walk (not regex) so that:
+ *   - Prompts mentioning "Date.now()" as string literals are accepted.
+ *   - Comments containing "Date.now()" are accepted.
+ *   - `Date.parse()`, `Date.UTC()`, `Math.floor()`, etc. are accepted (only `now` and `random` are blocked).
+ *
+ * Adapted from pi-dynamic-workflows/src/workflow.ts (MIT) — see NOTICE.md.
+ *
+ * The walker uses acorn's parse() with permissive flags (allowAwaitOutsideFunction,
+ * allowReturnOutsideFunction) so we don't reject perfectly valid workflow scripts
+ * that contain top-level `await` or `return`.
+ *
+ * On parse error, this function returns silently: jiti will surface a clearer
+ * parse error downstream. We don't double-report parse errors.
+ */
+
+import { parse } from "acorn";
+
+const NONDETERMINISM_ERROR =
+	"Workflow scripts must be deterministic: Date.now()/Math.random()/new Date() are unavailable. These introduce non-reproducible behavior across runs. Use ctx.vars for cached state, or pass a fixed seed via ctx.setArgs(). To bypass this check (escape hatch), set PI_CREW_DWF_SKIP_DETERMINISM_CHECK=1.";
+
+export class DeterminismError extends Error {
+	constructor() {
+		super(NONDETERMINISM_ERROR);
+		this.name = "DeterminismError";
+	}
+}
+
+/**
+ * Parse `script` and walk the AST looking for non-deterministic calls.
+ * Throws DeterminismError on the first hit. Silently returns on parse error
+ * (jiti will produce a clearer message downstream).
+ */
+export function assertDeterministicScript(script: string): void {
+	let ast: AstNode;
+	try {
+		ast = parse(script, {
+			ecmaVersion: "latest",
+			sourceType: "module",
+			allowAwaitOutsideFunction: true,
+			allowReturnOutsideFunction: true,
+			ranges: false,
+		}) as unknown as AstNode;
+	} catch {
+		// Parse errors are handled by jiti downstream — don't double-report.
+		return;
+	}
+	assertDeterministicAst(ast);
+}
+
+/**
+ * Escape hatch: when PI_CREW_DWF_SKIP_DETERMINISM_CHECK=1 the check is bypassed.
+ * Power users may need this when a workflow legitimately depends on time/random
+ * (e.g. randomized benchmark scripts).
+ */
+export function isDeterminismCheckEnabled(): boolean {
+	return process.env.PI_CREW_DWF_SKIP_DETERMINISM_CHECK !== "1";
+}
+
+// ---------------------------------------------------------------------------
+// AST walker
+// ---------------------------------------------------------------------------
+
+interface AstNode {
+	type: string;
+	[key: string]: unknown;
+}
+
+function asAstNode(value: unknown): AstNode | undefined {
+	if (!value || typeof value !== "object") return undefined;
+	const obj = value as Record<string, unknown>;
+	if (typeof obj.type !== "string") return undefined;
+	return obj as AstNode;
+}
+
+function astChildren(node: AstNode): AstNode[] {
+	const out: AstNode[] = [];
+	for (const value of Object.values(node)) {
+		if (Array.isArray(value)) {
+			for (const item of value) {
+				const child = asAstNode(item);
+				if (child) out.push(child);
+			}
+		} else {
+			const child = asAstNode(value);
+			if (child) out.push(child);
+		}
+	}
+	return out;
+}
+
+function assertDeterministicAst(node: AstNode): void {
+	if (isDateNowCall(node) || isMathRandomCall(node) || isNewDateExpression(node)) {
+		throw new DeterminismError();
+	}
+	for (const child of astChildren(node)) assertDeterministicAst(child);
+}
+
+function isDateNowCall(node: AstNode): boolean {
+	return node.type === "CallExpression" && isMemberExpression(node, "callee", "Date", "now");
+}
+
+function isMathRandomCall(node: AstNode): boolean {
+	return node.type === "CallExpression" && isMemberExpression(node, "callee", "Math", "random");
+}
+
+function isNewDateExpression(node: AstNode): boolean {
+	if (node.type !== "NewExpression") return false;
+	const callee = asAstNode(node.callee);
+	return callee?.type === "Identifier" && callee.name === "Date";
+}
+
+/**
+ * Test whether `node[childKey]` is a MemberExpression of shape `objectName.propertyName`,
+ * where the property is either a static Identifier or a resolvable static string.
+ * `childKey` is the property name on `node` (usually "callee" for CallExpression).
+ */
+function isMemberExpression(node: AstNode, childKey: string, objectName: string, propertyName: string): boolean {
+	const child = asAstNode(node[childKey]);
+	if (child?.type !== "MemberExpression") return false;
+	const object = asAstNode(child.object);
+	if (object?.type !== "Identifier" || object.name !== objectName) return false;
+	return propertyNameOf(child) === propertyName;
+}
+
+function propertyNameOf(node: AstNode): string | undefined {
+	const computed = node.computed === true;
+	const property = asAstNode(node.property);
+	if (!property) return undefined;
+	if (!computed && property.type === "Identifier") {
+		return property.name as string | undefined;
+	}
+	return staticStringOf(property);
+}
+
+function staticStringOf(node: AstNode | undefined): string | undefined {
+	if (!node) return undefined;
+	if (node.type === "Literal" && typeof node.value === "string") return node.value;
+	if (node.type === "TemplateLiteral") {
+		const expressions = node.expressions;
+		if (Array.isArray(expressions) && expressions.length > 0) return undefined;
+		const quasis = node.quasis;
+		if (!Array.isArray(quasis)) return undefined;
+		return quasis
+			.map((q) => {
+				const quasi = asAstNode(q);
+				const value = quasi?.value as { cooked?: string; raw?: string } | undefined;
+				return value?.cooked ?? value?.raw ?? "";
+			})
+			.join("");
+	}
+	if (node.type === "BinaryExpression" && node.operator === "+") {
+		const left = staticStringOf(asAstNode(node.left));
+		const right = staticStringOf(asAstNode(node.right));
+		if (left !== undefined && right !== undefined) return left + right;
+	}
+	return undefined;
+}

package/src/runtime/diagnostic-export.ts

@@ -0,0 +1,171 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { MetricRegistry } from "../observability/metric-registry.ts";
+import type { MetricSnapshot } from "../observability/metrics-primitives.ts";
+import { atomicWriteFile } from "../state/atomic-write.ts";
+import { readEvents, type TeamEvent } from "../state/event-log.ts";
+import { loadRunManifestById } from "../state/state-store.ts";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { type HeartbeatSummary, summarizeHeartbeats } from "../ui/heartbeat-aggregator.ts";
+import type { RunUiSnapshot } from "../ui/snapshot-types.ts";
+import { isSecretKey, redactSecrets } from "../utils/redaction.ts";
+import { readCrewAgents } from "./crew-agent-records.ts";
+import { buildRecoveryLedger, type RecoveryLedgerEntry } from "./recovery-recipes.ts";
+
+export { isSecretKey, redactSecrets } from "../utils/redaction.ts";
+
+export interface DiagnosticReport {
+	schemaVersion?: number;
+	runId: string;
+	exportedAt: string;
+	manifest: TeamRunManifest;
+	tasks: TeamTaskState[];
+	recentEvents: TeamEvent[];
+	heartbeat: HeartbeatSummary;
+	agents: unknown[];
+	envRedacted: Record<string, string>;
+	metricsSnapshot?: MetricSnapshot[];
+	// Layer 8: task diagnostics
+	taskDiagnostics: Record<string, Record<string, unknown>>;
+	// Layer 9: terminal evidence
+	terminalEvidence: Record<string, TeamTaskState["terminalEvidence"]>;
+	// Layer 10: model attempts and routing
+	modelAttempts: {
+		taskId: string;
+		attempts: TeamTaskState["modelAttempts"];
+		routing: TeamTaskState["modelRouting"];
+	}[];
+	// Layer 11: pending mailbox
+	pendingMailbox: {
+		taskId: string;
+		pendingSteers: TeamTaskState["pendingSteers"];
+	}[];
+	runMailboxUnread: RunUiSnapshot["mailbox"];
+	// Layer 12: recovery ledger
+	recoveryLedger: RecoveryLedgerEntry[];
+}
+
+const ENV_DEBUG_ALLOWLIST =
+	/^(PI_CREW_|PI_TEAMS_|PI_.*HOME|NODE_ENV|NODE_VERSION|OS|PROCESSOR|TERM|LANG|HOME|USERPROFILE|APPDATA|PLATFORM|ARCH|WIN32|DOCKER|CI|VERBOSE|DEBUG|NO_COLOR|FORCE_COLOR|NPM_CONFIG|npm_)/i;
+
+function envRedacted(): Record<string, string> {
+	const output: Record<string, string> = {};
+	for (const [key, value] of Object.entries(process.env)) {
+		if (isSecretKey(key)) output[key] = "***";
+		else if (typeof value === "string" && ENV_DEBUG_ALLOWLIST.test(key)) output[key] = value;
+		// All other env vars are omitted to prevent leaking sensitive paths or system topology.
+	}
+	return output;
+}
+
+function buildSnapshot(manifest: TeamRunManifest, tasks: TeamTaskState[]): RunUiSnapshot {
+	const agents = readCrewAgents(manifest);
+	return {
+		runId: manifest.runId,
+		cwd: manifest.cwd,
+		fetchedAt: Date.now(),
+		signature: `${manifest.runId}:${manifest.updatedAt}`,
+		manifest,
+		tasks,
+		agents,
+		progress: {
+			total: tasks.length,
+			completed: tasks.filter((task) => task.status === "completed").length,
+			running: tasks.filter((task) => task.status === "running").length,
+			failed: tasks.filter((task) => task.status === "failed").length,
+			queued: tasks.filter((task) => task.status === "queued").length,
+		},
+		usage: { tokensIn: 0, tokensOut: 0, toolUses: 0 },
+		mailbox: { inboxUnread: 0, outboxPending: 0, needsAttention: 0 },
+		recentEvents: [],
+		recentOutputLines: [],
+	};
+}
+
+export async function exportDiagnostic(
+	ctx: Pick<ExtensionContext, "cwd">,
+	runId: string,
+	options: { registry?: MetricRegistry } = {},
+): Promise<{ path: string; report: DiagnosticReport }> {
+	const loaded = loadRunManifestById(ctx.cwd, runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency;
+	if (!loaded) throw new Error(`Run '${runId}' not found.`);
+	const exportedAt = new Date().toISOString();
+	const safeTimestamp = exportedAt.replace(/[:.]/g, "-");
+	const recentEvents = readEvents(loaded.manifest.eventsPath).slice(-200);
+	const metricsSnapshot = options.registry?.snapshot();
+	const taskDiagnostics: Record<string, Record<string, unknown>> = {};
+	const terminalEvidence: Record<string, TeamTaskState["terminalEvidence"]> = {};
+	const modelAttempts: {
+		taskId: string;
+		attempts: TeamTaskState["modelAttempts"];
+		routing: TeamTaskState["modelRouting"];
+	}[] = [];
+	const pendingMailbox: {
+		taskId: string;
+		pendingSteers: TeamTaskState["pendingSteers"];
+	}[] = [];
+	for (const task of loaded.tasks) {
+		if (task.diagnostics) taskDiagnostics[task.id] = task.diagnostics;
+		if (task.terminalEvidence) terminalEvidence[task.id] = task.terminalEvidence;
+		if (task.modelAttempts || task.modelRouting) {
+			modelAttempts.push({
+				taskId: task.id,
+				attempts: task.modelAttempts,
+				routing: task.modelRouting,
+			});
+		}
+		if (task.pendingSteers) {
+			pendingMailbox.push({
+				taskId: task.id,
+				pendingSteers: task.pendingSteers,
+			});
+		}
+	}
+	const recoveryLedger = loaded.manifest.policyDecisions ? buildRecoveryLedger(loaded.manifest.policyDecisions).entries : [];
+	const snapshot = buildSnapshot(loaded.manifest, loaded.tasks);
+	const report: DiagnosticReport = {
+		...(metricsSnapshot ? { schemaVersion: 2 } : {}),
+		runId,
+		exportedAt,
+		manifest: redactSecrets(loaded.manifest) as TeamRunManifest,
+		tasks: redactSecrets(loaded.tasks) as TeamTaskState[],
+		recentEvents: redactSecrets(recentEvents) as TeamEvent[],
+		heartbeat: summarizeHeartbeats(snapshot),
+		agents: redactSecrets(readCrewAgents(loaded.manifest)) as unknown[],
+		envRedacted: envRedacted(),
+		...(metricsSnapshot
+			? {
+					metricsSnapshot: redactSecrets(metricsSnapshot) as MetricSnapshot[],
+				}
+			: {}),
+		taskDiagnostics,
+		terminalEvidence,
+		modelAttempts,
+		pendingMailbox,
+		runMailboxUnread: snapshot.mailbox,
+		recoveryLedger,
+	};
+	const dir = path.join(loaded.manifest.artifactsRoot, "diagnostic");
+	fs.mkdirSync(dir, { recursive: true });
+	const filePath = path.join(dir, `diagnostic-${safeTimestamp}.json`);
+	atomicWriteFile(filePath, `${JSON.stringify(report, null, 2)}\n`);
+	return { path: filePath, report };
+}
+
+export function listRecentDiagnostic(dir: string, windowMs: number, now = Date.now()): string | undefined {
+	try {
+		if (!fs.existsSync(dir)) return undefined;
+		return fs
+			.readdirSync(dir)
+			.filter((file) => file.startsWith("diagnostic-") && file.endsWith(".json"))
+			.map((file) => ({
+				file,
+				mtimeMs: fs.statSync(path.join(dir, file)).mtimeMs,
+			}))
+			.filter((entry) => now - entry.mtimeMs < windowMs)
+			.sort((a, b) => b.mtimeMs - a.mtimeMs)[0]?.file;
+	} catch {
+		return undefined;
+	}
+}

package/src/runtime/direct-run.ts

@@ -0,0 +1,45 @@
+import type { AgentConfig } from "../agents/agent-config.ts";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import type { TeamConfig } from "../teams/team-config.ts";
+import type { WorkflowConfig } from "../workflows/workflow-config.ts";
+
+export function isDirectRun(manifest: Pick<TeamRunManifest, "team" | "workflow">): boolean {
+	return manifest.workflow === "direct-agent";
+}
+
+export function directTeamAndWorkflowFromRun(
+	manifest: TeamRunManifest,
+	tasks: TeamTaskState[],
+	agents: AgentConfig[],
+): { team: TeamConfig; workflow: WorkflowConfig } | undefined {
+	if (!isDirectRun(manifest)) return undefined;
+	const firstTask = tasks[0];
+	const agentName = firstTask?.agent ?? (manifest.team.replace(/^direct-/, "") || "executor");
+	const agent = agents.find((candidate) => candidate.name === agentName);
+	const role = firstTask?.role ?? "agent";
+	const stepId = firstTask?.stepId ?? "01_agent";
+	return {
+		team: {
+			name: manifest.team,
+			description: `Direct subagent run for ${agentName}`,
+			source: "builtin",
+			filePath: "<generated>",
+			roles: [
+				{
+					name: role,
+					agent: agentName,
+					description: agent?.description,
+				},
+			],
+			defaultWorkflow: "direct-agent",
+			workspaceMode: manifest.workspaceMode,
+		},
+		workflow: {
+			name: manifest.workflow ?? "direct-agent",
+			description: `Direct task for ${agentName}`,
+			source: "builtin",
+			filePath: "<generated>",
+			steps: [{ id: stepId, role, task: "{goal}", model: firstTask?.model }],
+		},
+	};
+}

package/src/runtime/dwf-state-store.ts

@@ -0,0 +1,102 @@
+/**
+ * dwf-state-store.ts — Persistent checkpoint state for dynamic-workflow runs (P2-3, round-18).
+ *
+ * Modeled on GoalStore (goal-state-store.ts) and FileCheckpointStore (checkpoint.ts),
+ * but scoped to a single run's stateRoot (which is already <crewRoot>/state/runs/<runId>).
+ *
+ * Stores DwfCheckpointState as atomic JSON at <stateRoot>/dwf-checkpoint.json.
+ * atomicWriteJson (temp + rename + fsync) guarantees either the old or the new file,
+ * never a partial write — safe across crashes.
+ *
+ * Resume semantics (round-18): the runner loads a checkpoint on run start and hydrates
+ * ctx.vars/phases/logs from it; on clean completion the runner deletes it. A missing or
+ * corrupt checkpoint is treated as a fresh run (load() returns undefined). If a crash
+ * happens mid-agent, that agent simply re-runs from scratch on resume — agent results
+ * are expected to be idempotent-ish.
+ */
+
+import { existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs";
+import { dirname } from "node:path";
+import { atomicWriteJson } from "../state/atomic-write.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+
+export interface DwfCheckpointState {
+	runId: string;
+	vars: Record<string, unknown>;
+	phases: string[];
+	currentPhase: string | undefined;
+	logs: string[]; // capped copy (≤1000); the events log (dwf.log) is the durable source of truth
+	spent: number; // budget accumulator (round-14 P1-2)
+	agentCount: number;
+	/** PERS-1: per-agent-call idempotency cache. Maps a deterministic call ID to the
+	 *  cached result so a DWF resume skips already-completed agent calls (avoids
+	 *  duplicating artifacts/mailbox/tokens). Uses Record (not Map) for JSON
+	 *  serialization safety. */
+	completedAgentCalls?: Record<string, { text: string; usage?: { input: number; output: number } }>;
+	updatedAt: string;
+}
+
+/**
+ * DwfStore — atomic CRUD for a single run's DWF checkpoint.
+ *
+ * Concurrency: writes are atomic (atomicWriteJson). The DWF runner is the sole
+ * writer during a run; `team resume` loads the checkpoint read-only before the
+ * script re-executes. No file-lock is needed here because only one runner owns a
+ * run's stateRoot at a time (run locks protect manifest transitions elsewhere).
+ *
+ * Note: the constructor takes the run's stateRoot directly (NOT cwd + runId) to
+ * avoid a double-nesting bug — stateRoot is already <crewRoot>/state/runs/<runId>,
+ * so the checkpoint lands at <crewRoot>/state/runs/<runId>/dwf-checkpoint.json.
+ * This mirrors FileCheckpointStore (checkpoint.ts: constructor(stateRoot)).
+ */
+export class DwfStore {
+	private readonly stateRoot: string;
+
+	constructor(stateRoot: string) {
+		this.stateRoot = stateRoot;
+	}
+
+	private get path(): string {
+		return `${this.stateRoot}/dwf-checkpoint.json`;
+	}
+
+	/** Load the checkpoint for this run's stateRoot. Returns undefined if missing or corrupt (fresh run). */
+	load(): DwfCheckpointState | undefined {
+		const path = this.path;
+		try {
+			if (!existsSync(path)) return undefined;
+			const raw = readFileSync(path, "utf-8");
+			const parsed = JSON.parse(raw);
+			// Corrupt-guard: a valid checkpoint must be an object with a string runId
+			// (mirrors GoalStore.load's typeof parsed.goalId !== "string" check).
+			if (!parsed || typeof parsed !== "object" || typeof parsed.runId !== "string") return undefined;
+			return parsed as DwfCheckpointState;
+		} catch {
+			return undefined;
+		}
+	}
+
+	/** Atomically persist a checkpoint state. Stamps `updatedAt` (callers need not set it). */
+	save(state: DwfCheckpointState): void {
+		const path = this.path;
+		const next = { ...state, updatedAt: new Date().toISOString() };
+		try {
+			mkdirSync(dirname(path), { recursive: true });
+			atomicWriteJson(path, next);
+		} catch (error) {
+			logInternalError("dwf-state-store.save", error, `runId=${state.runId}`);
+			throw error;
+		}
+	}
+
+	/** Remove the checkpoint file (after a clean completion). Best-effort; never throws. */
+	delete(): void {
+		const path = this.path;
+		try {
+			if (!existsSync(path)) return;
+			unlinkSync(path);
+		} catch (error) {
+			logInternalError("dwf-state-store.delete", error);
+		}
+	}
+}