pi-crew 0.6.0 → 0.6.3

package/src/runtime/chain-parser.ts

@@ -114,7 +114,11 @@ function tokenize(input: string): Token[] {
 class ChainParser {
 	private pos = 0;
 
-	constructor(private tokens: Token[]) {}
+	private tokens: Token[];
+
+	constructor(tokens: Token[]) {
+		this.tokens = tokens;
+	}
 
 	parse(): ChainStep[] {
 		const steps: ChainStep[] = [];

package/src/runtime/chain-runner.ts

@@ -11,6 +11,8 @@
  */
 
 import type { HandoffSummary, HandoffManager, TaskPacket, TaskResult } from "./handoff-manager.ts";
+import { parseChainDSL } from "./chain-parser.ts";
+import type { ChainStep as DSLChainStep } from "./chain-parser.ts";
 
 /**
  * Single step in a chain.
@@ -104,10 +106,16 @@ export class ChainRunner {
 	/** Maximum size per handoff entry to prevent memory issues from large artifacts */
 	private static readonly MAX_HANDOFF_ENTRY_SIZE = 5000; // bytes per entry
 
+	private taskRunner: ChainTaskRunner;
+	private handoffManager: HandoffManager;
+
 	constructor(
-		private taskRunner: ChainTaskRunner,
-		private handoffManager: HandoffManager,
-	) {}
+		taskRunner: ChainTaskRunner,
+		handoffManager: HandoffManager,
+	) {
+		this.taskRunner = taskRunner;
+		this.handoffManager = handoffManager;
+	}
 
 	/**
 	 * Parse chain syntax: step1 -> step2 -> step3
@@ -123,10 +131,25 @@ export class ChainRunner {
 	 * parseChain('"Research AI trends" -> "Analyze findings"')
 	 * parseChain("@step1 --model claude-opus-3 -> @step2")
 	 * 
+	 * Also supports DSL syntax from chain-parser for advanced constructs:
+	 * parseChain("step1 -> parallel(step2, step3) -> step4")
+	 * parseChain("step1:3 -> step2 --with-context -> step3")
+	 * 
 	 * @param chainString - The chain string to parse
 	 * @returns Parsed chain specification
 	 */
 	parseChain(chainString: string): ChainSpec {
+		// Try DSL parser first for advanced syntax (parallel groups, loop counts, flags)
+		// Falls back to the simple split parser if DSL parsing fails
+		if (this.hasDSLConstructs(chainString)) {
+			try {
+				const dslSteps = parseChainDSL(chainString);
+				return this.dslToChainSpec(dslSteps, chainString);
+			} catch {
+				// DSL parse failed; fall through to simple parser
+			}
+		}
+
 		const stepStrings = chainString.split("->").map(s => s.trim());
 
 		const steps: ChainStep[] = stepStrings.map((step, index) => {
@@ -337,6 +360,47 @@ export class ChainRunner {
 		return parsed;
 	}
 
+	/**
+	 * Detect if chainString uses DSL constructs that require chain-parser.
+	 * DSL features: parallel(...), :loopCount, --with-context flag
+	 */
+	private hasDSLConstructs(chainString: string): boolean {
+		return /\bparallel\s*\(/.test(chainString) ||
+			/\w+:\d+\b/.test(chainString) ||
+			/--with-context/.test(chainString);
+	}
+
+	/**
+	 * Convert DSL AST steps (from chain-parser) to ChainSpec.
+	 */
+	private dslToChainSpec(dslSteps: DSLChainStep[], chainString: string): ChainSpec {
+		const steps: ChainStep[] = dslSteps.map((dslStep, index) => {
+			// For parallel groups, use a synthetic step name
+			if (dslStep.parallel) {
+				return {
+					name: dslStep.name,
+					context: {
+						parallel: dslStep.parallel.map(p => ({ name: p.name, loopCount: p.loopCount, withContext: p.withContext, args: p.args })),
+					},
+					loopCount: dslStep.loopCount,
+				};
+			}
+			const step: ChainStep = { name: dslStep.name };
+			if (dslStep.loopCount) step.context = { ...step.context, loopCount: dslStep.loopCount };
+			if (dslStep.withContext) step.context = { ...step.context, withContext: true };
+			if (dslStep.args && dslStep.args.length > 0) step.context = { ...step.context, args: dslStep.args };
+			return step;
+		});
+
+		// Extract global overrides using existing logic
+		const globalModel = this.extractGlobalFlag(chainString, "global-model");
+		const globalSkill = this.extractGlobalFlag(chainString, "global-skill");
+		const globalThinking = this.extractGlobalFlag(chainString, "global-thinking") as "fast" | "standard" | "deep" | undefined;
+		const continueOnError = this.extractGlobalFlag(chainString, "continue-on-error") === "true";
+
+		return { steps, globalModel, globalSkill, globalThinking, continueOnError };
+	}
+
 	/**
 	 * Sanitize identifier to prevent injection.
 	 */

package/src/runtime/checkpoint.ts

@@ -1,28 +1,30 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
+import { projectCrewRoot } from "../utils/paths.ts";
 import { assertSafePathId } from "../utils/safe-paths.ts";
+import { logInternalError } from "../utils/internal-error.ts";
 
 export interface Checkpoint {
-  runId: string;
-  taskId: string;
-  step: number;
-  context: string;
-  progress: string;
-  savedAt: number;
-  agentId: string;
-  agentModel?: string;
+	runId: string;
+	taskId: string;
+	step: number;
+	context: string;
+	progress: string;
+	savedAt: number;
+	agentId: string;
+	agentModel?: string;
 }
 
 export interface CheckpointStore {
-  save(checkpoint: Checkpoint): void;
-  load(runId: string, taskId: string): Checkpoint | null;
-  delete(runId: string, taskId: string): void;
-  list(runId: string): Checkpoint[];
-  hasCheckpoint(runId: string, taskId: string): boolean;
+	save(checkpoint: Checkpoint): void;
+	load(runId: string, taskId: string): Checkpoint | null;
+	delete(runId: string, taskId: string): void;
+	list(runId: string): Checkpoint[];
+	hasCheckpoint(runId: string, taskId: string): boolean;
 }
 
 interface CheckpointEntry {
-  checkpoints: Record<string, Checkpoint>;
+	checkpoints: Record<string, Checkpoint>;
 }
 
 /**
@@ -30,222 +32,302 @@ interface CheckpointEntry {
  * Saves checkpoints as JSON files in .crew/state/runs/<runId>/checkpoints/
  */
 export class FileCheckpointStore implements CheckpointStore {
-  private readonly stateRoot: string;
-
-  constructor(stateRoot: string) {
-    this.stateRoot = stateRoot;
-  }
-
-  private checkpointDir(): string {
-    return path.join(this.stateRoot, "checkpoints");
-  }
-
-  private checkpointPath(taskId: string): string {
-    return path.join(this.checkpointDir(), `${taskId}.json`);
-  }
-
-  private ensureDir(): void {
-    const dir = this.checkpointDir();
-    if (!fs.existsSync(dir)) {
-      fs.mkdirSync(dir, { recursive: true });
-    }
-  }
-
-  save(checkpoint: Checkpoint): void {
-    // Validate taskId to prevent path traversal: the taskId is used to
-    // build a file path under this.checkpointDir(). Without validation, a
-    // malicious or buggy taskId like "../../../etc/passwd" could escape
-    // the checkpoints directory.
-    assertSafePathId("taskId", checkpoint.taskId);
-    this.ensureDir();
-    const p = this.checkpointPath(checkpoint.taskId);
-    fs.writeFileSync(p, JSON.stringify(checkpoint, null, 2), "utf-8");
-  }
-
-  load(runId: string, taskId: string): Checkpoint | null {
-    assertSafePathId("taskId", taskId);
-    const p = this.checkpointPath(taskId);
-    if (!fs.existsSync(p)) return null;
-
-    try {
-      const data = JSON.parse(fs.readFileSync(p, "utf-8")) as Checkpoint;
-      // Verify it's for the correct run
-      if (data.runId !== runId) return null;
-      return data;
-    } catch {
-      return null;
-    }
-  }
-
-  delete(runId: string, taskId: string): void {
-    assertSafePathId("taskId", taskId);
-    const p = this.checkpointPath(taskId);
-    if (fs.existsSync(p)) {
-      try {
-        const data = JSON.parse(fs.readFileSync(p, "utf-8")) as Checkpoint;
-        if (data.runId === runId) {
-          fs.unlinkSync(p);
-        }
-      } catch {
-        // File existed but couldn't read — delete it anyway
-        try {
-          fs.unlinkSync(p);
-        } catch { /* ignore */ }
-      }
-    }
-  }
-
-  list(runId: string): Checkpoint[] {
-    const dir = this.checkpointDir();
-    if (!fs.existsSync(dir)) return [];
-
-    return fs.readdirSync(dir)
-      .filter((f) => f.endsWith(".json"))
-      .map((f) => {
-        try {
-          return JSON.parse(fs.readFileSync(path.join(dir, f), "utf-8")) as Checkpoint;
-        } catch {
-          return null;
-        }
-      })
-      .filter((c): c is Checkpoint => c !== null && c.runId === runId);
-  }
-
-  hasCheckpoint(runId: string, taskId: string): boolean {
-    return this.load(runId, taskId) !== null;
-  }
+	private readonly stateRoot: string;
+
+	constructor(stateRoot: string) {
+		this.stateRoot = stateRoot;
+	}
+
+	private checkpointDir(): string {
+		return path.join(this.stateRoot, "checkpoints");
+	}
+
+	private checkpointPath(taskId: string): string {
+		return path.join(this.checkpointDir(), `${taskId}.json`);
+	}
+
+	private ensureDir(): void {
+		const dir = this.checkpointDir();
+		if (!fs.existsSync(dir)) {
+			fs.mkdirSync(dir, { recursive: true });
+		}
+	}
+
+	save(checkpoint: Checkpoint): void {
+		// Validate taskId to prevent path traversal: the taskId is used to
+		// build a file path under this.checkpointDir(). Without validation, a
+		// malicious or buggy taskId like "../../../etc/passwd" could escape
+		// the checkpoints directory.
+		assertSafePathId("taskId", checkpoint.taskId);
+		this.ensureDir();
+		const p = this.checkpointPath(checkpoint.taskId);
+		// Atomic write: write to temp file first, then rename, then fsync parent.
+		// This guarantees either the old file or the new file, never a partial
+		// write, even on network filesystems or certain journal modes.
+		const tmp = path.join(this.checkpointDir(), ".tmp.checkpoint");
+		fs.writeFileSync(tmp, JSON.stringify(checkpoint, null, 2), "utf-8");
+		fs.renameSync(tmp, p);
+		// fsync parent directory to ensure the rename is durable
+		const dirFd = fs.openSync(this.checkpointDir(), "r");
+		try {
+			fs.fsyncSync(dirFd);
+		} catch {
+			// EPERM on Windows: opening a directory and fsync-ing it is not supported
+			// on all Windows configurations. Best-effort — the rename is still atomic.
+		} finally {
+			fs.closeSync(dirFd);
+		}
+	}
+
+	load(runId: string, taskId: string): Checkpoint | null {
+		assertSafePathId("taskId", taskId);
+		const p = this.checkpointPath(taskId);
+		if (!fs.existsSync(p)) return null;
+
+		try {
+			const data = JSON.parse(fs.readFileSync(p, "utf-8")) as Checkpoint;
+			// Verify it's for the correct run
+			if (data.runId !== runId) return null;
+			return data;
+		} catch {
+			// File existed but JSON was corrupt — log and rename for later inspection
+			logInternalError("checkpoint-load", new Error("JSON parse failed"), `file=${p}`);
+			try {
+				fs.renameSync(p, `${p}.corrupt.${Date.now()}`);
+			} catch {
+				// Best effort — ignore rename failure
+			}
+			return null;
+		}
+	}
+
+	delete(runId: string, taskId: string): void {
+		assertSafePathId("taskId", taskId);
+		const p = this.checkpointPath(taskId);
+		if (fs.existsSync(p)) {
+			try {
+				const data = JSON.parse(
+					fs.readFileSync(p, "utf-8"),
+				) as Checkpoint;
+				if (data.runId === runId) {
+					fs.unlinkSync(p);
+				}
+			} catch {
+				// File existed but couldn't read — delete it anyway
+				try {
+					fs.unlinkSync(p);
+				} catch {
+					/* ignore */
+				}
+			}
+		}
+	}
+
+	list(runId: string): Checkpoint[] {
+		const dir = this.checkpointDir();
+		if (!fs.existsSync(dir)) return [];
+
+		return fs
+			.readdirSync(dir)
+			.filter((f) => f.endsWith(".json"))
+			.map((f) => {
+				try {
+					return JSON.parse(
+						fs.readFileSync(path.join(dir, f), "utf-8"),
+					) as Checkpoint;
+				} catch {
+					return null;
+				}
+			})
+			.filter((c): c is Checkpoint => c !== null && c.runId === runId);
+	}
+
+	hasCheckpoint(runId: string, taskId: string): boolean {
+		return this.load(runId, taskId) !== null;
+	}
 }
 
+const MAX_STORES = 100;
 const _stores = new Map<string, FileCheckpointStore>();
 
 /**
  * Get checkpoint store for a run's state root.
+ * Uses LRU eviction when the store exceeds MAX_STORES entries.
  */
 export function getCheckpointStore(stateRoot: string): CheckpointStore {
-  if (!_stores.has(stateRoot)) {
-    _stores.set(stateRoot, new FileCheckpointStore(stateRoot));
-  }
-  return _stores.get(stateRoot)!;
+	if (!_stores.has(stateRoot)) {
+		if (_stores.size >= MAX_STORES) {
+			// Evict the oldest entry (first in insertion order)
+			const oldestKey = _stores.keys().next().value;
+			if (oldestKey !== undefined) {
+				_stores.delete(oldestKey);
+			}
+		}
+		_stores.set(stateRoot, new FileCheckpointStore(stateRoot));
+	}
+	return _stores.get(stateRoot)!;
 }
 
 /**
  * Clear all checkpoint stores (for testing).
  */
 export function clearCheckpointStores(): void {
-  _stores.clear();
+	_stores.clear();
 }
 
 /**
  * Save a checkpoint during agent execution.
  */
 export function saveCheckpoint(
-  runId: string,
-  taskId: string,
-  step: number,
-  context: string,
-  progress: string,
-  agentId: string,
-  agentModel?: string,
+	runId: string,
+	taskId: string,
+	step: number,
+	context: string,
+	progress: string,
+	agentId: string,
+	agentModel?: string,
+	cwd?: string,
 ): void {
-  // Validate both runId and taskId to prevent path traversal: these are
-  // used to build the file path under .crew/state/runs/<runId>/checkpoints/<taskId>.json.
-  assertSafePathId("runId", runId);
-  assertSafePathId("taskId", taskId);
-  const checkpoint: Checkpoint = {
-    runId,
-    taskId,
-    step,
-    context,
-    progress,
-    savedAt: Date.now(),
-    agentId,
-    agentModel,
-  };
-
-  // State root is parent of checkpoints dir
-  const stateRoot = path.join(process.cwd(), ".crew/state/runs", runId);
-  const store = getCheckpointStore(stateRoot);
-  store.save(checkpoint);
+	// Validate both runId and taskId to prevent path traversal: these are
+	// used to build the file path under <crewRoot>/state/runs/<runId>/checkpoints/<taskId>.json.
+	assertSafePathId("runId", runId);
+	assertSafePathId("taskId", taskId);
+	const checkpoint: Checkpoint = {
+		runId,
+		taskId,
+		step,
+		context,
+		progress,
+		savedAt: Date.now(),
+		agentId,
+		agentModel,
+	};
+
+	// State root is parent of checkpoints dir. Use projectCrewRoot() so the
+	// path lands in .pi/teams/state/runs/ for .pi-based projects (issue #29).
+	const stateRoot = path.join(
+		projectCrewRoot(cwd ?? process.cwd()),
+		"state",
+		"runs",
+		runId,
+	);
+	const store = getCheckpointStore(stateRoot);
+	store.save(checkpoint);
 }
 
 /**
  * Load a checkpoint for resuming.
  */
-export function loadCheckpoint(runId: string, taskId: string): Checkpoint | null {
-  assertSafePathId("runId", runId);
-  assertSafePathId("taskId", taskId);
-  const stateRoot = path.join(process.cwd(), ".crew/state/runs", runId);
-  const store = getCheckpointStore(stateRoot);
-  return store.load(runId, taskId);
+export function loadCheckpoint(
+	runId: string,
+	taskId: string,
+	cwd?: string,
+): Checkpoint | null {
+	assertSafePathId("runId", runId);
+	assertSafePathId("taskId", taskId);
+	const stateRoot = path.join(
+		projectCrewRoot(cwd ?? process.cwd()),
+		"state",
+		"runs",
+		runId,
+	);
+	const store = getCheckpointStore(stateRoot);
+	return store.load(runId, taskId);
 }
 
 /**
  * Delete a checkpoint after successful completion.
  */
-export function clearCheckpoint(runId: string, taskId: string): void {
-  assertSafePathId("runId", runId);
-  assertSafePathId("taskId", taskId);
-  const stateRoot = path.join(process.cwd(), ".crew/state/runs", runId);
-  const store = getCheckpointStore(stateRoot);
-  store.delete(runId, taskId);
+export function clearCheckpoint(
+	runId: string,
+	taskId: string,
+	cwd?: string,
+): void {
+	assertSafePathId("runId", runId);
+	assertSafePathId("taskId", taskId);
+	const stateRoot = path.join(
+		projectCrewRoot(cwd ?? process.cwd()),
+		"state",
+		"runs",
+		runId,
+	);
+	const store = getCheckpointStore(stateRoot);
+	store.delete(runId, taskId);
 }
 
 /**
  * Check if a checkpoint exists for a task.
  */
-export function hasCheckpoint(runId: string, taskId: string): boolean {
-  assertSafePathId("runId", runId);
-  assertSafePathId("taskId", taskId);
-  const stateRoot = path.join(process.cwd(), ".crew/state/runs", runId);
-  const store = getCheckpointStore(stateRoot);
-  return store.hasCheckpoint(runId, taskId);
+export function hasCheckpoint(
+	runId: string,
+	taskId: string,
+	cwd?: string,
+): boolean {
+	assertSafePathId("runId", runId);
+	assertSafePathId("taskId", taskId);
+	const stateRoot = path.join(
+		projectCrewRoot(cwd ?? process.cwd()),
+		"state",
+		"runs",
+		runId,
+	);
+	const store = getCheckpointStore(stateRoot);
+	return store.hasCheckpoint(runId, taskId);
 }
 
 /**
  * List all checkpoints for a run.
  */
-export function listCheckpoints(runId: string): Checkpoint[] {
-  assertSafePathId("runId", runId);
-  const stateRoot = path.join(process.cwd(), ".crew/state/runs", runId);
-  const store = getCheckpointStore(stateRoot);
-  return store.list(runId);
+export function listCheckpoints(runId: string, cwd?: string): Checkpoint[] {
+	assertSafePathId("runId", runId);
+	const stateRoot = path.join(
+		projectCrewRoot(cwd ?? process.cwd()),
+		"state",
+		"runs",
+		runId,
+	);
+	const store = getCheckpointStore(stateRoot);
+	return store.list(runId);
 }
 
 /**
  * Format a checkpoint for display.
  */
 export function formatCheckpoint(checkpoint: Checkpoint): string {
-  return [
-    `## Checkpoint: ${checkpoint.taskId}`,
-    "",
-    `**Agent:** ${checkpoint.agentId}`,
-    checkpoint.agentModel ? `**Model:** ${checkpoint.agentModel}` : "",
-    "",
-    `**Progress:** ${checkpoint.progress}`,
-    "",
-    `**Step:** ${checkpoint.step}`,
-    `**Saved:** ${new Date(checkpoint.savedAt).toISOString()}`,
-    "",
-    `**Context:** ${checkpoint.context.slice(0, 300)}${checkpoint.context.length > 300 ? "..." : ""}`,
-  ].filter(Boolean).join("\n");
+	return [
+		`## Checkpoint: ${checkpoint.taskId}`,
+		"",
+		`**Agent:** ${checkpoint.agentId}`,
+		checkpoint.agentModel ? `**Model:** ${checkpoint.agentModel}` : "",
+		"",
+		`**Progress:** ${checkpoint.progress}`,
+		"",
+		`**Step:** ${checkpoint.step}`,
+		`**Saved:** ${new Date(checkpoint.savedAt).toISOString()}`,
+		"",
+		`**Context:** ${checkpoint.context.slice(0, 300)}${checkpoint.context.length > 300 ? "..." : ""}`,
+	]
+		.filter(Boolean)
+		.join("\n");
 }
 
 /**
  * Format all checkpoints for a run.
  */
-export function formatAllCheckpoints(runId: string): string {
-  const checkpoints = listCheckpoints(runId);
-  if (checkpoints.length === 0) {
-    return `No checkpoints found for run ${runId}`;
-  }
-
-  return [
-    `# Checkpoints: ${runId}`,
-    "",
-    ...checkpoints.map((cp, i) =>
-      `${i + 1}. **${cp.taskId}** — ${cp.progress} (${new Date(cp.savedAt).toLocaleString()})`,
-    ),
-    "",
-    `Use \`team action='resume' runId=${runId} taskId=<taskId>\` to resume.`,
-  ].join("\n");
+export function formatAllCheckpoints(runId: string, cwd?: string): string {
+	const checkpoints = listCheckpoints(runId, cwd);
+	if (checkpoints.length === 0) {
+		return `No checkpoints found for run ${runId}`;
+	}
+
+	return [
+		`# Checkpoints: ${runId}`,
+		"",
+		...checkpoints.map(
+			(cp, i) =>
+				`${i + 1}. **${cp.taskId}** — ${cp.progress} (${new Date(cp.savedAt).toLocaleString()})`,
+		),
+		"",
+		`Use \`team action='resume' runId=${runId} taskId=<taskId>\` to resume.`,
+	].join("\n");
 }