pi-workflow-engine 0.8.0 → 0.10.0

package/.pi/extensions/pi-workflow-engine/src/concurrency.ts

@@ -1,5 +1,5 @@
 import { performance } from "node:perf_hooks";
-import { abortReason, throwIfAborted } from "./cancellation.ts";
+import { abortReason, isFatalWorkflowError, throwIfAborted } from "./cancellation.ts";
 
 /**
  * A counting semaphore: bounds how many async tasks run at once.
@@ -68,15 +68,24 @@ export interface ParallelOptions {
   readonly limit?: number;
 }
 
-/** Run every thunk concurrently and wait for all results (a barrier). */
-export async function parallel<T>(thunks: Array<() => Promise<T>>, options: ParallelOptions = {}): Promise<T[]> {
+export interface PipelineOptions {
+  readonly signal?: AbortSignal;
+  readonly abortController?: AbortController;
+}
+
+export function compactResults<T>(values: ReadonlyArray<T | null | undefined>): T[] {
+  return values.filter((value): value is T => value != null);
+}
+
+/** Run every thunk concurrently and wait for all results; recoverable failures become null slots. */
+export async function parallel<T>(thunks: Array<() => Promise<T>>, options: ParallelOptions = {}): Promise<Array<T | null>> {
   throwIfAborted(options.signal);
-  const results = new Array<T>(thunks.length);
+  const results = new Array<T | null>(thunks.length);
   let remaining = thunks.length;
   if (remaining === 0) return results;
 
   const limit = normalizeLimit(options.limit, thunks.length);
-  return await new Promise<T[]>((resolve, reject) => {
+  return await new Promise<Array<T | null>>((resolve, reject) => {
     let rejected = false;
     let active = 0;
     let next = 0;
@@ -88,6 +97,18 @@ export async function parallel<T>(thunks: Array<() => Promise<T>>, options: Para
       cleanup();
       reject(error);
     };
+    const settleSlot = (index: number, value: T | null) => {
+      if (rejected) return;
+      active--;
+      results[index] = value;
+      remaining--;
+      if (remaining === 0) {
+        cleanup();
+        resolve(results);
+        return;
+      }
+      launchMore();
+    };
     const onAbort = () => rejectOnce(abortReason(options.signal));
     const launchMore = () => {
       while (!rejected && active < limit && next < thunks.length) {
@@ -99,19 +120,14 @@ export async function parallel<T>(thunks: Array<() => Promise<T>>, options: Para
             return thunks[index]();
           })
           .then(
-            (value) => {
-              if (rejected) return;
-              active--;
-              results[index] = value;
-              remaining--;
-              if (remaining === 0) {
-                cleanup();
-                resolve(results);
+            (value) => settleSlot(index, value),
+            (error: unknown) => {
+              if (isFatalWorkflowError(error, options.signal)) {
+                rejectOnce(error);
                 return;
               }
-              launchMore();
+              settleSlot(index, null);
             },
-            (error: unknown) => rejectOnce(error),
           );
       }
     };
@@ -137,33 +153,77 @@ function normalizeLimit(limit: number | undefined, itemCount: number): number {
 export async function pipeline<Item, A>(
   items: readonly Item[],
   stage1: (prev: Item, item: Item, index: number) => Promise<A>,
-): Promise<A[]>;
+): Promise<Array<A | null>>;
 export async function pipeline<Item, A, B>(
   items: readonly Item[],
   stage1: (prev: Item, item: Item, index: number) => Promise<A>,
   stage2: (prev: A, item: Item, index: number) => Promise<B>,
-): Promise<B[]>;
+): Promise<Array<B | null>>;
 export async function pipeline<Item, A, B, C>(
   items: readonly Item[],
   stage1: (prev: Item, item: Item, index: number) => Promise<A>,
   stage2: (prev: A, item: Item, index: number) => Promise<B>,
   stage3: (prev: B, item: Item, index: number) => Promise<C>,
-): Promise<C[]>;
+): Promise<Array<C | null>>;
 export async function pipeline(
   items: readonly unknown[],
   ...stages: Array<(prev: unknown, item: unknown, index: number) => Promise<unknown>>
-): Promise<unknown[]>;
+): Promise<Array<unknown | null>>;
 export async function pipeline(
   items: readonly unknown[],
   ...stages: Array<(prev: unknown, item: unknown, index: number) => Promise<unknown>>
-): Promise<unknown[]> {
+): Promise<Array<unknown | null>> {
+  return pipelineWithOptions(items, stages);
+}
+
+export function bindPipeline(options: PipelineOptions): Pipeline {
+  async function boundPipeline<Item, A>(
+    items: readonly Item[],
+    stage1: (prev: Item, item: Item, index: number) => Promise<A>,
+  ): Promise<Array<A | null>>;
+  async function boundPipeline<Item, A, B>(
+    items: readonly Item[],
+    stage1: (prev: Item, item: Item, index: number) => Promise<A>,
+    stage2: (prev: A, item: Item, index: number) => Promise<B>,
+  ): Promise<Array<B | null>>;
+  async function boundPipeline<Item, A, B, C>(
+    items: readonly Item[],
+    stage1: (prev: Item, item: Item, index: number) => Promise<A>,
+    stage2: (prev: A, item: Item, index: number) => Promise<B>,
+    stage3: (prev: B, item: Item, index: number) => Promise<C>,
+  ): Promise<Array<C | null>>;
+  async function boundPipeline(
+    items: readonly unknown[],
+    ...stages: Array<(prev: unknown, item: unknown, index: number) => Promise<unknown>>
+  ): Promise<Array<unknown | null>> {
+    return pipelineWithOptions(items, stages, options);
+  }
+  return boundPipeline;
+}
+
+export async function pipelineWithOptions(
+  items: readonly unknown[],
+  stages: Array<(prev: unknown, item: unknown, index: number) => Promise<unknown>>,
+  options: PipelineOptions = {},
+): Promise<Array<unknown | null>> {
+  throwIfAborted(options.signal);
   return Promise.all(
     items.map(async (item, index) => {
+      throwIfAborted(options.signal);
       let acc: unknown = item;
-      for (const stage of stages) {
-        acc = await stage(acc, item, index);
+      try {
+        for (const stage of stages) {
+          throwIfAborted(options.signal);
+          acc = await stage(acc, item, index);
+        }
+        return acc;
+      } catch (error) {
+        if (isFatalWorkflowError(error, options.signal)) {
+          options.abortController?.abort(error);
+          throw error;
+        }
+        return null;
       }
-      return acc;
     }),
   );
 }

package/.pi/extensions/pi-workflow-engine/src/diff-capture.ts

@@ -1,6 +1,5 @@
-import { spawn } from "node:child_process";
-import { performance } from "node:perf_hooks";
 import type { PerfSink } from "./perf.ts";
+import { runBoundedProcess } from "./process-runner.ts";
 
 export interface AllowedDiffCommand {
   readonly file: "git" | "gh";
@@ -75,76 +74,23 @@ export async function captureDiff(command: string, options: DiffCaptureOptions):
     return { ok: false, stdout: "", durationMs: 0, bytes: 0, error: parsed.error };
   }
 
-  const start = performance.now();
-  let stdout = "";
-  let stderr = "";
-  let bytes = 0;
-  let error: string | undefined;
-  const child = spawn(parsed.file, [...parsed.args], {
+  const result = await runBoundedProcess({
+    file: parsed.file,
+    args: parsed.args,
     cwd: options.cwd,
     env: diffCaptureEnv(options.env),
-    stdio: ["ignore", "pipe", "pipe"],
-  });
-
-  return await new Promise<DiffCaptureResult>((resolve) => {
-    let settled = false;
-    let forceKillTimer: ReturnType<typeof setTimeout> | undefined;
-    const killGraceMs = Math.max(1, options.killGraceMs ?? 100);
-    const finish = (ok: boolean, finishError?: string) => {
-      if (settled) return;
-      settled = true;
-      cleanup();
-      const durationMs = performance.now() - start;
-      options.perf?.observe("diff.capture_ms", durationMs);
-      options.perf?.observe("diff.bytes", bytes);
-      resolve({ ok, stdout, durationMs, bytes, error: finishError });
-    };
-    const kill = (message: string) => {
-      error = error ?? message;
-      child.kill("SIGTERM");
-      forceKillTimer ??= setTimeout(() => {
-        child.kill("SIGKILL");
-        finish(false, error);
-      }, killGraceMs);
-    };
-    const onAbort = () => kill("diff capture aborted");
-    const timeout = setTimeout(() => kill(`diff capture timed out after ${options.timeoutMs}ms`), options.timeoutMs);
-    const cleanup = () => {
-      clearTimeout(timeout);
-      if (forceKillTimer) clearTimeout(forceKillTimer);
-      options.signal?.removeEventListener("abort", onAbort);
-    };
-
-    if (options.signal?.aborted) {
-      kill("diff capture aborted");
-    } else {
-      options.signal?.addEventListener("abort", onAbort, { once: true });
-    }
-
-    child.stdout?.on("data", (chunk: Buffer) => {
-      bytes += chunk.length;
-      if (bytes > options.maxBufferBytes) {
-        kill(`diff capture exceeded ${options.maxBufferBytes} bytes`);
-        return;
-      }
-      stdout += chunk.toString("utf8");
-    });
-    child.stderr?.on("data", (chunk: Buffer) => {
-      stderr += chunk.toString("utf8");
-    });
-    child.on("error", (spawnError) => finish(false, spawnError.message));
-    child.on("close", (code, signal) => {
-      if (error) {
-        finish(false, error);
-        return;
-      }
-      if (code === 0) {
-        finish(true);
-        return;
-      }
-      finish(false, stderr.trim() || `diff command exited with code ${code ?? `signal ${signal ?? "unknown"}`}`);
-    });
+    signal: options.signal,
+    timeoutMs: options.timeoutMs,
+    maxBufferBytes: options.maxBufferBytes,
+    killGraceMs: options.killGraceMs,
+    abortError: "diff capture aborted",
+    timeoutError: `diff capture timed out after ${options.timeoutMs}ms`,
+    maxBufferError: `diff capture exceeded ${options.maxBufferBytes} bytes`,
+    exitError: (stderr, code, signal) => stderr.trim() || `diff command exited with code ${code ?? `signal ${signal ?? "unknown"}`}`,
   });
+  options.perf?.observe("diff.capture_ms", result.durationMs);
+  options.perf?.observe("diff.bytes", result.bytes);
+  return result;
 }
 
 function diffCaptureEnv(env: NodeJS.ProcessEnv | undefined): NodeJS.ProcessEnv {

package/.pi/extensions/pi-workflow-engine/src/dynamax.ts

@@ -32,6 +32,7 @@ Inline workflow rules:
 - Use the injected Type object for schemas, for example Type.Object({ ok: Type.Boolean() }); do not import typebox.
 - Do not use import statements or dynamic import() in inline workflow scripts.
 - Set thinkingLevel explicitly on each agent() call so fan-out remains bounded.
+- Subagents receive no skills by default. Add \`skills: ["skill-name"]\` per agent only when that stage should load that skill; grant the smallest useful set.
 - Provide exactly one of workflow.name or workflow.script.
 `;
 

package/.pi/extensions/pi-workflow-engine/src/engine.ts

@@ -1,6 +1,7 @@
 import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
-import { parallel, pipeline, Semaphore } from "./concurrency.ts";
+import { bindPipeline, parallel, Semaphore } from "./concurrency.ts";
 import { linkAbortSignal } from "./cancellation.ts";
+import { createBudget } from "./budget.ts";
 import { runAgent, type RunContext } from "./agent-runner.ts";
 import { ProgressTracker } from "./progress.ts";
 import { createPerfRecorder, type PerfSnapshot } from "./perf.ts";
@@ -8,6 +9,8 @@ import { createWorkflowUsageRecorder } from "./usage.ts";
 import { defaultConcurrency, resolveWorkflowRunOptions } from "./options.ts";
 import type { AgentOptions, WorkflowApi, WorkflowModule, WorkflowProgressEvent, WorkflowRef, WorkflowRunOptions } from "./types.ts";
 import { WorkflowInspector } from "./ui/workflow-inspector.ts";
+import { createWorkflowJournal, createWorkflowRunId, pruneWorkflowJournals, workflowJournalPath } from "./journal.ts";
+import { WorktreeRegistry } from "./worktree.ts";
 
 /** Default global cap on concurrent agents per run. */
 const DEFAULT_CONCURRENCY = defaultConcurrency();
@@ -60,6 +63,14 @@ export async function runWorkflow(
 
   const perf = resolvedOptions.perfRecorder ?? createPerfRecorder(resolvedOptions.perf);
   const usage = createWorkflowUsageRecorder();
+  const budget = createBudget(resolvedOptions.budget ?? null, usage);
+  const runId = resolvedOptions.runId ?? createWorkflowRunId();
+  const journalPath = workflowJournalPath(ctx.cwd, runId);
+  const resumePath = resolvedOptions.resumeFromRunId ? workflowJournalPath(ctx.cwd, resolvedOptions.resumeFromRunId) : undefined;
+  const journal = await createWorkflowJournal({ resumePath, writePath: journalPath });
+  const worktrees = new WorktreeRegistry(ctx.cwd);
+  resolvedOptions.onRunMetadata?.({ runId, resumedFromRunId: resolvedOptions.resumeFromRunId, journalPath });
+  progress.log(resolvedOptions.resumeFromRunId ? `run id: ${runId} (resuming from ${resolvedOptions.resumeFromRunId})` : `run id: ${runId}`);
   const runAbortController = new AbortController();
   const unlinkContextAbortSignal = linkAbortSignal(ctx.signal, runAbortController);
   const unlinkOptionAbortSignal = linkAbortSignal(resolvedOptions.signal, runAbortController);
@@ -72,6 +83,9 @@ export async function runWorkflow(
     signal: runAbortController.signal,
     perf,
     usage,
+    budget,
+    journal,
+    worktrees,
   };
 
   try {
@@ -99,6 +113,13 @@ export async function runWorkflow(
       resolvedOptions.onProgressSource?.(undefined);
       unlinkContextAbortSignal();
       unlinkOptionAbortSignal();
+      const cleanupResults = await worktrees.removeAll();
+      for (const result of cleanupResults) {
+        if (!result.ok) {
+          progress.log(`failed to remove isolated worktree ${result.path} (${result.error ?? (result.stderr.trim() || "unknown error")})`);
+        }
+      }
+      await pruneWorkflowJournals(ctx.cwd);
     }
   }
 }
@@ -156,12 +177,13 @@ export async function runWorkflowWithContext(
         abortController: opts.abortController,
         limit: opts.submissionLimit,
       }),
-    pipeline,
+    pipeline: bindPipeline({ signal: rc.signal, abortController: opts.abortController }),
     phase: scope.phase,
     log: scope.log,
     progress: scope.event,
     args,
     cwd: rc.cwd,
+    budget: rc.budget,
     signal: rc.signal,
   };
 

package/.pi/extensions/pi-workflow-engine/src/journal.ts

@@ -0,0 +1,200 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readdir, readFile, rm, stat, appendFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import type { AgentOptions } from "./types.ts";
+
+export const WORKFLOW_RUNS_DIR = join(".pi", ".workflow-runs");
+export const WORKFLOW_JOURNAL_KEEP = 50;
+
+export interface JournalEntry {
+  readonly key: string;
+  readonly value: unknown;
+}
+
+export type JournalLookup = { readonly hit: true; readonly value: unknown } | { readonly hit: false };
+export type JournalRecordResult = { readonly ok: true } | { readonly ok: false; readonly error: string };
+
+export interface WorkflowJournal {
+  lookup(key: string): JournalLookup;
+  record(key: string, value: unknown): Promise<JournalRecordResult>;
+}
+
+export class WorkflowJournalLoadError extends Error {
+  override readonly name = "WorkflowJournalLoadError";
+}
+
+export function createWorkflowRunId(): string {
+  return randomUUID();
+}
+
+export function workflowJournalPath(cwd: string, runId: string): string {
+  return join(cwd, WORKFLOW_RUNS_DIR, `${validateRunId(runId)}.jsonl`);
+}
+
+export function agentJournalKey(prompt: string, opts: AgentOptions = {}): string {
+  const behaviorHash = hashAgentCall(prompt, opts);
+  const cacheKey = opts.cacheKey?.trim();
+  if (!cacheKey) return `agent:${behaviorHash}`;
+  const cacheKeyHash = createHash("sha256").update(cacheKey).digest("hex");
+  return `agent:${cacheKeyHash}:${behaviorHash}`;
+}
+
+export function hashAgentCall(prompt: string, opts: AgentOptions = {}): string {
+  const behavior = {
+    prompt,
+    opts: {
+      model: opts.model,
+      thinkingLevel: opts.thinkingLevel,
+      tools: sortedArray(opts.tools),
+      toolHints: sortedArray(opts.toolHints),
+      skills: sortedArray(opts.skills),
+      schema: opts.schema,
+      isolation: opts.isolation,
+    },
+  };
+  return createHash("sha256").update(stableStringify(behavior)).digest("hex");
+}
+
+export async function loadJournalEntries(path: string, options: { readonly required?: boolean } = {}): Promise<JournalEntry[]> {
+  let content: string;
+  try {
+    content = await readFile(path, "utf8");
+  } catch (error) {
+    if (options.required) {
+      throw new WorkflowJournalLoadError(`Could not load workflow resume journal at ${path}: ${formatError(error)}`);
+    }
+    return [];
+  }
+
+  const entries: JournalEntry[] = [];
+  let lineNumber = 0;
+  for (const line of content.split(/\r?\n/)) {
+    lineNumber += 1;
+    if (line.trim() === "") continue;
+    try {
+      const parsed = JSON.parse(line) as unknown;
+      if (isJournalEntry(parsed)) {
+        entries.push(parsed);
+        continue;
+      }
+      if (options.required) {
+        throw new WorkflowJournalLoadError(`Workflow resume journal at ${path}:${lineNumber} has an incompatible entry format.`);
+      }
+    } catch (error) {
+      if (options.required) {
+        if (error instanceof WorkflowJournalLoadError) throw error;
+        throw new WorkflowJournalLoadError(`Workflow resume journal at ${path}:${lineNumber} is not valid JSON.`);
+      }
+    }
+  }
+  return entries;
+}
+
+export async function createWorkflowJournal(options: {
+  readonly resumePath?: string;
+  readonly writePath: string;
+}): Promise<WorkflowJournal> {
+  const priorEntries = options.resumePath ? await loadJournalEntries(options.resumePath, { required: true }) : [];
+  return createMemoryBackedJournal(priorEntries, options.writePath);
+}
+
+export function createMemoryBackedJournal(priorEntries: readonly JournalEntry[] = [], writePath?: string): WorkflowJournal {
+  const priorByKey = new Map<string, JournalEntry | "ambiguous">();
+  for (const entry of priorEntries) {
+    priorByKey.set(entry.key, priorByKey.has(entry.key) ? "ambiguous" : entry);
+  }
+
+  return {
+    lookup(key) {
+      const entry = priorByKey.get(key);
+      if (!entry || entry === "ambiguous") return { hit: false };
+      return { hit: true, value: entry.value };
+    },
+    async record(key, value) {
+      const entry = { key, value };
+      if (writePath) {
+        try {
+          await mkdir(dirname(writePath), { recursive: true });
+          await appendFile(writePath, `${JSON.stringify(entry)}\n`, "utf8");
+        } catch (error) {
+          return { ok: false, error: formatError(error) };
+        }
+      }
+      return { ok: true };
+    },
+  };
+}
+
+export async function pruneWorkflowJournals(cwd: string, keep = WORKFLOW_JOURNAL_KEEP): Promise<void> {
+  const dir = join(cwd, WORKFLOW_RUNS_DIR);
+  let files: string[];
+  try {
+    files = await readdir(dir);
+  } catch {
+    return;
+  }
+
+  const candidates = await Promise.all(
+    files
+      .filter((file) => file.endsWith(".jsonl"))
+      .map(async (file) => {
+        const path = join(dir, file);
+        try {
+          const info = await stat(path);
+          return { path, mtimeMs: info.mtimeMs };
+        } catch {
+          return undefined;
+        }
+      }),
+  );
+
+  const stale = candidates
+    .filter((entry): entry is { readonly path: string; readonly mtimeMs: number } => entry !== undefined)
+    .sort((a, b) => b.mtimeMs - a.mtimeMs)
+    .slice(Math.max(0, keep));
+
+  await Promise.all(stale.map((entry) => rm(entry.path, { force: true }).catch(() => undefined)));
+}
+
+function validateRunId(runId: string): string {
+  const trimmed = runId.trim();
+  if (!/^[A-Za-z0-9_.-]+$/.test(trimmed)) {
+    throw new Error("Workflow run id must contain only letters, numbers, dots, underscores, or hyphens.");
+  }
+  return trimmed;
+}
+
+function sortedArray(values: readonly string[] | undefined): readonly string[] | undefined {
+  return values ? [...values].sort() : undefined;
+}
+
+function stableStringify(value: unknown): string {
+  return JSON.stringify(normalizeStable(value));
+}
+
+function normalizeStable(value: unknown): unknown {
+  if (value === undefined) return undefined;
+  if (value === null) return null;
+  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
+  if (Array.isArray(value)) return value.map(normalizeStable);
+  if (typeof value === "object") {
+    const record = value as Record<string, unknown>;
+    const out: Record<string, unknown> = {};
+    for (const key of Object.keys(record).sort()) {
+      const normalized = normalizeStable(record[key]);
+      if (normalized !== undefined) out[key] = normalized;
+    }
+    return out;
+  }
+  return String(value);
+}
+
+function isJournalEntry(value: unknown): value is JournalEntry {
+  if (typeof value !== "object" || value === null) return false;
+  const entry = value as { readonly key?: unknown; readonly value?: unknown };
+  return typeof entry.key === "string" && "value" in entry;
+}
+
+function formatError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/.pi/extensions/pi-workflow-engine/src/options.ts

@@ -1,10 +1,14 @@
 import { cpus } from "node:os";
 import type { WorkflowRunOptions } from "./types.ts";
 
+export const WORKFLOW_BUDGET_MIN = 1;
+export const WORKFLOW_BUDGET_MAX = 1_000_000_000;
+
 export interface ResolvedWorkflowRunOptions extends WorkflowRunOptions {
   readonly perf: boolean;
   readonly concurrency: number;
   readonly parallelSubmissionLimit?: number;
+  readonly budget?: number;
 }
 
 export function defaultConcurrency(cpuCount = cpus().length): number {
@@ -21,16 +25,18 @@ export function resolveWorkflowRunOptions(
     1,
     10_000,
   );
+  const budget = resolveBudget(input.budget, env.PI_WORKFLOW_BUDGET);
   return {
     ...input,
     perf: input.perf ?? env.PI_WORKFLOW_PERF === "1",
     concurrency,
     parallelSubmissionLimit,
+    budget,
   };
 }
 
 function optionalClampedInteger(value: number | undefined, min: number, max: number): number | undefined {
-  if (value === undefined) return undefined;
+  if (value === undefined || !Number.isFinite(value)) return undefined;
   return clampInteger(value, min, max, value);
 }
 
@@ -44,3 +50,23 @@ function parseInteger(value: string | undefined): number | undefined {
   const parsed = Number(value);
   return Number.isFinite(parsed) ? Math.trunc(parsed) : undefined;
 }
+
+function resolveBudget(inputBudget: number | undefined, envBudget: string | undefined): number | undefined {
+  if (inputBudget !== undefined) return normalizeExplicitBudget(inputBudget);
+  return envBudget === undefined ? undefined : parseWorkflowBudgetString(envBudget);
+}
+
+function normalizeExplicitBudget(value: number): number {
+  if (!Number.isFinite(value) || !Number.isInteger(value) || value < WORKFLOW_BUDGET_MIN || value > WORKFLOW_BUDGET_MAX) {
+    throw new RangeError(`Workflow budget must be an integer between ${WORKFLOW_BUDGET_MIN} and ${WORKFLOW_BUDGET_MAX}.`);
+  }
+  return value;
+}
+
+export function parseWorkflowBudgetString(value: string): number | undefined {
+  const trimmed = value.trim();
+  if (!/^\d+$/.test(trimmed)) return undefined;
+  const parsed = Number(trimmed);
+  if (!Number.isSafeInteger(parsed) || parsed < WORKFLOW_BUDGET_MIN || parsed > WORKFLOW_BUDGET_MAX) return undefined;
+  return parsed;
+}