pi-workflow-engine 0.12.0 → 1.0.0

package/.pi/extensions/pi-workflow-engine/src/agent-session.ts

@@ -1,4 +1,4 @@
-import type { Api, Model } from "@earendil-works/pi-ai";
+import type { Api, AssistantMessage, Model } from "@earendil-works/pi-ai";
 import {
   createAgentSessionFromServices,
   createAgentSessionServices,
@@ -12,7 +12,7 @@ import {
 import { prepareAgentSkillResources } from "./agent-skills.ts";
 import type {
   AgentExecutionOptions,
-  AgentRunTags,
+  AgentProgress,
   AgentRunnerSession,
   RunContext,
 } from "./agent-runner-types.ts";
@@ -24,7 +24,8 @@ import {
 import { providerErrorFromMessages } from "./agent-retry.ts";
 import { synchronizeWorkflowModelRuntime } from "./agent-session-providers.ts";
 import { matchesAgentToolHint, WorkflowToolHintUnavailableError } from "./tool-capabilities.ts";
-import type { AgentToolHint } from "./types.ts";
+import { truncateText } from "./text.ts";
+import { WORKFLOW_TOOL_NAME, type AgentToolHint } from "./types.ts";
 
 export const FINAL_TOOL = "final_answer";
 
@@ -32,17 +33,12 @@ const SCHEMA_REPROMPT =
   `You ended your turn without calling the ${FINAL_TOOL} tool, so no result was recorded. ` +
   `Call ${FINAL_TOOL} now with your final answer as its arguments. Do not reply with plain text.`;
 const DEFAULT_SEARCH_BASE_TOOLS = ["read", "bash", "grep", "find", "ls"];
-
-export interface ResolvedAgentModelRequest {
-  readonly ref: string;
-  readonly provider: string;
-  readonly id: string;
-}
-
-export interface ResolvedAgentModel {
-  readonly model: Model<Api> | undefined;
-  readonly requested: ResolvedAgentModelRequest | undefined;
-}
+const RETRY_REASON_CHARS = 160;
+const RETRY_DISABLED_WARNING =
+  "pi auto-retry is disabled (retry.enabled: false in pi settings), so subagents will not retry failed provider turns. " +
+  "pi-workflow-engine 0.12.0 and 0.13.0 wrote that setting into ~/.pi/agent/settings.json; remove it unless you disabled retry on purpose.";
+/** Runs (keyed by their shared progress sink) already warned about disabled pi auto-retry. */
+const retryDisabledWarnedRuns = new WeakSet<AgentProgress>();
 
 export interface AgentSessionHandle {
   readonly session: AgentRunnerSession;
@@ -51,19 +47,14 @@ export interface AgentSessionHandle {
   structuredResult(): unknown;
 }
 
-export function resolveAgentModel(
-  modelRef: string | undefined,
-  modelRegistry: Pick<ModelRegistry, "find">,
-  hostModel: Model<Api> | undefined,
-): ResolvedAgentModel {
-  if (modelRef === undefined) return { model: hostModel, requested: undefined };
-
+/** Resolve an explicit `agent({ model })` ref; unknown refs fail instead of falling back to another model. */
+export function resolveAgentModel(modelRef: string, modelRegistry: Pick<ModelRegistry, "find">): Model<Api> {
   const parsed = parseAgentModelRef(modelRef);
   const found = modelRegistry.find(parsed.provider, parsed.id);
   if (!found) {
     throw new Error(`Agent model "${modelRef}" not found (resolved as ${parsed.provider}/${parsed.id}).`);
   }
-  return { model: found, requested: parsed };
+  return found;
 }
 
 export async function openAgentSession(input: {
@@ -73,9 +64,8 @@ export async function openAgentSession(input: {
   readonly cwd: string;
   readonly model: Model<Api> | undefined;
   readonly label: string;
-  readonly tags: AgentRunTags;
 }): Promise<AgentSessionHandle> {
-  const { rc, prompt, opts, cwd, model, label, tags } = input;
+  const { rc, prompt, opts, cwd, model, label } = input;
   let captured = false;
   let structuredResult: unknown;
   const customTools: ToolDefinition[] = opts.schema
@@ -97,26 +87,22 @@ export async function openAgentSession(input: {
   let session: AgentRunnerSession | undefined;
   try {
     throwIfAborted(rc.signal);
-    const resourceInput = { rc, prompt, opts, cwd, model, customTools, label };
-    const resources = rc.createSession
-      ? await prepareAgentSessionResources(resourceInput)
-      : await rc.perf.time(
-          "agent.session_resources_ms",
-          () => prepareAgentSessionResources(resourceInput),
-          tags,
-        );
+    const resources = await rc.perf.time(
+      "agent.session_resources_ms",
+      () => prepareAgentSessionResources({ rc, prompt, opts, cwd, model, customTools, label }),
+    );
     const toolSelection = buildToolSelection(opts, resources.selectedSkills.length > 0);
     session = (
       await rc.perf.time(
         "agent.create_session_ms",
         () => resources.createSession(toolSelection.sessionOptions),
-        tags,
       )
     ).session;
-    session.setAutoRetryEnabled(false);
     const matchedToolHints = toolSelection.toolHints.length === 0
       ? new Set<AgentToolHint>()
       : applyDynamicToolHints(session, toolSelection);
+    // A nested run would get its own concurrency cap and budget, so subagents may start one only when allowlisted.
+    if (!opts.tools?.includes(WORKFLOW_TOOL_NAME)) withholdWorkflowTool(session);
     if (opts.requireToolHints) {
       const missing = toolSelection.toolHints.filter((hint) => !matchedToolHints.has(hint));
       if (missing.length > 0) throw new WorkflowToolHintUnavailableError(missing);
@@ -129,11 +115,8 @@ export async function openAgentSession(input: {
       structuredResult: () => structuredResult,
     };
   } catch (error) {
-    if (session) {
-      const activeSession = session;
-      session = undefined;
-      rc.perf.timeSync("agent.dispose_ms", () => activeSession.dispose(), tags);
-    }
+    const created = session;
+    if (created) rc.perf.timeSync("agent.dispose_ms", () => created.dispose());
     throw error;
   }
 }
@@ -145,70 +128,79 @@ export async function promptAgentSession(input: {
   readonly opts: AgentExecutionOptions;
   readonly label: string;
   readonly rowId: number;
-  readonly tags: AgentRunTags;
+  readonly phase: string;
 }): Promise<unknown> {
-  const { rc, handle, prompt, opts, label, rowId, tags } = input;
+  const { rc, handle, prompt, opts, label, rowId, phase } = input;
   const { session } = handle;
+  throwIfAborted(rc.signal);
+  // pi drops a retried turn's failed message from session.messages; keep it so usage covers every attempt.
+  const retriedFailures: AssistantMessage[] = [];
+  let lastFailure: AssistantMessage | undefined;
   const unsubscribe = session.subscribe((event) => {
-    if (event.type === "tool_execution_start" && event.toolName !== undefined && event.toolName !== FINAL_TOOL) {
-      rc.progress.agentTool(label, event.toolName, rowId);
+    if (event.type === "tool_execution_start" && event.toolName !== FINAL_TOOL) {
+      rc.progress.agentTool(rowId, event.toolName);
+    } else if (event.type === "message_end" && event.message.role === "assistant" && event.message.stopReason === "error") {
+      lastFailure = event.message;
+    } else if (event.type === "auto_retry_start") {
+      if (lastFailure) retriedFailures.push(lastFailure);
+      lastFailure = undefined;
+      const reason = truncateText(event.errorMessage, RETRY_REASON_CHARS);
+      rc.progress.log(
+        `${label}: transient provider failure (${reason}); retrying turn ${event.attempt}/${event.maxAttempts} in ${event.delayMs}ms`,
+      );
+      rc.perf.counter("agent.turn_retry");
     }
   });
+  const unlinkPromptAbort = linkSessionAbort(rc.signal, session);
+  const promptSession = async (text: string) => {
+    await rc.perf.time("agent.prompt_ms", () => raceWithAbort(() => session.prompt(text), rc.signal));
+    const failure = providerErrorFromMessages(session.messages, {
+      pauseOnUsageLimit: rc.pauseOnProviderUsageLimit,
+    });
+    if (failure) throw failure;
+  };
 
   try {
-    const finalPrompt = opts.schema
-      ? `${prompt}\n\nWhen finished, return your result by calling the ${FINAL_TOOL} tool.`
-      : prompt;
-    throwIfAborted(rc.signal);
-    const unlinkPromptAbort = linkSessionAbort(rc.signal, session);
-    try {
-      try {
-        const promptSession = async (text: string) => {
-          await rc.perf.time("agent.prompt_ms", () => raceWithAbort(() => session.prompt(text), rc.signal), tags);
-          const failure = providerErrorFromMessages(session.messages, {
-            pauseOnUsageLimit: rc.pauseOnProviderUsageLimit,
-          });
-          if (failure) throw failure;
-        };
-        await promptSession(finalPrompt);
-        if (opts.schema) {
-          for (let attempt = 0; !handle.hasStructuredResult() && attempt < MAX_SCHEMA_REPAIR_ATTEMPTS; attempt++) {
-            throwIfAborted(rc.signal);
-            session.setActiveToolsByName([FINAL_TOOL]);
-            rc.progress.log(`${label}: no final answer; re-prompting (${attempt + 1}/${MAX_SCHEMA_REPAIR_ATTEMPTS})`);
-            rc.perf.counter("agent.structured_reprompt", 1, tags);
-            await promptSession(SCHEMA_REPROMPT);
-          }
-        }
-
+    await promptSession(
+      opts.schema ? `${prompt}\n\nWhen finished, return your result by calling the ${FINAL_TOOL} tool.` : prompt,
+    );
+    if (opts.schema && !handle.hasStructuredResult()) {
+      const activeTools = session.getActiveToolNames();
+      for (let attempt = 0; !handle.hasStructuredResult() && attempt < MAX_SCHEMA_REPAIR_ATTEMPTS; attempt++) {
         throwIfAborted(rc.signal);
-        return rc.perf.timeSync(
-          "agent.extract_result_ms",
-          () => {
-            if (!opts.schema) {
-              return session.getLastAssistantText() ?? "";
-            }
-            if (!handle.hasStructuredResult()) {
-              rc.progress.log(`${label}: no structured answer returned`);
-              rc.perf.counter("agent.structured_missing", 1, tags);
-              throw new WorkflowStructuredOutputError(label, MAX_SCHEMA_REPAIR_ATTEMPTS);
-            }
-            return handle.structuredResult();
-          },
-          tags,
-        );
-      } finally {
-        rc.usage.recordAgentSession({ label, phase: tags.phase, messages: session.messages });
+        session.setActiveToolsByName([FINAL_TOOL]);
+        rc.progress.log(`${label}: no final answer; re-prompting (${attempt + 1}/${MAX_SCHEMA_REPAIR_ATTEMPTS})`);
+        rc.perf.counter("agent.structured_reprompt");
+        await promptSession(SCHEMA_REPROMPT);
       }
-    } finally {
-      unlinkPromptAbort();
+      // Narrowing also rebuilds pi's system prompt; restore both so replay validation sees the captured identity.
+      session.setActiveToolsByName(activeTools);
     }
+
+    throwIfAborted(rc.signal);
+    return rc.perf.timeSync(
+      "agent.extract_result_ms",
+      () => {
+        if (!opts.schema) {
+          return session.getLastAssistantText() ?? "";
+        }
+        if (!handle.hasStructuredResult()) {
+          rc.progress.log(`${label}: no structured answer returned`);
+          rc.perf.counter("agent.structured_missing");
+          throw new WorkflowStructuredOutputError(label, MAX_SCHEMA_REPAIR_ATTEMPTS);
+        }
+        return handle.structuredResult();
+      },
+    );
   } finally {
+    // Detach first: recording publishes usage to the UI and must not be able to skip listener cleanup.
+    unlinkPromptAbort();
     unsubscribe();
+    rc.usage.recordAgentSession({ label, phase, messages: [...retriedFailures, ...session.messages] });
   }
 }
 
-function parseAgentModelRef(modelRef: string): ResolvedAgentModelRequest {
+function parseAgentModelRef(modelRef: string): { readonly provider: string; readonly id: string } {
   const normalized = modelRef.trim();
   if (normalized.length === 0) {
     throw new Error('Invalid agent model ref: expected a bare model id or "provider/id".');
@@ -218,13 +210,13 @@ function parseAgentModelRef(modelRef: string): ResolvedAgentModelRequest {
   }
 
   const slash = modelRef.indexOf("/");
-  if (slash === -1) return { ref: modelRef, provider: "anthropic", id: modelRef };
+  if (slash === -1) return { provider: "anthropic", id: modelRef };
   const provider = modelRef.slice(0, slash);
   const id = modelRef.slice(slash + 1);
   if (provider.length === 0 || id.length === 0 || id.startsWith("/")) {
     throw new Error(`Invalid agent model ref "${modelRef}": expected "provider/id".`);
   }
-  return { ref: modelRef, provider, id };
+  return { provider, id };
 }
 
 interface AgentSessionResources {
@@ -282,6 +274,10 @@ async function prepareAgentSessionResources(input: {
   for (const diagnostic of services.diagnostics) {
     rc.progress.log(`${label}: session ${diagnostic.type}: ${diagnostic.message}`);
   }
+  if (!services.settingsManager.getRetryEnabled() && !retryDisabledWarnedRuns.has(rc.progress)) {
+    retryDisabledWarnedRuns.add(rc.progress);
+    rc.progress.log(RETRY_DISABLED_WARNING);
+  }
   const selectedSkills = preparedSkills.resolve(services.resourceLoader);
   return {
     selectedSkills,
@@ -322,16 +318,8 @@ function buildToolSelection(opts: AgentExecutionOptions, skillsEnabled: boolean)
       ? ["read"]
       : undefined;
   const activeTools = buildToolList(opts, skillsEnabled, fallback);
-  if (toolHints.length === 0) {
-    return {
-      sessionOptions: { tools: activeTools ? [...activeTools] : undefined },
-      activeTools,
-      toolHints,
-    };
-  }
-
   return {
-    sessionOptions: { noTools: "builtin" },
+    sessionOptions: toolHints.length === 0 ? { tools: activeTools } : { noTools: "builtin" },
     activeTools,
     toolHints,
   };
@@ -350,11 +338,16 @@ function buildToolList(
   return allow;
 }
 
+function withholdWorkflowTool(session: AgentRunnerSession): void {
+  const active = session.getActiveToolNames();
+  if (active.includes(WORKFLOW_TOOL_NAME)) session.setActiveToolsByName(active.filter((name) => name !== WORKFLOW_TOOL_NAME));
+}
+
 function applyDynamicToolHints(
   session: AgentRunnerSession,
   selection: ToolSelection,
 ): ReadonlySet<AgentToolHint> {
-  const active = new Set(selection.activeTools ?? []);
+  const active = new Set(selection.activeTools);
   const matched = new Set<AgentToolHint>();
   for (const tool of session.getAllTools()) {
     for (const hint of selection.toolHints) {

package/.pi/extensions/pi-workflow-engine/src/agent-skills.ts

@@ -62,12 +62,13 @@ export function resolveAgentSkillRequest(prompt: string, explicitSkills: unknown
 }
 
 export function extractSkillSelectorsFromText(text: string): string[] {
-  const selectors: string[] = [];
-  collectPatternMatches(text, SLASH_SKILL_PATTERN, selectors, 1);
-  collectListPatternMatches(text, SKILLS_FIELD_PATTERN, selectors);
-  collectListPatternMatches(text, VERB_SKILL_LIST_PATTERN, selectors);
-  collectPatternMatches(text, VERB_NAMED_SKILL_PATTERN, selectors, 2);
-  return uniqueSelectors(selectors);
+  const selectors = [
+    ...Array.from(text.matchAll(SLASH_SKILL_PATTERN), (match) => match[1] ?? ""),
+    ...[...text.matchAll(SKILLS_FIELD_PATTERN), ...text.matchAll(VERB_SKILL_LIST_PATTERN)]
+      .flatMap((match) => parseSkillList(match[1] ?? "")),
+    ...Array.from(text.matchAll(VERB_NAMED_SKILL_PATTERN), (match) => match[2] ?? ""),
+  ].map(normalizeSkillSelector);
+  return [...new Set(selectors.filter((value) => value && !isGenericSkillWord(value)))];
 }
 
 export function normalizeSkillSelector(value: string): string {
@@ -179,23 +180,6 @@ export async function captureAgentSkillIdentities(
   return { kind: "verified", skills: identities };
 }
 
-function collectPatternMatches(text: string, pattern: RegExp, selectors: string[], captureIndex: number): void {
-  pattern.lastIndex = 0;
-  for (let match = pattern.exec(text); match !== null; match = pattern.exec(text)) {
-    const value = match[captureIndex];
-    if (value !== undefined) selectors.push(value);
-  }
-}
-
-function collectListPatternMatches(text: string, pattern: RegExp, selectors: string[]): void {
-  pattern.lastIndex = 0;
-  for (let match = pattern.exec(text); match !== null; match = pattern.exec(text)) {
-    const value = match[1];
-    if (value === undefined) continue;
-    selectors.push(...parseSkillList(value));
-  }
-}
-
 function parseSkillList(fragment: string): string[] {
   const scoped = fragment.replace(PURPOSE_BOUNDARY_PATTERN, "");
   return scoped
@@ -208,9 +192,7 @@ function normalizeExplicitSkillSelectors(value: unknown): string[] {
   if (!Array.isArray(value)) {
     throw new Error("Invalid subagent skills option: expected an array of skill names.");
   }
-  const seen = new Set<string>();
-  const result: string[] = [];
-  for (const item of value) {
+  const selectors = value.map((item: unknown) => {
     if (typeof item !== "string") {
       throw new Error("Invalid subagent skills option: every skill name must be a string.");
     }
@@ -218,11 +200,9 @@ function normalizeExplicitSkillSelectors(value: unknown): string[] {
     if (!normalized || isGenericSkillWord(normalized)) {
       throw new Error(`Invalid subagent skills option: \"${item}\" is not a valid skill name.`);
     }
-    if (seen.has(normalized)) continue;
-    seen.add(normalized);
-    result.push(normalized);
-  }
-  return result;
+    return normalized;
+  });
+  return [...new Set(selectors)];
 }
 
 function logSkillDiagnostics(log: ((message: string) => void) | undefined, diagnostics: readonly ResourceDiagnostic[], requested: boolean): void {
@@ -233,18 +213,6 @@ function logSkillDiagnostics(log: ((message: string) => void) | undefined, diagn
   }
 }
 
-function uniqueSelectors(values: Iterable<string>): string[] {
-  const seen = new Set<string>();
-  const result: string[] = [];
-  for (const value of values) {
-    const normalized = normalizeSkillSelector(value);
-    if (!normalized || isGenericSkillWord(normalized) || seen.has(normalized)) continue;
-    seen.add(normalized);
-    result.push(normalized);
-  }
-  return result;
-}
-
 function isGenericSkillWord(value: string): boolean {
   return value === "skill" || value === "skills" || value === "the" || value === "this" || value === "these" || value === "that";
 }

package/.pi/extensions/pi-workflow-engine/src/agent-workspace.ts

@@ -1,5 +1,4 @@
 import type { AgentExecutionOptions, AgentProgress } from "./agent-runner-types.ts";
-import { unknownErrorMessage } from "./unknown-error.ts";
 import type { WorktreeRegistry } from "./worktree.ts";
 
 interface AgentWorkspaceBase {
@@ -21,7 +20,7 @@ export type AgentWorkspace = SharedAgentWorkspace | IsolatedAgentWorkspace;
 
 export interface AgentWorkspaceContext {
   readonly cwd: string;
-  readonly worktrees: Pick<WorktreeRegistry, "probe" | "add" | "capturePatch" | "remove">;
+  readonly worktrees: Pick<WorktreeRegistry, "probe" | "add" | "capturePatch" | "applyPatch" | "remove">;
   readonly signal: AbortSignal | undefined;
   readonly progress: Pick<AgentProgress, "log">;
 }
@@ -31,6 +30,7 @@ export async function createAgentWorkspace(
   opts: AgentExecutionOptions,
   label: string,
 ): Promise<AgentWorkspace> {
+  if (opts.candidatePatch && opts.isolation !== "worktree") throw new Error("Candidate evaluation requires worktree isolation");
   if (opts.isolation !== "worktree") {
     return {
       kind: "shared",
@@ -44,7 +44,7 @@ export async function createAgentWorkspace(
 
   const probe = await rc.worktrees.probe(rc.signal);
   if (!probe.ok) {
-    throw new Error(`Failed to check git worktree availability for isolated agent: ${probe.error ?? "unknown git error"}`);
+    throw new Error(`Failed to check git worktree availability for isolated agent: ${probe.error}`);
   }
   if (!probe.inside) {
     throw new Error("Agent requested worktree isolation, but the workflow cwd is not inside a git work tree.");
@@ -53,6 +53,19 @@ export async function createAgentWorkspace(
   const added = await rc.worktrees.add(rc.signal, opts.worktreeBaseline);
   if ("error" in added) throw new Error(`Failed to create isolated worktree: ${added.error}`);
   const worktreePath = added.path;
+  if (opts.candidatePatch) {
+    try {
+      if (opts.candidatePatch.baselineOid !== added.baselineOid) throw new Error("Candidate baseline differs from evaluator baseline");
+      if (opts.candidatePatch.patch.trim()) {
+        const applied = await rc.worktrees.applyPatch(worktreePath, opts.candidatePatch.patch, rc.signal);
+        if (!applied.ok) throw new Error(`Candidate patch could not be applied: ${applied.error}`);
+      }
+    } catch (error) {
+      const removed = await rc.worktrees.remove(worktreePath);
+      if (!removed.ok) rc.progress.log(`${label}: evaluator setup cleanup failed: ${removed.error}`);
+      throw error;
+    }
+  }
   rc.progress.log(`${label}: using isolated worktree ${worktreePath}`);
 
   return {
@@ -62,28 +75,15 @@ export async function createAgentWorkspace(
     async wrapResult(result) {
       const patch = await rc.worktrees.capturePatch(worktreePath, added.baselineOid, rc.signal);
       if ("error" in patch) throw new Error(`Failed to capture isolated worktree patch: ${patch.error}`);
-      return { result, patch: patch.patch, changed: patch.changed };
+      return { result, patch: patch.patch, changed: patch.changed, baselineOid: added.baselineOid };
     },
     async dispose() {
       const removed = await rc.worktrees.remove(worktreePath);
       if (!removed.ok) {
         rc.progress.log(
-          `${label}: failed to remove isolated worktree (${removed.error ?? (removed.stderr.trim() || "unknown error")})`,
+          `${label}: failed to remove isolated worktree (${removed.error})`,
         );
       }
     },
   };
 }
-
-export async function disposeAgentWorkspace(
-  rc: Pick<AgentWorkspaceContext, "progress">,
-  label: string,
-  workspace: AgentWorkspace | undefined,
-): Promise<void> {
-  if (!workspace) return;
-  try {
-    await workspace.dispose();
-  } catch (error) {
-    rc.progress.log(`${label}: failed to dispose isolated workspace (${unknownErrorMessage(error)})`);
-  }
-}