pi-workflow-engine 0.11.0 → 0.12.0

package/.pi/extensions/pi-workflow-engine/src/agent-replay.ts

@@ -2,7 +2,8 @@ import type { Skill } from "@earendil-works/pi-coding-agent";
 import { Value } from "typebox/value";
 import { captureAgentSkillIdentities, extractSkillSelectorsFromText } from "./agent-skills.ts";
 import type { AgentExecutionOptions, AgentRunnerSession, AgentRunTags, RunContext } from "./agent-runner-types.ts";
-import { captureEffectiveSession, FINAL_TOOL } from "./agent-session.ts";
+import { captureEffectiveAgentSessionIdentity } from "./agent-session-identity.ts";
+import { FINAL_TOOL } from "./agent-session.ts";
 import type { AgentWorkspace, IsolatedAgentWorkspace } from "./agent-workspace.ts";
 import { captureAgentJournalKey } from "./journal.ts";
 import { unknownErrorMessage } from "./unknown-error.ts";
@@ -116,7 +117,9 @@ export async function lookupReplayResult(input: {
   readonly opts: AgentExecutionOptions;
   readonly workspace: AgentWorkspace;
 }): Promise<{ readonly hit: true; readonly result: unknown } | { readonly hit: false; readonly reason?: string }> {
-  const cached = input.rc.journal.lookup(input.key, input.identity);
+  const cached = input.rc.journal.lookup(input.key, input.identity, {
+    allowWorkflowSourceMismatch: input.rc.resumeEditedWorkflow,
+  });
   if (!cached.hit) return cached;
   const validation = await validateCachedResult(cached.value, input.opts, input.rc, input.workspace);
   return validation.ok
@@ -147,7 +150,9 @@ export async function validateReplayIdentity(input: {
     sessionCwd: input.sessionCwd,
   });
   if (current.kind === "unverifiable") return { ok: false, reason: current.reason };
-  const mismatch = resumeContextMismatchReason(input.identity, current.identity);
+  const mismatch = resumeContextMismatchReason(input.identity, current.identity, {
+    allowWorkflowSourceMismatch: input.rc.resumeEditedWorkflow,
+  });
   return mismatch ? { ok: false, reason: mismatch } : { ok: true };
 }
 
@@ -160,6 +165,7 @@ export async function settleAgentAttempt(input: {
 }): Promise<AgentAttemptSettlement> {
   const { rc, label, replay, outcome } = input;
   if (outcome.kind === "live-unrecordable") {
+    recordAgentResultSource(rc, "live");
     return { kind: "done", result: outcome.result };
   }
   if (!isReplayEnabled(replay)) throw new Error("Replayable agent outcome produced without an enabled replay plan.");
@@ -183,10 +189,23 @@ export async function settleAgentAttempt(input: {
   if (outcome.kind === "cache-hit") {
     rc.progress.log(`${label}: using cached result from workflow journal`);
     rc.perf.counter("agent.cache_hit", 1, input.tags);
+    recordAgentResultSource(rc, "cached");
+  } else {
+    recordAgentResultSource(rc, "live");
   }
   return { kind: "done", result: outcome.result };
 }
 
+function recordAgentResultSource(rc: RunContext, source: "cached" | "live"): void {
+  if (!rc.resumeEditedWorkflow) return;
+  rc.progress.event({
+    type: "counter_delta",
+    key: `resume.${source}`,
+    label: source === "cached" ? "Cached agents" : "Live agents",
+    delta: 1,
+  });
+}
+
 async function captureVerifiedReplayIdentity(input: {
   readonly rc: RunContext;
   readonly repository: VerifiedRepositoryResumeContext;
@@ -201,7 +220,11 @@ async function captureVerifiedReplayIdentity(input: {
       workspaceRoot: input.rc.cwd,
       signal: input.rc.signal,
     }),
-    captureEffectiveSession(input.session, input.sessionCwd, input.rc.cwd, input.rc.signal),
+    captureEffectiveAgentSessionIdentity(input.session, {
+      sessionCwd: input.sessionCwd,
+      workspaceRoot: input.rc.cwd,
+      signal: input.rc.signal,
+    }),
   ]);
   if (skills.kind === "unverifiable") return skills;
   if (effectiveSession.kind === "unverifiable") return effectiveSession;

package/.pi/extensions/pi-workflow-engine/src/agent-retry.ts

@@ -0,0 +1,116 @@
+import { isRetryableAssistantError, type AssistantMessage } from "@earendil-works/pi-ai";
+import { abortReason, throwIfAborted } from "./cancellation.ts";
+import {
+  providerUsageLimitFromMessages,
+  type WorkflowProviderUsageLimitError,
+} from "./provider-usage-limit.ts";
+
+export const AGENT_RETRY_BASE_DELAY_MS = 1_000;
+export const AGENT_RETRY_MAX_DELAY_MS = 30_000;
+export const WORKFLOW_PROVIDER_ERROR_CODE = "WORKFLOW_PROVIDER_ERROR";
+
+export interface AgentRetryScheduler {
+  sleep(delayMs: number, signal: AbortSignal | undefined): Promise<void>;
+}
+
+export interface ProviderErrorDetails {
+  readonly stopReason: "error";
+  readonly retryable: boolean;
+  readonly provider?: string;
+  readonly model?: string;
+  readonly api?: string;
+}
+
+/** Provider failure reconstructed from pi's terminal assistant-message metadata. */
+export class WorkflowProviderError extends Error {
+  override readonly name = "WorkflowProviderError";
+  readonly code = WORKFLOW_PROVIDER_ERROR_CODE;
+
+  constructor(
+    message: string,
+    readonly details: ProviderErrorDetails,
+  ) {
+    super(message);
+  }
+
+  get retryable(): boolean {
+    return this.details.retryable;
+  }
+
+  toJSON(): {
+    readonly name: string;
+    readonly message: string;
+    readonly code: string;
+    readonly details: ProviderErrorDetails;
+  } {
+    return { name: this.name, message: this.message, code: this.code, details: this.details };
+  }
+}
+
+export const defaultAgentRetryScheduler: AgentRetryScheduler = {
+  async sleep(delayMs, signal) {
+    throwIfAborted(signal);
+    await new Promise<void>((resolve, reject) => {
+      const cleanup = () => signal?.removeEventListener("abort", onAbort);
+      const timer = setTimeout(() => {
+        cleanup();
+        resolve();
+      }, delayMs);
+      const onAbort = () => {
+        clearTimeout(timer);
+        cleanup();
+        reject(abortReason(signal));
+      };
+      signal?.addEventListener("abort", onAbort, { once: true });
+      if (signal?.aborted) onAbort();
+    });
+  },
+};
+
+export function agentRetryDelayMs(retryAttempt: number): number {
+  return Math.min(AGENT_RETRY_MAX_DELAY_MS, AGENT_RETRY_BASE_DELAY_MS * 2 ** Math.max(0, retryAttempt - 1));
+}
+
+export function providerErrorFromMessages(
+  messages: readonly unknown[],
+  options: { readonly pauseOnUsageLimit?: boolean } = {},
+): WorkflowProviderError | WorkflowProviderUsageLimitError | undefined {
+  const usageLimit = providerUsageLimitFromMessages(messages);
+  if (usageLimit && options.pauseOnUsageLimit) return usageLimit;
+  const message = messages.findLast(isAssistantMessage);
+  if (!message || message.stopReason !== "error") return undefined;
+  const errorMessage = typeof message.errorMessage === "string" && message.errorMessage.length > 0
+    ? message.errorMessage
+    : "Provider session ended with an unspecified error.";
+  const retryable = isRetryableAssistantError(message as AssistantMessage);
+  const provider = stringDetail(message.provider);
+  const model = stringDetail(message.model);
+  const api = stringDetail(message.api);
+  return new WorkflowProviderError(errorMessage, {
+    stopReason: "error",
+    retryable,
+    ...(provider ? { provider } : {}),
+    ...(model ? { model } : {}),
+    ...(api ? { api } : {}),
+  });
+}
+
+function isAssistantMessage(value: unknown): value is {
+  readonly role: "assistant";
+  readonly stopReason?: unknown;
+  readonly errorMessage?: unknown;
+  readonly provider?: unknown;
+  readonly model?: unknown;
+  readonly api?: unknown;
+} {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    "role" in value &&
+    value.role === "assistant"
+  );
+}
+
+function stringDetail(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}

package/.pi/extensions/pi-workflow-engine/src/agent-runner-types.ts

@@ -1,52 +1,37 @@
 import type { Api, Model } from "@earendil-works/pi-ai";
-import type { CreateAgentSessionOptions, ModelRegistry } from "@earendil-works/pi-coding-agent";
+import type {
+  AgentSession,
+  CreateAgentSessionOptions,
+  ModelRegistry,
+} from "@earendil-works/pi-coding-agent";
 import type { WorkflowBudget } from "./budget.ts";
 import type { Semaphore } from "./concurrency.ts";
+import type { WorkflowAgentLimiter } from "./agent-limits.ts";
+import type { AgentRetryScheduler } from "./agent-retry.ts";
+import type { ResolvedWorkflowModelProfiles } from "./model-profiles.ts";
 import type { WorkflowJournal } from "./journal.ts";
 import type { PerfSink } from "./perf.ts";
-import type { AgentOptions } from "./types.ts";
+import type { AgentOptions, WorkflowProgressEvent } from "./types.ts";
 import type { WorkflowUsageSink } from "./usage.ts";
 import type { WorktreeBaseline, WorktreeRegistry } from "./worktree.ts";
-import type {
-  EffectiveAgentModelLike,
-  EffectiveToolDefinitionLike,
-} from "./agent-session-identity.ts";
-
-export interface AgentRunnerEvent {
-  readonly type: string;
-  readonly toolName?: string;
-}
-
-export interface AgentRunnerToolInfo {
-  readonly name: string;
-  readonly description?: string;
-  readonly parameters?: unknown;
-  readonly promptGuidelines?: readonly string[];
-  readonly sourceInfo?: {
-    readonly path: string;
-    readonly source: string;
-    readonly scope: string;
-    readonly origin: string;
-    readonly baseDir?: string;
-  };
-}
 
-export interface AgentRunnerSession {
-  readonly state: {
-    readonly messages: readonly unknown[];
-    readonly systemPrompt?: string;
-    readonly model?: EffectiveAgentModelLike;
-    readonly thinkingLevel?: string;
-  };
-  prompt(text: string): Promise<void>;
-  subscribe(listener: (event: AgentRunnerEvent) => void): () => void;
-  dispose(): void;
-  abort(): Promise<void>;
-  getAllTools?(): readonly AgentRunnerToolInfo[];
-  getActiveToolNames?(): readonly string[];
-  getToolDefinition?(name: string): EffectiveToolDefinitionLike | undefined;
-  setActiveToolsByName?(toolNames: readonly string[]): void;
-}
+export type AgentRunnerSession = Pick<
+  AgentSession,
+  | "messages"
+  | "systemPrompt"
+  | "model"
+  | "thinkingLevel"
+  | "prompt"
+  | "subscribe"
+  | "dispose"
+  | "abort"
+  | "getAllTools"
+  | "getActiveToolNames"
+  | "getToolDefinition"
+  | "setActiveToolsByName"
+  | "setAutoRetryEnabled"
+  | "getLastAssistantText"
+>;
 
 export type CreateAgentSession = (options: CreateAgentSessionOptions) => Promise<{ session: AgentRunnerSession }>;
 
@@ -56,15 +41,21 @@ export interface AgentProgress {
   agentTool(label: string, tool: string, id?: number): void;
   agentDone(label: string, id?: number): void;
   agentFailed(label: string, error: unknown, id?: number): void;
+  event(event: WorkflowProgressEvent): void;
   log(message: string): void;
 }
 
-/** Shared per-run context threaded into every agent() call. */
-export interface RunContext {
+interface RunContextBase {
   cwd: string;
   hostModel: Model<Api> | undefined;
-  modelRegistry: Pick<ModelRegistry, "find">;
   semaphore: Semaphore;
+  agentLimiter: WorkflowAgentLimiter;
+  agentTimeoutMs: number;
+  agentRetries: number;
+  pauseOnProviderUsageLimit?: boolean;
+  resumeEditedWorkflow?: boolean;
+  retryScheduler: AgentRetryScheduler;
+  modelProfiles: ResolvedWorkflowModelProfiles;
   progress: AgentProgress;
   signal: AbortSignal | undefined;
   perf: PerfSink;
@@ -72,9 +63,22 @@ export interface RunContext {
   budget: WorkflowBudget;
   journal: WorkflowJournal;
   worktrees: WorktreeRegistry;
-  createSession?: CreateAgentSession;
 }
 
+/** Shared per-run context threaded into every agent() call. */
+export type RunContext = RunContextBase & (
+  | {
+      /** Production sessions inherit the host's live provider and auth state. */
+      modelRegistry: ModelRegistry;
+      createSession?: undefined;
+    }
+  | {
+      /** Injected test sessions bypass Pi's resource loader and therefore do not resolve skills. */
+      modelRegistry: Pick<ModelRegistry, "find">;
+      createSession: CreateAgentSession;
+    }
+);
+
 /** Runtime-only options. The authored WorkflowApi exposes only AgentOptions. */
 export type AgentExecutionOptions = AgentOptions & {
   readonly worktreeBaseline?: WorktreeBaseline;

package/.pi/extensions/pi-workflow-engine/src/agent-runner.ts

@@ -1,10 +1,12 @@
 import { assertWorkflowBudgetAvailable } from "./budget.ts";
-import { throwIfAborted } from "./cancellation.ts";
+import { WorkflowAgentTimeoutError } from "./agent-limits.ts";
+import { abortReason, linkAbortSignal, throwIfAborted } from "./cancellation.ts";
 import { executeAgentAttempt } from "./agent-attempt.ts";
 import {
   createAgentReplayPlan,
   isReplayEnabled,
   settleAgentAttempt,
+  type AgentAttemptResult,
   type AgentReplayPlan,
 } from "./agent-replay.ts";
 import {
@@ -18,13 +20,16 @@ import type {
 } from "./agent-runner-types.ts";
 import type { AgentResumeBaseContext } from "./resume-context.ts";
 import { unknownErrorMessage } from "./unknown-error.ts";
+import {
+  agentRetryDelayMs,
+  WorkflowProviderError,
+} from "./agent-retry.ts";
+import { resolveAgentModelProfile } from "./model-profiles.ts";
 
 export type {
   AgentExecutionOptions,
   AgentProgress,
-  AgentRunnerEvent,
   AgentRunnerSession,
-  AgentRunnerToolInfo,
   CreateAgentSession,
   RunContext,
 } from "./agent-runner-types.ts";
@@ -35,7 +40,7 @@ export type { ResolvedAgentModel, ResolvedAgentModelRequest } from "./agent-sess
  * Run one subagent to completion in an isolated in-memory session.
  *
  * Session construction, replay validation, and workspace cleanup live in focused
- * modules; this boundary owns only admission, progress, and the one cache-to-live retry.
+ * modules; this boundary owns admission, progress, and bounded cache/provider retries.
  */
 export async function runAgent(
   rc: RunContext,
@@ -53,35 +58,63 @@ export async function runAgent(
 
   return await rc.perf.time("agent.total_ms", async () => {
     throwIfAborted(rc.signal);
-    const model = resolveModel(rc, opts, label);
-    const replay = createAgentReplayPlan(prompt, opts);
+    const routing = resolveAgentRouting(rc, opts, label);
+    const effectiveOpts = routing.thinkingLevel === opts.thinkingLevel
+      ? opts
+      : { ...opts, thinkingLevel: routing.thinkingLevel };
+    const replay = createAgentReplayPlan(prompt, effectiveOpts);
     if (!isReplayEnabled(replay)) assertWorkflowBudgetAvailable(rc.budget);
 
     const rowId = rc.progress.agentQueued(opts.phase, label);
+    const liveScope = createAgentLiveScope(rc, label);
     try {
       return await rc.semaphore.run(
         async () => {
-          throwIfAborted(rc.signal);
-          if (!isReplayEnabled(replay)) assertWorkflowBudgetAvailable(rc.budget);
+          const agentRc: RunContext = { ...rc, signal: liveScope.signal };
+          throwIfAborted(agentRc.signal);
+          if (!isReplayEnabled(replay)) assertWorkflowBudgetAvailable(agentRc.budget);
           rc.progress.agentStart(opts.phase, label, rowId);
           if (replay.kind === "disabled") {
             rc.progress.log(`${label}: resume disabled for this call (${replay.reason})`);
           }
 
           let attemptPlan: AgentReplayPlan = replay;
+          let providerRetries = 0;
           while (true) {
-            const outcome = await executeAgentAttempt({
-              rc,
-              prompt,
-              opts,
-              resumeBaseContext,
-              model,
-              replay: attemptPlan,
-              label,
-              rowId,
-              tags,
-            });
-            const settlement = await settleAgentAttempt({ rc, label, tags, replay: attemptPlan, outcome });
+            let outcome: AgentAttemptResult;
+            try {
+              outcome = await executeAgentAttempt({
+                rc: agentRc,
+                prompt,
+                opts: effectiveOpts,
+                resumeBaseContext,
+                model: routing.model,
+                replay: attemptPlan,
+                label,
+                rowId,
+                tags,
+                admitLiveAgent: liveScope.admit,
+              });
+            } catch (error) {
+              if (!(error instanceof WorkflowProviderError) || !error.retryable) throw error;
+              if (providerRetries >= agentRc.agentRetries) {
+                if (agentRc.agentRetries > 0) {
+                  rc.progress.log(`${label}: transient provider failure after ${providerRetries} retries; giving up`);
+                }
+                throw error;
+              }
+
+              assertWorkflowBudgetAvailable(agentRc.budget);
+              providerRetries++;
+              const delayMs = agentRetryDelayMs(providerRetries);
+              rc.progress.log(`${label}: transient provider failure; retry ${providerRetries}/${agentRc.agentRetries} in ${delayMs}ms`);
+              rc.perf.counter("agent.provider_retry", 1, tags);
+              rc.perf.observe("agent.provider_retry_delay_ms", delayMs, tags);
+              attemptPlan = { kind: "off" };
+              await agentRc.retryScheduler.sleep(delayMs, agentRc.signal);
+              continue;
+            }
+            const settlement = await settleAgentAttempt({ rc: agentRc, label, tags, replay: attemptPlan, outcome });
             if (settlement.kind === "retry-live") {
               attemptPlan = { kind: "off" };
               continue;
@@ -96,20 +129,56 @@ export async function runAgent(
         },
       );
     } catch (error) {
-      rc.progress.agentFailed(label, error, rowId);
-      rc.progress.log(`${label} failed: ${unknownErrorMessage(error)}`);
-      throw error;
+      let failure = error;
+      if (liveScope.signal.aborted) failure = abortReason(liveScope.signal);
+      if (rc.signal?.aborted) failure = abortReason(rc.signal);
+      rc.progress.agentFailed(label, failure, rowId);
+      rc.progress.log(`${label} failed: ${unknownErrorMessage(failure)}`);
+      throw failure;
+    } finally {
+      liveScope.dispose();
     }
   }, tags);
 }
 
-function resolveModel(
+function createAgentLiveScope(rc: RunContext, label: string) {
+  const controller = new AbortController();
+  const unlink = linkAbortSignal(rc.signal, controller);
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let timerStarted = false;
+
+  return {
+    signal: controller.signal,
+    admit() {
+      rc.agentLimiter.admit(controller.signal);
+      if (timerStarted) return;
+      timerStarted = true;
+      timer = setTimeout(
+        () => controller.abort(new WorkflowAgentTimeoutError(label, rc.agentTimeoutMs)),
+        rc.agentTimeoutMs,
+      );
+    },
+    dispose() {
+      if (timer) clearTimeout(timer);
+      unlink();
+    },
+  };
+}
+
+function resolveAgentRouting(
   rc: RunContext,
   opts: AgentExecutionOptions,
   label: string,
-): ResolvedAgentModel["model"] {
+): { readonly model: ResolvedAgentModel["model"]; readonly thinkingLevel: AgentExecutionOptions["thinkingLevel"] } {
   try {
-    return resolveAgentModel(opts.model, rc.modelRegistry, rc.hostModel).model;
+    return resolveAgentModelProfile(
+      {
+        request: opts,
+        profiles: rc.modelProfiles,
+        resolveExplicitModel: (modelRef) => resolveAgentModel(modelRef, rc.modelRegistry, rc.hostModel).model,
+        hostModel: rc.hostModel,
+      },
+    );
   } catch (error) {
     rc.progress.agentFailed(label, error);
     throw error;

package/.pi/extensions/pi-workflow-engine/src/agent-session-providers.ts

@@ -0,0 +1,50 @@
+import type { Api, Model } from "@earendil-works/pi-ai";
+import type { ModelRegistry, ModelRuntime } from "@earendil-works/pi-coding-agent";
+
+type HostProviderRegistry = Pick<
+  ModelRegistry,
+  | "getApiKeyForProvider"
+  | "getProviderAuthStatus"
+  | "getRegisteredProviderConfig"
+  | "getRegisteredProviderIds"
+  | "isUsingOAuth"
+>;
+
+/** Mirror the host's live provider and selected-model auth state into a child runtime. */
+export async function synchronizeWorkflowModelRuntime(input: {
+  readonly host: HostProviderRegistry;
+  readonly child: ModelRuntime;
+  readonly selectedModel: Model<Api> | undefined;
+  readonly removeChildOnlyProviders: boolean;
+}): Promise<void> {
+  const { host, child, selectedModel, removeChildOnlyProviders } = input;
+  const hostProviderIds = new Set(host.getRegisteredProviderIds());
+
+  if (removeChildOnlyProviders) {
+    for (const providerId of child.getRegisteredProviderIds()) {
+      if (!hostProviderIds.has(providerId)) child.unregisterProvider(providerId);
+    }
+  }
+
+  for (const providerId of hostProviderIds) {
+    const config = host.getRegisteredProviderConfig(providerId);
+    if (!config) continue;
+    child.unregisterProvider(providerId);
+    child.registerProvider(providerId, config);
+  }
+
+  const selectedProvider = selectedModel?.provider;
+  if (!selectedModel || !selectedProvider) return;
+
+  const hostAuth = host.getProviderAuthStatus(selectedProvider);
+  const childAuth = child.getProviderAuthStatus(selectedProvider);
+  if (hostAuth.configured && !childAuth.configured && host.isUsingOAuth(selectedModel)) {
+    throw new Error(
+      `Workflow subagents cannot inherit OAuth credentials for "${selectedProvider}" from a host-only credential store; configure OAuth in Pi's shared agent directory.`,
+    );
+  }
+  if (hostAuth.source === "runtime" || (hostAuth.configured && !childAuth.configured)) {
+    const apiKey = await host.getApiKeyForProvider(selectedProvider);
+    if (apiKey !== undefined) await child.setRuntimeApiKey(selectedProvider, apiKey);
+  }
+}