pi-workflow-engine 0.11.0 → 0.13.0

package/.pi/extensions/pi-workflow-engine/src/advisory-evidence.ts

@@ -0,0 +1,90 @@
+import { createHash } from "node:crypto";
+import { Type, type Static } from "typebox";
+import { AdvisorySeveritySchema, type IdentifiedAdvisoryCandidate, type AdvisoryCandidate, type AdvisoryReport } from "./advisory-schema.ts";
+import type { WorkflowApi } from "./types.ts";
+
+export interface AdvisoryStageCoverage {
+  stage: string;
+  expected: number;
+  completed: number;
+  failed: number;
+  failures: { branch: string; reason: string; candidate?: AdvisoryCandidate }[];
+}
+
+/** Settled results are required here: a missing branch is not a negative finding. */
+export async function collectAdvisoryStage<T>(
+  api: Pick<WorkflowApi, "parallel">,
+  stage: string,
+  branches: { id: string; candidate?: AdvisoryCandidate; run(): Promise<T> }[],
+  coverage: AdvisoryStageCoverage[],
+): Promise<T[]> {
+  const results = await api.parallel(branches.map((branch) => branch.run), { settled: true });
+  const failures = results.flatMap((result, index) => result.ok ? [] : [{ branch: branches[index]!.id, reason: result.error.message, ...(branches[index]!.candidate ? { candidate: branches[index]!.candidate } : {}) }]);
+  coverage.push({ stage, expected: branches.length, completed: results.length - failures.length, failed: failures.length, failures });
+  return results.flatMap((result) => result.ok ? [result.value] : []);
+}
+
+export function identifyCandidates(candidates: readonly AdvisoryCandidate[], lens: string): IdentifiedAdvisoryCandidate[] {
+  return candidates.map((candidate, index) => {
+    const candidateId = createHash("sha256").update(JSON.stringify([lens, index, candidate])).digest("hex").slice(0, 20);
+    // Identities are assigned by the workflow, never accepted from a finder.
+    return { ...candidate, candidateId, sourceCandidateIds: [candidateId] };
+  });
+}
+
+function normalizedSummary(summary: string): string {
+  return summary.toLowerCase().replace(/\s+/g, " ").trim().replace(/[.!?]+$/, "");
+}
+
+function candidateDedupKey(candidate: AdvisoryCandidate): string {
+  const location = candidate.locations[0];
+  return JSON.stringify([candidate.category, location?.file.replace(/^\.\//, "").replace(/^[ab]\//, ""),
+    location?.symbol ?? location?.line ?? null, normalizedSummary(candidate.summary)]);
+}
+
+/** Collapse only equivalent claims at the same anchor; keep every discovery ID and location. */
+export function dedupeCandidates<T extends IdentifiedAdvisoryCandidate>(candidates: readonly T[]): T[] {
+  const seen = new Map<string, T>();
+  for (const candidate of candidates) {
+    const key = candidateDedupKey(candidate);
+    const previous = seen.get(key);
+    if (!previous) seen.set(key, { ...candidate });
+    else {
+      previous.sourceCandidateIds = [...new Set([...previous.sourceCandidateIds, ...candidate.sourceCandidateIds])];
+      previous.locations = uniqueLocations([...previous.locations, ...candidate.locations]);
+      previous.impact = [...new Set([previous.impact, candidate.impact])].join("\n");
+      previous.discoveryEvidence = [...new Set([...(previous.discoveryEvidence ?? []), ...(candidate.discoveryEvidence ?? [])])];
+    }
+  }
+  return [...seen.values()];
+}
+
+export function uniqueLocations(locations: AdvisoryCandidate["locations"]): AdvisoryCandidate["locations"] {
+  return [...new Map(locations.map((location) => [JSON.stringify(location), location])).values()];
+}
+
+// Synthesis selects evidence records. It has no fields with which to replace their evidence.
+export const AdvisorySynthesisSchema = Type.Object({
+  summary: Type.String(),
+  findings: Type.Array(Type.Object({
+    sourceCandidateIds: Type.Array(Type.String(), { minItems: 1 }),
+    severity: AdvisorySeveritySchema,
+    recommendation: Type.String(),
+  })),
+  nextSteps: Type.Array(Type.String()),
+});
+export type AdvisorySynthesis = Static<typeof AdvisorySynthesisSchema>;
+export const SYNTHESIS_ID_INSTRUCTIONS = "Select or merge findings only by sourceCandidateIds shown below. Return severity and an advisory recommendation per selection. Never select an ID absent from these verified records. ";
+
+export function withAdvisoryCoverage<T extends AdvisoryReport>(report: T, coverage: AdvisoryStageCoverage[]) {
+  const incomplete = coverage.some((stage) => stage.failed > 0);
+  const gaps = coverage.flatMap((stage) => stage.failures.map((failure) => `${stage.stage}/${failure.branch}: ${failure.reason}`));
+  return {
+    ...report,
+    status: incomplete ? "incomplete" as const : "complete" as const,
+    summary: incomplete ? `Incomplete review: ${gaps.length} branch(es) failed. ${report.findings.length} finding(s) available; no clean conclusion is possible.` : report.summary,
+    nextSteps: incomplete ? ["Inspect failed branches and rerun missing work.", ...(report.findings.length > 0 ? report.nextSteps : [])] : report.nextSteps,
+    coverage,
+    gaps,
+  };
+}

package/.pi/extensions/pi-workflow-engine/src/advisory-schema.ts

@@ -9,7 +9,7 @@ export const AdvisoryConfidenceSchema = Type.Union([Type.Literal("low"), Type.Li
   description: "Confidence that the finding is actionable.",
 });
 
-export const AdvisoryVerifierVerdictSchema = Type.Union([Type.Literal("CONFIRMED"), Type.Literal("PLAUSIBLE"), Type.Literal("REFUTED")], {
+export const AdvisoryVerifierVerdictSchema = Type.Union([Type.Literal("CONFIRMED"), Type.Literal("PLAUSIBLE"), Type.Literal("REFUTED"), Type.Literal("NOT_SUBSTANTIATED")], {
   description: "Verifier judgment for a candidate finding.",
 });
 
@@ -20,6 +20,8 @@ export const AdvisoryLocationSchema = Type.Object({
 });
 
 export const AdvisoryCandidateSchema = Type.Object({
+  discoveryEvidence: Type.Optional(Type.Array(Type.String())),
+  reviewAnchor: Type.Optional(AdvisoryLocationSchema),
   summary: Type.String({ description: "One-line candidate finding or hypothesis." }),
   category: Type.String({ description: "Workflow-specific category such as bug, duplication, root-cause, or io." }),
   locations: Type.Array(AdvisoryLocationSchema, { description: "Relevant code or configuration locations." }),
@@ -38,6 +40,9 @@ export const AdvisoryVerdictSchema = Type.Object({
 });
 
 export const AdvisoryFindingSchema = Type.Object({
+  verdict: Type.Optional(AdvisoryVerifierVerdictSchema),
+  sourceCandidateIds: Type.Optional(Type.Array(Type.String())),
+  reviewAnchor: Type.Optional(AdvisoryLocationSchema),
   summary: Type.String({ description: "One-line final finding." }),
   category: Type.String({ description: "Workflow-specific category for grouping and display." }),
   severity: AdvisorySeveritySchema,
@@ -49,6 +54,12 @@ export const AdvisoryFindingSchema = Type.Object({
 });
 
 export const AdvisoryReportSchema = Type.Object({
+  status: Type.Optional(Type.Union([Type.Literal("complete"), Type.Literal("incomplete")])),
+  gaps: Type.Optional(Type.Array(Type.String())),
+  coverage: Type.Optional(Type.Array(Type.Object({
+    stage: Type.String(), expected: Type.Number(), completed: Type.Number(), failed: Type.Number(),
+    failures: Type.Array(Type.Object({ branch: Type.String(), reason: Type.String(), candidate: Type.Optional(AdvisoryCandidateSchema) })),
+  }))),
   summary: Type.String({ description: "Short overall advisory summary." }),
   findings: Type.Array(AdvisoryFindingSchema, { description: "Verified and ranked advisory findings." }),
   nextSteps: Type.Array(Type.String({ description: "Concrete follow-up commands, inspections, or decisions." })),
@@ -59,12 +70,9 @@ export const AdvisoryReportWithStatsSchema = Type.Object({
   stats: Type.Optional(Type.Record(Type.String(), Type.Union([Type.String(), Type.Number()]))),
 });
 
-export type AdvisorySeverity = Static<typeof AdvisorySeveritySchema>;
-export type AdvisoryConfidence = Static<typeof AdvisoryConfidenceSchema>;
-export type AdvisoryVerifierVerdict = Static<typeof AdvisoryVerifierVerdictSchema>;
 export type AdvisoryLocation = Static<typeof AdvisoryLocationSchema>;
 export type AdvisoryCandidate = Static<typeof AdvisoryCandidateSchema>;
-export type AdvisoryCandidates = Static<typeof AdvisoryCandidatesSchema>;
+export type IdentifiedAdvisoryCandidate = AdvisoryCandidate & { candidateId: string; sourceCandidateIds: string[] };
 export type AdvisoryVerdict = Static<typeof AdvisoryVerdictSchema>;
 export type AdvisoryFinding = Static<typeof AdvisoryFindingSchema>;
 export type AdvisoryReport = Static<typeof AdvisoryReportSchema>;

package/.pi/extensions/pi-workflow-engine/src/agent-attempt.ts

@@ -37,8 +37,9 @@ export async function executeAgentAttempt(input: {
   readonly label: string;
   readonly rowId: number;
   readonly tags: AgentRunTags;
+  readonly admitLiveAgent: () => void;
 }): Promise<AgentAttemptResult> {
-  const { rc, prompt, opts, resumeBaseContext, model, replay, label, rowId, tags } = input;
+  const { rc, prompt, opts, resumeBaseContext, model, replay, label, rowId, tags, admitLiveAgent } = input;
   let repositoryBefore: RepositoryResumeContext | undefined;
   let evidence: AgentReplayEvidence | undefined;
   if (isReplayEnabled(replay)) {
@@ -104,6 +105,7 @@ export async function executeAgentAttempt(input: {
     }
 
     assertWorkflowBudgetAvailable(rc.budget);
+    admitLiveAgent();
     const rawResult = await promptAgentSession({
       rc,
       handle,

package/.pi/extensions/pi-workflow-engine/src/agent-limits.ts

@@ -0,0 +1,35 @@
+import { throwIfAborted } from "./cancellation.ts";
+
+export class WorkflowAgentLimitError extends Error {
+  override readonly name = "WorkflowAgentLimitError";
+  readonly code = "WORKFLOW_AGENT_LIMIT_EXCEEDED";
+
+  constructor(readonly maxAgents: number) {
+    super(`Workflow live-agent limit of ${maxAgents} has been reached; no new model request was started.`);
+  }
+}
+
+export class WorkflowAgentTimeoutError extends Error {
+  override readonly name = "WorkflowAgentTimeoutError";
+  readonly code = "WORKFLOW_AGENT_TIMEOUT";
+
+  constructor(
+    readonly label: string,
+    readonly timeoutMs: number,
+  ) {
+    super(`Workflow agent "${label}" exceeded its ${timeoutMs}ms duration limit.`);
+  }
+}
+
+/** Shared run-level admission counter. Cache hits do not call admit(). */
+export class WorkflowAgentLimiter {
+  private admitted = 0;
+
+  constructor(readonly maxAgents: number) {}
+
+  admit(signal: AbortSignal | undefined): void {
+    throwIfAborted(signal);
+    if (this.admitted >= this.maxAgents) throw new WorkflowAgentLimitError(this.maxAgents);
+    this.admitted++;
+  }
+}

package/.pi/extensions/pi-workflow-engine/src/agent-replay.ts

@@ -2,7 +2,8 @@ import type { Skill } from "@earendil-works/pi-coding-agent";
 import { Value } from "typebox/value";
 import { captureAgentSkillIdentities, extractSkillSelectorsFromText } from "./agent-skills.ts";
 import type { AgentExecutionOptions, AgentRunnerSession, AgentRunTags, RunContext } from "./agent-runner-types.ts";
-import { captureEffectiveSession, FINAL_TOOL } from "./agent-session.ts";
+import { captureEffectiveAgentSessionIdentity } from "./agent-session-identity.ts";
+import { FINAL_TOOL } from "./agent-session.ts";
 import type { AgentWorkspace, IsolatedAgentWorkspace } from "./agent-workspace.ts";
 import { captureAgentJournalKey } from "./journal.ts";
 import { unknownErrorMessage } from "./unknown-error.ts";
@@ -116,7 +117,9 @@ export async function lookupReplayResult(input: {
   readonly opts: AgentExecutionOptions;
   readonly workspace: AgentWorkspace;
 }): Promise<{ readonly hit: true; readonly result: unknown } | { readonly hit: false; readonly reason?: string }> {
-  const cached = input.rc.journal.lookup(input.key, input.identity);
+  const cached = input.rc.journal.lookup(input.key, input.identity, {
+    allowWorkflowSourceMismatch: input.rc.resumeEditedWorkflow,
+  });
   if (!cached.hit) return cached;
   const validation = await validateCachedResult(cached.value, input.opts, input.rc, input.workspace);
   return validation.ok
@@ -147,7 +150,9 @@ export async function validateReplayIdentity(input: {
     sessionCwd: input.sessionCwd,
   });
   if (current.kind === "unverifiable") return { ok: false, reason: current.reason };
-  const mismatch = resumeContextMismatchReason(input.identity, current.identity);
+  const mismatch = resumeContextMismatchReason(input.identity, current.identity, {
+    allowWorkflowSourceMismatch: input.rc.resumeEditedWorkflow,
+  });
   return mismatch ? { ok: false, reason: mismatch } : { ok: true };
 }
 
@@ -160,6 +165,7 @@ export async function settleAgentAttempt(input: {
 }): Promise<AgentAttemptSettlement> {
   const { rc, label, replay, outcome } = input;
   if (outcome.kind === "live-unrecordable") {
+    recordAgentResultSource(rc, "live");
     return { kind: "done", result: outcome.result };
   }
   if (!isReplayEnabled(replay)) throw new Error("Replayable agent outcome produced without an enabled replay plan.");
@@ -183,10 +189,23 @@ export async function settleAgentAttempt(input: {
   if (outcome.kind === "cache-hit") {
     rc.progress.log(`${label}: using cached result from workflow journal`);
     rc.perf.counter("agent.cache_hit", 1, input.tags);
+    recordAgentResultSource(rc, "cached");
+  } else {
+    recordAgentResultSource(rc, "live");
   }
   return { kind: "done", result: outcome.result };
 }
 
+function recordAgentResultSource(rc: RunContext, source: "cached" | "live"): void {
+  if (!rc.resumeEditedWorkflow) return;
+  rc.progress.event({
+    type: "counter_delta",
+    key: `resume.${source}`,
+    label: source === "cached" ? "Cached agents" : "Live agents",
+    delta: 1,
+  });
+}
+
 async function captureVerifiedReplayIdentity(input: {
   readonly rc: RunContext;
   readonly repository: VerifiedRepositoryResumeContext;
@@ -201,7 +220,11 @@ async function captureVerifiedReplayIdentity(input: {
       workspaceRoot: input.rc.cwd,
       signal: input.rc.signal,
     }),
-    captureEffectiveSession(input.session, input.sessionCwd, input.rc.cwd, input.rc.signal),
+    captureEffectiveAgentSessionIdentity(input.session, {
+      sessionCwd: input.sessionCwd,
+      workspaceRoot: input.rc.cwd,
+      signal: input.rc.signal,
+    }),
   ]);
   if (skills.kind === "unverifiable") return skills;
   if (effectiveSession.kind === "unverifiable") return effectiveSession;

package/.pi/extensions/pi-workflow-engine/src/agent-retry.ts

@@ -0,0 +1,116 @@
+import { isRetryableAssistantError, type AssistantMessage } from "@earendil-works/pi-ai";
+import { abortReason, throwIfAborted } from "./cancellation.ts";
+import {
+  providerUsageLimitFromMessages,
+  type WorkflowProviderUsageLimitError,
+} from "./provider-usage-limit.ts";
+
+export const AGENT_RETRY_BASE_DELAY_MS = 1_000;
+export const AGENT_RETRY_MAX_DELAY_MS = 30_000;
+export const WORKFLOW_PROVIDER_ERROR_CODE = "WORKFLOW_PROVIDER_ERROR";
+
+export interface AgentRetryScheduler {
+  sleep(delayMs: number, signal: AbortSignal | undefined): Promise<void>;
+}
+
+export interface ProviderErrorDetails {
+  readonly stopReason: "error";
+  readonly retryable: boolean;
+  readonly provider?: string;
+  readonly model?: string;
+  readonly api?: string;
+}
+
+/** Provider failure reconstructed from pi's terminal assistant-message metadata. */
+export class WorkflowProviderError extends Error {
+  override readonly name = "WorkflowProviderError";
+  readonly code = WORKFLOW_PROVIDER_ERROR_CODE;
+
+  constructor(
+    message: string,
+    readonly details: ProviderErrorDetails,
+  ) {
+    super(message);
+  }
+
+  get retryable(): boolean {
+    return this.details.retryable;
+  }
+
+  toJSON(): {
+    readonly name: string;
+    readonly message: string;
+    readonly code: string;
+    readonly details: ProviderErrorDetails;
+  } {
+    return { name: this.name, message: this.message, code: this.code, details: this.details };
+  }
+}
+
+export const defaultAgentRetryScheduler: AgentRetryScheduler = {
+  async sleep(delayMs, signal) {
+    throwIfAborted(signal);
+    await new Promise<void>((resolve, reject) => {
+      const cleanup = () => signal?.removeEventListener("abort", onAbort);
+      const timer = setTimeout(() => {
+        cleanup();
+        resolve();
+      }, delayMs);
+      const onAbort = () => {
+        clearTimeout(timer);
+        cleanup();
+        reject(abortReason(signal));
+      };
+      signal?.addEventListener("abort", onAbort, { once: true });
+      if (signal?.aborted) onAbort();
+    });
+  },
+};
+
+export function agentRetryDelayMs(retryAttempt: number): number {
+  return Math.min(AGENT_RETRY_MAX_DELAY_MS, AGENT_RETRY_BASE_DELAY_MS * 2 ** Math.max(0, retryAttempt - 1));
+}
+
+export function providerErrorFromMessages(
+  messages: readonly unknown[],
+  options: { readonly pauseOnUsageLimit?: boolean } = {},
+): WorkflowProviderError | WorkflowProviderUsageLimitError | undefined {
+  const usageLimit = providerUsageLimitFromMessages(messages);
+  if (usageLimit && options.pauseOnUsageLimit) return usageLimit;
+  const message = messages.findLast(isAssistantMessage);
+  if (!message || message.stopReason !== "error") return undefined;
+  const errorMessage = typeof message.errorMessage === "string" && message.errorMessage.length > 0
+    ? message.errorMessage
+    : "Provider session ended with an unspecified error.";
+  const retryable = isRetryableAssistantError(message as AssistantMessage);
+  const provider = stringDetail(message.provider);
+  const model = stringDetail(message.model);
+  const api = stringDetail(message.api);
+  return new WorkflowProviderError(errorMessage, {
+    stopReason: "error",
+    retryable,
+    ...(provider ? { provider } : {}),
+    ...(model ? { model } : {}),
+    ...(api ? { api } : {}),
+  });
+}
+
+function isAssistantMessage(value: unknown): value is {
+  readonly role: "assistant";
+  readonly stopReason?: unknown;
+  readonly errorMessage?: unknown;
+  readonly provider?: unknown;
+  readonly model?: unknown;
+  readonly api?: unknown;
+} {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    "role" in value &&
+    value.role === "assistant"
+  );
+}
+
+function stringDetail(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}

package/.pi/extensions/pi-workflow-engine/src/agent-runner-types.ts

@@ -1,52 +1,37 @@
 import type { Api, Model } from "@earendil-works/pi-ai";
-import type { CreateAgentSessionOptions, ModelRegistry } from "@earendil-works/pi-coding-agent";
+import type {
+  AgentSession,
+  CreateAgentSessionOptions,
+  ModelRegistry,
+} from "@earendil-works/pi-coding-agent";
 import type { WorkflowBudget } from "./budget.ts";
 import type { Semaphore } from "./concurrency.ts";
+import type { WorkflowAgentLimiter } from "./agent-limits.ts";
+import type { AgentRetryScheduler } from "./agent-retry.ts";
+import type { ResolvedWorkflowModelProfiles } from "./model-profiles.ts";
 import type { WorkflowJournal } from "./journal.ts";
 import type { PerfSink } from "./perf.ts";
-import type { AgentOptions } from "./types.ts";
+import type { AgentOptions, WorkflowProgressEvent } from "./types.ts";
 import type { WorkflowUsageSink } from "./usage.ts";
 import type { WorktreeBaseline, WorktreeRegistry } from "./worktree.ts";
-import type {
-  EffectiveAgentModelLike,
-  EffectiveToolDefinitionLike,
-} from "./agent-session-identity.ts";
-
-export interface AgentRunnerEvent {
-  readonly type: string;
-  readonly toolName?: string;
-}
-
-export interface AgentRunnerToolInfo {
-  readonly name: string;
-  readonly description?: string;
-  readonly parameters?: unknown;
-  readonly promptGuidelines?: readonly string[];
-  readonly sourceInfo?: {
-    readonly path: string;
-    readonly source: string;
-    readonly scope: string;
-    readonly origin: string;
-    readonly baseDir?: string;
-  };
-}
 
-export interface AgentRunnerSession {
-  readonly state: {
-    readonly messages: readonly unknown[];
-    readonly systemPrompt?: string;
-    readonly model?: EffectiveAgentModelLike;
-    readonly thinkingLevel?: string;
-  };
-  prompt(text: string): Promise<void>;
-  subscribe(listener: (event: AgentRunnerEvent) => void): () => void;
-  dispose(): void;
-  abort(): Promise<void>;
-  getAllTools?(): readonly AgentRunnerToolInfo[];
-  getActiveToolNames?(): readonly string[];
-  getToolDefinition?(name: string): EffectiveToolDefinitionLike | undefined;
-  setActiveToolsByName?(toolNames: readonly string[]): void;
-}
+export type AgentRunnerSession = Pick<
+  AgentSession,
+  | "messages"
+  | "systemPrompt"
+  | "model"
+  | "thinkingLevel"
+  | "prompt"
+  | "subscribe"
+  | "dispose"
+  | "abort"
+  | "getAllTools"
+  | "getActiveToolNames"
+  | "getToolDefinition"
+  | "setActiveToolsByName"
+  | "setAutoRetryEnabled"
+  | "getLastAssistantText"
+>;
 
 export type CreateAgentSession = (options: CreateAgentSessionOptions) => Promise<{ session: AgentRunnerSession }>;
 
@@ -56,15 +41,21 @@ export interface AgentProgress {
   agentTool(label: string, tool: string, id?: number): void;
   agentDone(label: string, id?: number): void;
   agentFailed(label: string, error: unknown, id?: number): void;
+  event(event: WorkflowProgressEvent): void;
   log(message: string): void;
 }
 
-/** Shared per-run context threaded into every agent() call. */
-export interface RunContext {
+interface RunContextBase {
   cwd: string;
   hostModel: Model<Api> | undefined;
-  modelRegistry: Pick<ModelRegistry, "find">;
   semaphore: Semaphore;
+  agentLimiter: WorkflowAgentLimiter;
+  agentTimeoutMs: number;
+  agentRetries: number;
+  pauseOnProviderUsageLimit?: boolean;
+  resumeEditedWorkflow?: boolean;
+  retryScheduler: AgentRetryScheduler;
+  modelProfiles: ResolvedWorkflowModelProfiles;
   progress: AgentProgress;
   signal: AbortSignal | undefined;
   perf: PerfSink;
@@ -72,9 +63,22 @@ export interface RunContext {
   budget: WorkflowBudget;
   journal: WorkflowJournal;
   worktrees: WorktreeRegistry;
-  createSession?: CreateAgentSession;
 }
 
+/** Shared per-run context threaded into every agent() call. */
+export type RunContext = RunContextBase & (
+  | {
+      /** Production sessions inherit the host's live provider and auth state. */
+      modelRegistry: ModelRegistry;
+      createSession?: undefined;
+    }
+  | {
+      /** Injected test sessions bypass Pi's resource loader and therefore do not resolve skills. */
+      modelRegistry: Pick<ModelRegistry, "find">;
+      createSession: CreateAgentSession;
+    }
+);
+
 /** Runtime-only options. The authored WorkflowApi exposes only AgentOptions. */
 export type AgentExecutionOptions = AgentOptions & {
   readonly worktreeBaseline?: WorktreeBaseline;