pi-workflow-engine 0.10.3 → 0.12.0

package/.pi/extensions/pi-workflow-engine/src/engine.ts

@@ -1,19 +1,41 @@
 import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
-import { bindPipeline, parallel, Semaphore } from "./concurrency.ts";
-import { linkAbortSignal } from "./cancellation.ts";
+import type { Static, TSchema } from "typebox";
+import { bindParallel, bindPipeline, Semaphore } from "./concurrency.ts";
+import { WorkflowAgentLimiter } from "./agent-limits.ts";
+import { defaultAgentRetryScheduler, type AgentRetryScheduler } from "./agent-retry.ts";
+import { resolveWorkflowModelProfiles, type ResolvedWorkflowModelProfiles } from "./model-profiles.ts";
+import { abortReason, isWorkflowPauseError, linkAbortSignal, throwIfAborted } from "./cancellation.ts";
 import { createBudget } from "./budget.ts";
-import { runAgent, type RunContext } from "./agent-runner.ts";
+import { runAgent, type AgentExecutionOptions, type RunContext } from "./agent-runner.ts";
 import { ProgressTracker } from "./progress.ts";
-import { createPerfRecorder, type PerfSnapshot } from "./perf.ts";
-import { createWorkflowUsageRecorder } from "./usage.ts";
-import { defaultConcurrency, resolveWorkflowRunOptions } from "./options.ts";
-import type { AgentOptions, WorkflowApi, WorkflowModule, WorkflowProgressEvent, WorkflowRef, WorkflowRunOptions } from "./types.ts";
+import { createPerfRecorder, type PerfSink, type PerfSnapshot } from "./perf.ts";
+import { createWorkflowUsageRecorder, type WorkflowUsageSink } from "./usage.ts";
+import {
+  resolveWorkflowRunOptions,
+  type ResolvedWorkflowRunOptions,
+} from "./options.ts";
+import type { AgentOptions, IsolatedAgentResult, LoadedWorkflow, WorkflowApi, WorkflowProgressEvent, WorkflowRef, WorkflowRunOptions } from "./types.ts";
 import { WorkflowInspector } from "./ui/workflow-inspector.ts";
+import { WORKFLOW_VIEWER_OVERLAY_OPTIONS } from "./ui/workflow-viewer-layout.ts";
 import { createWorkflowJournal, createWorkflowRunId, pruneWorkflowJournals, workflowJournalPath } from "./journal.ts";
 import { WorktreeRegistry } from "./worktree.ts";
-
-/** Default global cap on concurrent agents per run. */
-const DEFAULT_CONCURRENCY = defaultConcurrency();
+import { runFinalizers } from "./finalizers.ts";
+import { unknownErrorMessage } from "./unknown-error.ts";
+import {
+  captureWorkflowResumeContext,
+  type AgentResumeBaseContext,
+} from "./resume-context.ts";
+import {
+  DurableWorkflowRun,
+  ProjectWorkflowRunStore,
+  workflowRunRecordPath,
+  type WorkflowRunStore,
+} from "./workflow-run-store.ts";
+import { createWorkflowRunRecord } from "./workflow-run-record.ts";
+import {
+  createProviderUsageLimitPauseRecord,
+  WorkflowProviderUsageLimitError,
+} from "./provider-usage-limit.ts";
 
 /** The workflow-facing slice of the progress tracker (satisfied by `ProgressTracker`). */
 export interface WorkflowProgress {
@@ -22,6 +44,9 @@ export interface WorkflowProgress {
   event(event: WorkflowProgressEvent): void;
 }
 
+/** Engine state shared by the workflow and its individual agent calls. */
+export type WorkflowRunContext = RunContext;
+
 /** Per-context knobs threaded through `runWorkflowWithContext` (and re-derived for each sub-workflow). */
 export interface WorkflowContextOptions {
   /** Controller whose `.signal` equals `rc.signal`; aborts in-flight `parallel` siblings on failure. */
@@ -29,7 +54,7 @@ export interface WorkflowContextOptions {
   /** Eager-submission cap for `parallel`. */
   submissionLimit: number;
   /** Resolve a sub-workflow reference for `api.workflow()`. Omit to disable composition. */
-  resolveWorkflow?: (ref: WorkflowRef) => Promise<WorkflowModule>;
+  resolveWorkflow?: (ref: WorkflowRef) => Promise<LoadedWorkflow>;
   /** Nesting depth. Sub-workflows run at depth >= 1, where `api.workflow()` rejects. */
   depth?: number;
   /** Prefix applied to phase titles so sub-workflow phases nest as "<name> ▸ <title>". */
@@ -38,59 +63,126 @@ export interface WorkflowContextOptions {
   progressNamespace?: string;
 }
 
+type Outcome<T> =
+  | { readonly ok: true; readonly value: T }
+  | { readonly ok: false; readonly error: unknown };
+
+export interface WorkflowEngineDependencies {
+  readonly worktrees?: WorktreeRegistry;
+  readonly retryScheduler?: AgentRetryScheduler;
+  readonly modelProfiles?: ResolvedWorkflowModelProfiles;
+}
+
 /**
  * Run a workflow module: build the per-run primitives (binding agent/parallel/pipeline
  * to a shared semaphore + progress tracker), invoke the workflow, return its result.
  */
 export async function runWorkflow(
   ctx: ExtensionContext,
-  mod: WorkflowModule,
+  mod: LoadedWorkflow,
   args: string,
   options: WorkflowRunOptions = {},
+  dependencies: WorkflowEngineDependencies = {},
 ): Promise<unknown> {
-  const resolvedOptions = resolveWorkflowRunOptions(options);
-  const progress = new ProgressTracker(ctx, mod.meta.name);
-  const progressSource = { snapshot: () => progress.snapshot() };
-  resolvedOptions.onProgressSource?.(progressSource);
-  if (resolvedOptions.inspect && ctx.hasUI) {
-    void ctx.ui
-      .custom<void>(
-        (tui, theme, _keybindings, done) => new WorkflowInspector(() => progress.snapshot(), tui, theme, () => done(undefined)),
-        { overlay: true, overlayOptions: { anchor: "right-center", width: "60%", maxHeight: "80%", margin: 1 } },
-      )
-      .catch((error: unknown) => progress.log(`inspector failed: ${error instanceof Error ? error.message : String(error)}`));
-  }
+  return await runResolvedWorkflow(ctx, mod, args, resolveWorkflowRunOptions(options), dependencies);
+}
 
-  const perf = resolvedOptions.perfRecorder ?? createPerfRecorder(resolvedOptions.perf);
-  const usage = createWorkflowUsageRecorder();
-  const budget = createBudget(resolvedOptions.budget ?? null, usage);
+/** Execute an options-normalized workflow without re-reading environment defaults. */
+export async function runResolvedWorkflow(
+  ctx: ExtensionContext,
+  mod: LoadedWorkflow,
+  args: string,
+  resolvedOptions: ResolvedWorkflowRunOptions,
+  dependencies: WorkflowEngineDependencies = {},
+): Promise<unknown> {
   const runId = resolvedOptions.runId ?? createWorkflowRunId();
+  let durableProgress: DurableWorkflowRun | undefined;
+  const progress = new ProgressTracker(ctx, mod.meta.name, runId, (snapshot) => durableProgress?.updateProgress(snapshot));
+  const progressSource = { snapshot: () => progress.snapshot() };
+  const perf = resolvedOptions.perfRecorder ?? createPerfRecorder(resolvedOptions.perf);
+  const usage = createWorkflowUsageRecorder((snapshot) => progress.updateUsage(snapshot));
+  const budget = createBudget(resolvedOptions.budget, usage);
   const journalPath = workflowJournalPath(ctx.cwd, runId);
   const resumePath = resolvedOptions.resumeFromRunId ? workflowJournalPath(ctx.cwd, resolvedOptions.resumeFromRunId) : undefined;
-  const journal = await createWorkflowJournal({ resumePath, writePath: journalPath });
-  const worktrees = new WorktreeRegistry(ctx.cwd);
-  resolvedOptions.onRunMetadata?.({ runId, resumedFromRunId: resolvedOptions.resumeFromRunId, journalPath });
-  progress.log(resolvedOptions.resumeFromRunId ? `run id: ${runId} (resuming from ${resolvedOptions.resumeFromRunId})` : `run id: ${runId}`);
+  const runStore = new ProjectWorkflowRunStore(ctx.cwd);
+  let persistenceFailureReported = false;
+  const durableRun = new DurableWorkflowRun(
+    runStore,
+    createWorkflowRunRecord({
+      runId,
+      workflow: mod,
+      options: resolvedOptions,
+      progress: progress.snapshot(),
+      argumentsPresent: args.length > 0,
+    }),
+    (error) => {
+      if (persistenceFailureReported) return;
+      persistenceFailureReported = true;
+      process.stderr.write(`[${mod.meta.name}] run persistence failed: ${unknownErrorMessage(error)}\n`);
+    },
+  );
+  durableProgress = durableRun;
+  const worktrees = dependencies.worktrees ?? new WorktreeRegistry(ctx.cwd);
   const runAbortController = new AbortController();
   const unlinkContextAbortSignal = linkAbortSignal(ctx.signal, runAbortController);
   const unlinkOptionAbortSignal = linkAbortSignal(resolvedOptions.signal, runAbortController);
-  const rc: RunContext = {
-    cwd: ctx.cwd,
-    hostModel: ctx.model,
-    modelRegistry: ctx.modelRegistry,
-    semaphore: new Semaphore(resolvedOptions.concurrency ?? DEFAULT_CONCURRENCY),
-    progress,
-    signal: runAbortController.signal,
-    perf,
-    usage,
-    budget,
-    journal,
-    worktrees,
-  };
+  const workflowOutcome = await captureOutcome(async () => {
+    await notifyLifecycleObserver(progress, "progress source callback", () => resolvedOptions.onProgressSource?.(progressSource));
+    if (resolvedOptions.inspect && ctx.hasUI && ctx.mode === "tui") {
+      void ctx.ui
+        .custom<void>(
+          (tui, theme, _keybindings, done) => new WorkflowInspector(() => progress.snapshot(), tui, theme, () => done(undefined)),
+          WORKFLOW_VIEWER_OVERLAY_OPTIONS,
+        )
+        .catch((error: unknown) => {
+          try {
+            progress.log(`inspector failed: ${unknownErrorMessage(error)}`);
+          } catch {
+            // Inspector reporting is detached and must never become another rejection.
+          }
+        });
+    }
+
+    const journal = await createWorkflowJournal({ resumePath, writePath: journalPath });
+    durableRun.transition({ state: "running", progress: progress.snapshot() });
+    await durableRun.flush().catch(() => undefined);
+    await notifyLifecycleObserver(progress, "run metadata callback", () =>
+      resolvedOptions.onRunMetadata?.({
+        runId,
+        resumedFromRunId: resolvedOptions.resumeFromRunId,
+        journalPath,
+        recordPath: workflowRunRecordPath(ctx.cwd, runId),
+      }),
+    );
+    progress.log(resolvedOptions.resumeFromRunId ? `run id: ${runId} (resuming from ${resolvedOptions.resumeFromRunId})` : `run id: ${runId}`);
+    const modelProfiles = dependencies.modelProfiles ?? resolveWorkflowModelProfiles({
+      cwd: ctx.cwd,
+      modelRegistry: ctx.modelRegistry,
+      hostModel: ctx.model,
+    });
+    const rc: WorkflowRunContext = {
+      cwd: ctx.cwd,
+      hostModel: ctx.model,
+      modelRegistry: ctx.modelRegistry,
+      semaphore: new Semaphore(resolvedOptions.concurrency),
+      agentLimiter: new WorkflowAgentLimiter(resolvedOptions.maxAgents),
+      agentTimeoutMs: resolvedOptions.agentTimeoutMs,
+      agentRetries: resolvedOptions.agentRetries,
+      pauseOnProviderUsageLimit: resolvedOptions.background !== undefined,
+      resumeEditedWorkflow: resolvedOptions.resumeEditedWorkflow,
+      retryScheduler: dependencies.retryScheduler ?? defaultAgentRetryScheduler,
+      modelProfiles,
+      progress,
+      signal: runAbortController.signal,
+      perf,
+      usage,
+      budget,
+      journal,
+      worktrees,
+    };
 
-  try {
     // perf.total_ms wraps the whole tree: sub-workflows run inside this span via api.workflow().
-    return await perf.time("workflow.total_ms", () =>
+    return perf.time("workflow.total_ms", () =>
       runWorkflowWithContext(rc, progress, mod, args, {
         abortController: runAbortController,
         submissionLimit: resolvedOptions.parallelSubmissionLimit ?? resolvedOptions.concurrency * 2,
@@ -99,29 +191,198 @@ export async function runWorkflow(
         progressPrefix: "",
       }),
     );
-  } finally {
-    try {
-      resolvedOptions.onUsageSnapshot?.(usage.snapshot());
-      const snapshot = perf.snapshot();
-      if (resolvedOptions.perf) {
-        resolvedOptions.onPerfSnapshot?.(snapshot);
-        progress.log(formatPerfSummary(snapshot));
-      }
-      progress.done();
-      resolvedOptions.onProgressSnapshot?.(progress.snapshot());
-    } finally {
-      resolvedOptions.onProgressSource?.(undefined);
-      unlinkContextAbortSignal();
-      unlinkOptionAbortSignal();
-      const cleanupResults = await worktrees.removeAll();
-      for (const result of cleanupResults) {
-        if (!result.ok) {
-          progress.log(`failed to remove isolated worktree ${result.path} (${result.error ?? (result.stderr.trim() || "unknown error")})`);
+  });
+
+  const finalizationOutcome = await captureOutcome(() =>
+    finalizeWorkflowRun({
+      cwd: ctx.cwd,
+      options: resolvedOptions,
+      perf,
+      usage,
+      progress,
+      runStore,
+      worktrees,
+      unlinkSignals: [unlinkContextAbortSignal, unlinkOptionAbortSignal],
+    }),
+  );
+
+  if (workflowOutcome.ok && finalizationOutcome.ok) {
+    durableRun.transition({
+      state: "completed",
+      progress: progress.snapshot(),
+      usage: usage.snapshot(),
+      result: workflowOutcome.value,
+    });
+  } else if (!workflowOutcome.ok) {
+    const error = finalizationOutcome.ok
+      ? workflowOutcome.error
+      : combinedWorkflowError(workflowOutcome.error, finalizationOutcome.error);
+    persistTerminalWorkflowError(error);
+  } else if (!finalizationOutcome.ok) {
+    persistTerminalWorkflowError(finalizationOutcome.error);
+  }
+  await durableRun.flush().catch(() => undefined);
+
+  function persistTerminalWorkflowError(error: unknown): void {
+    const pauseError = backgroundPauseError(error, ctx.signal, resolvedOptions.signal);
+    if (pauseError) {
+      if (pauseError instanceof WorkflowProviderUsageLimitError) {
+        if (resolvedOptions.background === undefined) {
+          durableRun.transition({
+            state: "failed",
+            progress: progress.snapshot(),
+            usage: usage.snapshot(),
+            error: pauseError,
+          });
+          return;
         }
+        const providerPause = createProviderUsageLimitPauseRecord(
+          pauseError,
+          resolvedOptions,
+          mod.source.kind === "file" && args.length === 0,
+        );
+        durableRun.transition({
+          state: "paused",
+          progress: progress.snapshot(),
+          ...providerPause,
+        });
+        return;
       }
-      await pruneWorkflowJournals(ctx.cwd);
+      durableRun.transition({
+        state: "paused",
+        progress: progress.snapshot(),
+        message: unknownErrorMessage(pauseError),
+      });
+      return;
     }
+    durableRun.transition({
+      state: ctx.signal?.aborted || resolvedOptions.signal?.aborted ? "stopped" : "failed",
+      progress: progress.snapshot(),
+      usage: usage.snapshot(),
+      error,
+    });
   }
+
+  if (workflowOutcome.ok) {
+    if (finalizationOutcome.ok) return workflowOutcome.value;
+    throw finalizationOutcome.error;
+  }
+  if (finalizationOutcome.ok) throw workflowOutcome.error;
+  throw combinedWorkflowError(workflowOutcome.error, finalizationOutcome.error);
+}
+
+function backgroundPauseError(error: unknown, ...signals: Array<AbortSignal | undefined>): unknown {
+  if (isWorkflowPauseError(error)) return error;
+  for (const signal of signals) {
+    if (!signal?.aborted) continue;
+    const reason = abortReason(signal);
+    if (isWorkflowPauseError(reason)) return reason;
+  }
+  return undefined;
+}
+
+async function captureOutcome<T>(operation: () => Promise<T>): Promise<Outcome<T>> {
+  try {
+    return { ok: true, value: await operation() };
+  } catch (error) {
+    return { ok: false, error };
+  }
+}
+
+interface WorkflowFinalizationInput {
+  readonly cwd: string;
+  readonly options: ResolvedWorkflowRunOptions;
+  readonly perf: PerfSink;
+  readonly usage: WorkflowUsageSink;
+  readonly progress: ProgressTracker;
+  readonly runStore: WorkflowRunStore;
+  readonly worktrees: WorktreeRegistry;
+  readonly unlinkSignals: readonly (() => void)[];
+}
+
+async function finalizeWorkflowRun(input: WorkflowFinalizationInput): Promise<void> {
+  await runFinalizers(
+    [
+      {
+        name: "usage snapshot callback",
+        criticality: "best-effort",
+        run: () => input.options.onUsageSnapshot?.(input.usage.snapshot()),
+      },
+      {
+        name: "performance snapshot callback",
+        criticality: "best-effort",
+        run: async () => {
+          if (!input.options.perf) return;
+          const snapshot = input.perf.snapshot();
+          await input.options.onPerfSnapshot?.(snapshot);
+          input.progress.log(formatPerfSummary(snapshot));
+        },
+      },
+      ...input.unlinkSignals.map((run, index) => ({
+        name: `abort signal unlink ${index + 1}`,
+        criticality: "required" as const,
+        run,
+      })),
+      {
+        name: "worktree cleanup",
+        criticality: "required",
+        run: () => input.worktrees.removeAll(),
+      },
+      { name: "journal pruning", criticality: "best-effort", run: () => pruneWorkflowJournals(input.cwd) },
+      { name: "run record pruning", criticality: "best-effort", run: () => input.runStore.prune() },
+      { name: "progress completion", criticality: "best-effort", run: () => input.progress.done() },
+      {
+        name: "progress snapshot callback",
+        criticality: "best-effort",
+        run: () => input.options.onProgressSnapshot?.(input.progress.snapshot()),
+      },
+      {
+        name: "progress source clear",
+        criticality: "best-effort",
+        run: () => input.options.onProgressSource?.(undefined),
+      },
+    ],
+    {
+      onBestEffortFailure: (failure) => {
+        try {
+          input.progress.log(`${failure.name} failed: ${unknownErrorMessage(failure.error)}`);
+        } catch {
+          // Observer failures must never replace the workflow outcome.
+        }
+      },
+    },
+  );
+}
+
+function combinedWorkflowError(workflowError: unknown, finalizationError: unknown): AggregateError {
+  return new AggregateError(
+    [workflowError, finalizationError],
+    `Workflow failed: ${unknownErrorMessage(workflowError)}; finalization also failed: ${unknownErrorMessage(finalizationError)}`,
+  );
+}
+
+function createWorkflowAgent(
+  rc: WorkflowRunContext,
+  scope: WorkflowScope,
+  mod: LoadedWorkflow,
+  resumeContext: AgentResumeBaseContext,
+): WorkflowApi["agent"] {
+  function agent<S extends TSchema>(
+    prompt: string,
+    opts: AgentOptions<S> & { schema: S; isolation: "worktree" },
+  ): Promise<IsolatedAgentResult<Static<S>>>;
+  function agent(prompt: string, opts: AgentOptions & { isolation: "worktree" }): Promise<IsolatedAgentResult<string>>;
+  function agent<S extends TSchema>(prompt: string, opts: AgentOptions<S> & { schema: S }): Promise<Static<S>>;
+  function agent(prompt: string, opts?: AgentOptions): Promise<string>;
+  function agent(prompt: string, agentOpts?: AgentOptions): Promise<unknown> {
+    const scopedOptions = scope.agentOptions(agentOpts);
+    const executionOptions: AgentExecutionOptions =
+      scopedOptions.isolation === "worktree" && mod.isolatedWorktreeBaseline !== undefined
+        ? { ...scopedOptions, worktreeBaseline: mod.isolatedWorktreeBaseline }
+        : scopedOptions;
+    return runAgent(rc, prompt, executionOptions, resumeContext);
+  }
+  return agent;
 }
 
 /**
@@ -130,16 +391,19 @@ export async function runWorkflow(
  * abort signal, and perf sink. No setup/teardown of its own — that belongs to `runWorkflow`.
  */
 export async function runWorkflowWithContext(
-  rc: RunContext,
+  rc: WorkflowRunContext,
   progress: WorkflowProgress,
-  mod: WorkflowModule,
+  mod: LoadedWorkflow,
   args: string,
   opts: WorkflowContextOptions,
 ): Promise<unknown> {
   const depth = opts.depth ?? 0;
   const scope = createWorkflowScope(progress, opts.progressPrefix ?? "", opts.progressNamespace);
+  const resumeContext: AgentResumeBaseContext = {
+    workflow: await captureWorkflowResumeContext(mod, rc.signal),
+  };
 
-  const agent = ((prompt: string, agentOpts?: AgentOptions) => runAgent(rc, prompt, scope.agentOptions(agentOpts))) as WorkflowApi["agent"];
+  const agent = createWorkflowAgent(rc, scope, mod, resumeContext);
 
   const workflow: WorkflowApi["workflow"] =
     depth >= 1
@@ -171,12 +435,11 @@ export async function runWorkflowWithContext(
   const api: WorkflowApi = {
     agent,
     workflow,
-    parallel: (thunks) =>
-      parallel(thunks, {
-        signal: rc.signal,
-        abortController: opts.abortController,
-        limit: opts.submissionLimit,
-      }),
+    parallel: bindParallel({
+      signal: rc.signal,
+      abortController: opts.abortController,
+      limit: opts.submissionLimit,
+    }),
     pipeline: bindPipeline({ signal: rc.signal, abortController: opts.abortController }),
     phase: scope.phase,
     log: scope.log,
@@ -187,7 +450,10 @@ export async function runWorkflowWithContext(
     signal: rc.signal,
   };
 
-  return await mod.default(api);
+  throwIfAborted(rc.signal);
+  const result = await mod.default(api);
+  throwIfAborted(rc.signal);
+  return result;
 }
 
 interface WorkflowScope {
@@ -242,3 +508,19 @@ function formatPerfSummary(snapshot: PerfSnapshot): string {
     .map((aggregate) => `${aggregate.name} ${Math.round(aggregate.total)}ms`);
   return parts.length > 0 ? `perf: ${parts.join(", ")}` : "perf: no samples";
 }
+
+async function notifyLifecycleObserver(
+  progress: ProgressTracker,
+  name: string,
+  notify: () => void | Promise<void>,
+): Promise<void> {
+  try {
+    await notify();
+  } catch (error) {
+    try {
+      progress.log(`${name} failed: ${unknownErrorMessage(error)}`);
+    } catch {
+      // Lifecycle observers are advisory and must not replace the workflow outcome.
+    }
+  }
+}

package/.pi/extensions/pi-workflow-engine/src/filesystem-error.ts

@@ -0,0 +1,4 @@
+/** Identify a missing filesystem path reported by Node's filesystem APIs. */
+export function isMissingPathError(error: unknown): boolean {
+  return error instanceof Error && "code" in error && error.code === "ENOENT";
+}

package/.pi/extensions/pi-workflow-engine/src/finalizers.ts

@@ -0,0 +1,66 @@
+import { unknownErrorMessage } from "./unknown-error.ts";
+
+export type FinalizerCriticality = "required" | "best-effort";
+
+export interface Finalizer {
+  readonly name: string;
+  readonly criticality: FinalizerCriticality;
+  readonly run: () => void | Promise<void>;
+}
+
+export interface FinalizerFailure {
+  readonly name: string;
+  readonly criticality: FinalizerCriticality;
+  readonly error: unknown;
+}
+
+export interface RunFinalizersOptions {
+  readonly onBestEffortFailure?: (failure: FinalizerFailure) => void;
+}
+
+export class RequiredFinalizerError extends AggregateError {
+  readonly failures: readonly FinalizerFailure[];
+
+  constructor(failures: readonly FinalizerFailure[]) {
+    super(
+      failures.map((failure) => failure.error),
+      `Required workflow finalization failed: ${failures.map((failure) => `${failure.name} (${unknownErrorMessage(failure.error)})`).join(", ")}`,
+    );
+    this.name = "RequiredFinalizerError";
+    this.failures = failures;
+  }
+}
+
+/** Execute every finalizer once, reporting advisory failures and aggregating only required failures. */
+export async function runFinalizers(
+  finalizers: readonly Finalizer[],
+  options: RunFinalizersOptions = {},
+): Promise<readonly FinalizerFailure[]> {
+  const requiredFailures: FinalizerFailure[] = [];
+  const bestEffortFailures: FinalizerFailure[] = [];
+
+  for (const finalizer of finalizers) {
+    try {
+      await finalizer.run();
+    } catch (error) {
+      const failure: FinalizerFailure = {
+        name: finalizer.name,
+        criticality: finalizer.criticality,
+        error,
+      };
+      if (finalizer.criticality === "required") {
+        requiredFailures.push(failure);
+        continue;
+      }
+      bestEffortFailures.push(failure);
+      try {
+        options.onBestEffortFailure?.(failure);
+      } catch {
+        // Failure observers are advisory and must not change finalization semantics.
+      }
+    }
+  }
+
+  if (requiredFailures.length > 0) throw new RequiredFinalizerError(requiredFailures);
+  return bestEffortFailures;
+}