pi-pignon 0.1.1

package/src/extension.ts

@@ -0,0 +1,439 @@
+/**
+ * pignon — Pi agent extension.
+ *
+ * Shifts to the right model for each prompt by asking a decider (today: the
+ * local Laya System-1 model, via a stdio worker it spawns and supervises) how
+ * hard the prompt is, then looking the answer up in the routing table.
+ *
+ *   /pignon                -> show current mode
+ *   /pignon live           -> apply decisions
+ *   /pignon shadow         -> observe only (default)
+ *   /pignon off            -> stop calling the decider
+ *   /pignon unpin          -> re-enable routing after manual model selection
+ *   /pignon log            -> recent decider diagnostics
+ *   /pignon config         -> routing table and settings in use
+ *   /pignon config migrate -> convert a laya-router config file
+ *   /pignon init [preset]  -> write a starter config file
+ *   /pignon doctor         -> check deciders, models and config
+ *   /pignon-stats          -> session statistics
+ *   /pignon-stats compare  -> how two deciders agree (parallel strategy)
+ *   /pignon-stats export [path] -> decisions as JSON lines
+ *
+ * Reports open in a dismissible overlay (see `report.ts`). The `clear`
+ * subcommands are still accepted and remove the widgets older versions left.
+ *
+ * `/laya` and `/laya-stats` remain as aliases for one release.
+ *
+ * This module only wires Pi to the router; the routing itself is in
+ * `router.ts`, the policy in `policy.ts`, and the deciders in `deciders/`.
+ */
+
+import type {
+  ExtensionAPI,
+  ExtensionContext,
+  SessionEntry,
+} from "@earendil-works/pi-coding-agent";
+
+import { buildCompareLines, defaultExportPath, exportDecisions } from "./compare.js";
+import { describeConfig } from "./config/describe.js";
+import { configPaths, loadConfig } from "./config/load.js";
+import { migrateConfigFile } from "./config/migrate.js";
+import { PRESETS, PRESET_NAMES, isPresetName } from "./config/presets.js";
+import { createDecider as createConfiguredDecider } from "./deciders/create.js";
+import type { Decider } from "./deciders/types.js";
+import { type RouterHost, routePrompt } from "./router.js";
+import { showReport } from "./report.js";
+import { buildStatsLines } from "./stats.js";
+import type { DeciderSpec, RouterConfig, RouterLogEntry, RouterMode } from "./types.js";
+import { hideDeciding, renderDecisionCard, showDeciding } from "./ui.js";
+import {
+  type ModelLookup,
+  choosePreset,
+  detectDeciders,
+  modelStatus,
+  runDoctor,
+  starterConfig,
+  writeStarterConfig,
+} from "./onboarding.js";
+
+/** Decider log lines shown by `/pignon log` (the report scrolls). */
+const LOG_REPORT_LINES = 200;
+
+/** Custom entry type of decision cards. */
+const ENTRY_TYPE = "pignon-decision";
+/** Entry type written by laya-router; still rendered and counted. */
+const LEGACY_ENTRY_TYPE = "laya-decision";
+
+const SUBCOMMANDS = [
+  "shadow",
+  "live",
+  "off",
+  "unpin",
+  "log",
+  "config",
+  "config migrate",
+  "init",
+  ...PRESET_NAMES.map((name) => `init ${name}`),
+  "doctor",
+];
+
+type PiModel = Parameters<ExtensionAPI["setModel"]>[0];
+
+// ---------------------------------------------------------------------------
+// UI helpers (no-ops without a UI, e.g. print or RPC mode)
+// ---------------------------------------------------------------------------
+
+function renderStatus(ctx: ExtensionContext, text: string): void {
+  if (ctx.hasUI) ctx.ui.setStatus("pignon", text);
+}
+
+function notify(ctx: ExtensionContext, text: string, level: "info" | "warning" | "error" = "info"): void {
+  if (ctx.hasUI) ctx.ui.notify(text, level);
+}
+
+/** Remove a report widget left by an older pignon (reports now use an overlay). */
+function clearWidget(ctx: ExtensionContext, name: string): void {
+  if (ctx.hasUI) ctx.ui.setWidget(name, undefined);
+}
+
+/** Show report lines whose first line is their title. */
+function showTitledReport(ctx: ExtensionContext, [title = "pignon", ...body]: string[]): Promise<void> {
+  return showReport(ctx, title, body);
+}
+
+/** Adapt Pi's API and a context to what the router needs. */
+function piHost(
+  pi: ExtensionAPI,
+  ctx: ExtensionContext,
+  switchModel: (model: PiModel) => Promise<boolean>,
+): RouterHost<PiModel> {
+  return {
+    get model() {
+      return ctx.model;
+    },
+    get contextTokens() {
+      return ctx.getContextUsage()?.tokens ?? 0;
+    },
+    get signal() {
+      return ctx.signal;
+    },
+    findModel: (provider, modelId) => ctx.modelRegistry.find(provider, modelId),
+    switchModel,
+    setThinkingLevel: (level) => pi.setThinkingLevel(level),
+    status: (text) => renderStatus(ctx, text),
+    notify: (text, level) => notify(ctx, text, level),
+    showDeciding: (deciderModel) => showDeciding(ctx, deciderModel),
+    hideDeciding: () => hideDeciding(ctx),
+  };
+}
+
+function isDecisionEntry(entry: SessionEntry): boolean {
+  if (entry.type !== "custom") return false;
+  const type = (entry as { customType?: string }).customType;
+  return type === ENTRY_TYPE || type === LEGACY_ENTRY_TYPE;
+}
+
+// ---------------------------------------------------------------------------
+// Entry point
+// ---------------------------------------------------------------------------
+
+export interface ExtensionOptions {
+  /** Builds the decider from the loaded config. Defaults to the one the config describes. */
+  createDecider?: (config: RouterConfig) => Decider;
+  /** Finds the deciders `/pignon init` writes. Defaults to probing this machine. */
+  detectDeciders?: () => Promise<DeciderSpec[]>;
+}
+
+/** Build the extension; tests inject their own decider. */
+export function createExtension(options: ExtensionOptions = {}): (pi: ExtensionAPI) => void {
+
+  return (pi) => {
+    let mode: RouterMode = "shadow";
+    let manualPin = false;
+    let promptsSinceSwitch: number | undefined;
+
+    // A small synchronous file read; the decider only starts work later.
+    const loaded = loadConfig();
+    const { config } = loaded;
+
+    const { decider, notes: deciderNotes } = options.createDecider
+      ? { decider: options.createDecider(config), notes: [] }
+      : createConfiguredDecider(config);
+
+    // pi.setModel() emits model_select with source "set", exactly like a manual
+    // /model choice. Flag our own switches so they do not pin the router.
+    let routerSwitching = false;
+    const switchModel = async (model: PiModel) => {
+      routerSwitching = true;
+      try {
+        return await pi.setModel(model);
+      } finally {
+        routerSwitching = false;
+      }
+    };
+
+    // --- Decision cards ---------------------------------------------------
+
+    pi.registerEntryRenderer<RouterLogEntry>(ENTRY_TYPE, renderDecisionCard);
+    pi.registerEntryRenderer<RouterLogEntry>(LEGACY_ENTRY_TYPE, renderDecisionCard);
+
+    // Routing runs in before_agent_start, before Pi posts the user message.
+    // The entry is held until that message is in so its card renders below
+    // the prompt it describes, not above it.
+    let pendingEntry: RouterLogEntry | undefined;
+    const flushPendingEntry = () => {
+      if (!pendingEntry) return;
+      pi.appendEntry(ENTRY_TYPE, pendingEntry);
+      pendingEntry = undefined;
+    };
+
+    // --- Session lifecycle ------------------------------------------------
+
+    // Warming up can take a while (the first run downloads the model), so it
+    // runs in the background: neither session start nor a prompt waits for it.
+    let sessionActive = false;
+    let warming: Promise<void> | undefined;
+    const warmUp = (ctx: ExtensionContext) => {
+      if (warming) return;
+      warming = decider
+        .warmup()
+        .then(
+          () => {
+            if (sessionActive) renderStatus(ctx, `pignon ${mode} · ${decider.model ?? "unknown model"}`);
+          },
+          (err) => {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (sessionActive) renderStatus(ctx, `pignon ${mode} · ⚠ ${msg.slice(0, 60)} (/pignon log)`);
+          },
+        )
+        .finally(() => {
+          warming = undefined;
+        });
+    };
+
+    pi.on("session_start", async (_event, ctx) => {
+      manualPin = false;
+      promptsSinceSwitch = undefined;
+      sessionActive = true;
+      if (loaded.errors.length > 0) {
+        notify(ctx, `pignon: config problems, using defaults for:\n${loaded.errors.join("\n")}`, "warning");
+      }
+      const warnings = [...loaded.warnings, ...deciderNotes];
+      if (warnings.length > 0) notify(ctx, `pignon: ${warnings.join("\n")}`, "warning");
+      renderStatus(ctx, `pignon ${mode} · loading model`);
+      warmUp(ctx);
+    });
+
+    pi.on("session_shutdown", () => {
+      flushPendingEntry();
+      sessionActive = false;
+      decider.stop();
+    });
+
+    // --- Routing hook -----------------------------------------------------
+
+    pi.on("before_agent_start", async (event, ctx) => {
+      flushPendingEntry();
+      if (mode === "off") return;
+      if (manualPin) {
+        renderStatus(ctx, "pignon ⏸ pinned");
+        return;
+      }
+      // Fail open while the decider loads (or reloads after a crash): route
+      // nothing rather than hold the prompt.
+      if (!decider.isReady) {
+        renderStatus(ctx, "pignon ⏳ model loading — prompt not routed");
+        warmUp(ctx);
+        return;
+      }
+
+      if (promptsSinceSwitch !== undefined) promptsSinceSwitch++;
+      const { applied, entry } = await routePrompt(
+        piHost(pi, ctx, switchModel),
+        { decider, config, mode, promptsSinceSwitch },
+        event.prompt,
+      );
+      pendingEntry = entry;
+      if (applied) promptsSinceSwitch = 0;
+    });
+
+    pi.on("message_end", async (event) => {
+      if (event.message.role === "user") flushPendingEntry();
+    });
+
+    // Fallback for runs that end without posting a user message.
+    pi.on("agent_end", async () => {
+      flushPendingEntry();
+    });
+
+    // --- Manual model selection guard -------------------------------------
+
+    pi.on("model_select", async (event, ctx) => {
+      if (event.source === "set" && !routerSwitching) {
+        manualPin = true;
+        renderStatus(ctx, "pignon ⏸ pinned");
+      }
+    });
+
+    // --- Commands ---------------------------------------------------------
+
+    const modeCommand = async (args: string, ctx: ExtensionContext) => {
+      const arg = args.trim();
+      if (arg === "log clear") {
+        clearWidget(ctx, "pignon-log");
+        return;
+      }
+      if (arg === "log") {
+        const lines = decider.recentLogs.slice(-LOG_REPORT_LINES);
+        if (lines.length === 0) {
+          notify(ctx, "pignon: no decider output yet");
+          return;
+        }
+        await showReport(ctx, "pignon log", lines);
+        return;
+      }
+      if (arg === "config clear") {
+        clearWidget(ctx, "pignon-config");
+        return;
+      }
+      if (arg === "config") {
+        await showTitledReport(ctx, describeConfig(config, loaded.source));
+        return;
+      }
+      if (arg === "init" || arg.startsWith("init ")) {
+        const requested = arg.slice("init".length).trim();
+        if (requested && !isPresetName(requested)) {
+          notify(ctx, `pignon: unknown preset "${requested}" (${PRESET_NAMES.join(", ")})`, "error");
+          return;
+        }
+        const lookup = ctx.modelRegistry as ModelLookup<unknown>;
+        const preset = requested && isPresetName(requested) ? requested : choosePreset(lookup);
+        const result = writeStarterConfig(configPaths().path, starterConfig(preset, await (options.detectDeciders ?? detectDeciders)()));
+        if (!result.ok) {
+          notify(ctx, `pignon: ${result.message}`, "error");
+          return;
+        }
+        const specs = Object.values(PRESETS[preset].models);
+        const usable = specs.filter((spec) => modelStatus(spec, lookup) === "ok").length;
+        notify(
+          ctx,
+          `pignon: wrote ${result.path} (preset ${preset}, ${usable}/${specs.length} models usable); /reload to use it, /pignon doctor to check it`,
+        );
+        return;
+      }
+      if (arg === "doctor clear") {
+        clearWidget(ctx, "pignon-doctor");
+        return;
+      }
+      if (arg === "doctor") {
+        const report = runDoctor({
+          config,
+          configSource: loaded.source,
+          configErrors: loaded.errors,
+          decider,
+          lookup: ctx.modelRegistry as ModelLookup<unknown>,
+        });
+        // The overlay opens at once; the decider probe can take seconds.
+        await showReport(ctx, "pignon doctor", report.then(([, ...body]) => body));
+        return;
+      }
+      if (arg === "config migrate") {
+        if (!loaded.legacy) {
+          notify(ctx, "pignon: config is already in the pignon format");
+          return;
+        }
+        const result = migrateConfigFile();
+        if (!result.ok) {
+          notify(ctx, `pignon: ${result.message}`, "error");
+          return;
+        }
+        const backup = result.backup ? ` (previous file kept as ${result.backup})` : "";
+        notify(ctx, `pignon: wrote ${result.to} from ${result.from}${backup}; /reload to use it`);
+        return;
+      }
+      if (arg === "unpin") {
+        manualPin = false;
+        notify(ctx, "pignon: routing re-enabled");
+        renderStatus(ctx, `pignon ${mode}`);
+        return;
+      }
+      if (arg === "shadow" || arg === "live" || arg === "off") {
+        mode = arg;
+        manualPin = false;
+        notify(ctx, `pignon: mode ${mode}`);
+        renderStatus(ctx, `pignon ${mode}`);
+        return;
+      }
+      const pinned = manualPin ? " (model pinned manually)" : "";
+      const configNote = loaded.source ? ` · config ${loaded.source}` : "";
+      notify(ctx, `pignon: mode ${mode}${pinned}${configNote}`);
+    };
+
+    const statsCommand = async (args: string, ctx: ExtensionContext) => {
+      const [sub = "", ...rest] = args.trim().split(/\s+/);
+      if (sub === "clear") {
+        clearWidget(ctx, "pignon-stats");
+        return;
+      }
+
+      const rows = ctx.sessionManager
+        .getEntries()
+        .filter(isDecisionEntry)
+        .map((e) => (e as { data?: RouterLogEntry }).data)
+        .filter((d): d is RouterLogEntry => d !== undefined);
+
+      if (rows.length === 0) {
+        notify(ctx, "pignon: no decisions in this session");
+        return;
+      }
+
+      if (sub === "compare") {
+        const lines = buildCompareLines(rows, config.table);
+        if (lines) await showTitledReport(ctx, lines);
+        else notify(ctx, "pignon: no decisions answered by two deciders yet (set strategy.mode to parallel to compare them)");
+        return;
+      }
+      if (sub === "export") {
+        const path = rest.join(" ") || defaultExportPath();
+        try {
+          exportDecisions(rows, path);
+          notify(ctx, `pignon: wrote ${rows.length} decisions to ${path}`);
+        } catch (err) {
+          notify(ctx, `pignon: could not write ${path}: ${err instanceof Error ? err.message : String(err)}`, "error");
+        }
+        return;
+      }
+
+      await showTitledReport(ctx, buildStatsLines(rows, config.table));
+    };
+
+    const completions = (prefix: string) =>
+      SUBCOMMANDS.filter((v) => v.startsWith(prefix)).map((v) => ({ value: v, label: v }));
+
+    pi.registerCommand("pignon", {
+      description: "pignon mode (shadow | live | off | unpin | log | config)",
+      getArgumentCompletions: completions,
+      handler: modeCommand,
+    });
+    const statsCompletions = (prefix: string) =>
+      ["compare", "export"].filter((v) => v.startsWith(prefix)).map((v) => ({ value: v, label: v }));
+
+    pi.registerCommand("pignon-stats", {
+      description: "Session statistics (compare: decider vs decider · export [path]: JSON lines)",
+      getArgumentCompletions: statsCompletions,
+      handler: statsCommand,
+    });
+    pi.registerCommand("laya", {
+      description: "Alias of /pignon (deprecated)",
+      getArgumentCompletions: completions,
+      handler: modeCommand,
+    });
+    pi.registerCommand("laya-stats", {
+      description: "Alias of /pignon-stats (deprecated)",
+      getArgumentCompletions: statsCompletions,
+      handler: statsCommand,
+    });
+  };
+}
+
+export default createExtension();

package/src/onboarding.ts

@@ -0,0 +1,203 @@
+/**
+ * `/pignon init` and `/pignon doctor`: get a working setup, and find out
+ * what is wrong with one.
+ *
+ * Pi-free: the model registry is reached through `ModelLookup`.
+ */
+
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+import { type PresetName, PRESETS, PRESET_NAMES } from "./config/presets.js";
+import { CONFIG_SCHEMA_URL } from "./config/schema.js";
+import { DEFAULT_API_KEY_ENV } from "./deciders/jev.js";
+import { type WorkerLaunch, layaRuntimeStatus } from "./deciders/laya-local.js";
+import { probeLayaServe } from "./deciders/laya-serve.js";
+import { StrategyDecider } from "./deciders/strategy.js";
+import { type Decider, DeciderError } from "./deciders/types.js";
+import { buildQuestions } from "./deciders/questions.js";
+import type { DeciderSpec, ModelSpec, RouterConfig } from "./types.js";
+
+/** The part of Pi's model registry pignon checks models against. */
+export interface ModelLookup<M = unknown> {
+  find(provider: string, modelId: string): M | undefined;
+  hasConfiguredAuth(model: M): boolean;
+}
+
+export type ModelStatus = "ok" | "missing" | "no-auth";
+
+export function modelStatus<M>(spec: ModelSpec, lookup: ModelLookup<M>): ModelStatus {
+  const model = lookup.find(spec.provider, spec.modelId);
+  if (model === undefined) return "missing";
+  return lookup.hasConfiguredAuth(model) ? "ok" : "no-auth";
+}
+
+// ---------------------------------------------------------------------------
+// init
+// ---------------------------------------------------------------------------
+
+/**
+ * The preset to start from: the first whose four models all exist and have
+ * credentials in Pi, else the one with the most usable models.
+ */
+export function choosePreset<M>(lookup: ModelLookup<M>): PresetName {
+  const usable = (name: PresetName) =>
+    Object.values(PRESETS[name].models).filter((spec) => modelStatus(spec, lookup) === "ok").length;
+  return PRESET_NAMES.reduce((best, name) => (usable(name) > usable(best) ? name : best), PRESET_NAMES[0]!);
+}
+
+/**
+ * The decider to start with, local first: a running laya-serve on its default
+ * address, else the experimental worker when installed, else Jev when its key
+ * is set.
+ */
+export async function detectDeciders(
+  env: NodeJS.ProcessEnv = process.env,
+  layaStatus: typeof layaRuntimeStatus = layaRuntimeStatus,
+  probe: () => Promise<boolean> = () => probeLayaServe(),
+): Promise<DeciderSpec[]> {
+  if (await probe()) return [{ type: "laya-serve" }];
+  if (layaStatus(env).ok) return [{ type: "laya-local" }];
+  if (env[DEFAULT_API_KEY_ENV]?.trim()) return [{ type: "jev" }];
+  return [];
+}
+
+/**
+ * A starter config: the preset's models written out (so they are easy to
+ * edit), the deciders that can run here, and the built-in tiers.
+ */
+export function starterConfig(preset: PresetName, deciders: DeciderSpec[]): Record<string, unknown> {
+  return {
+    $schema: CONFIG_SCHEMA_URL,
+    version: 2,
+    ...(deciders.length > 0 ? { deciders } : {}),
+    models: PRESETS[preset].models,
+  };
+}
+
+export type InitResult = { ok: true; path: string } | { ok: false; message: string };
+
+/** Write a starter config; never replaces an existing file. */
+export function writeStarterConfig(path: string, config: Record<string, unknown>): InitResult {
+  if (existsSync(path)) {
+    return { ok: false, message: `${path} already exists; edit it, or move it away to start over` };
+  }
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`, { flag: "wx" });
+  return { ok: true, path };
+}
+
+// ---------------------------------------------------------------------------
+// doctor
+// ---------------------------------------------------------------------------
+
+export interface DoctorInput<M> {
+  config: RouterConfig;
+  configSource: string | null;
+  configErrors: readonly string[];
+  decider: Decider;
+  lookup: ModelLookup<M>;
+  env?: NodeJS.ProcessEnv;
+  layaStatus?: typeof layaRuntimeStatus;
+}
+
+/** A fixed, harmless prompt for the decider round trip. */
+const PROBE_PROMPT = "Rename the variable `cnt` to `count` in src/stats.ts";
+
+/**
+ * One ✓/⚠/✗ line per check: config file, each decider (with one real
+ * decision when it is ready), and each model of the routing table.
+ */
+export async function runDoctor<M>(input: DoctorInput<M>): Promise<string[]> {
+  const { config, decider, lookup } = input;
+  const env = input.env ?? process.env;
+  const lines: string[] = ["pignon doctor"];
+  const ok = (text: string) => lines.push(`  ✓ ${text}`);
+  const warn = (text: string) => lines.push(`  ⚠ ${text}`);
+  const fail = (text: string) => lines.push(`  ✗ ${text}`);
+
+  lines.push("config");
+  if (input.configErrors.length > 0) {
+    fail(`${input.configSource ?? "config"}: ${input.configErrors.length} problem(s), defaults used for those parts`);
+    for (const error of input.configErrors) lines.push(`      ${error}`);
+  } else {
+    ok(input.configSource ?? "no config file: built-in defaults (/pignon init writes one)");
+  }
+
+  lines.push("deciders");
+  const members = decider instanceof StrategyDecider ? decider.members : [decider];
+  if (members.length > 1) ok(`${config.strategy.mode} strategy over ${members.map((d) => d.id).join(", ")}`);
+  for (const member of members) {
+    await checkDecider(member, config, env, input.layaStatus ?? layaRuntimeStatus, { ok, warn, fail });
+  }
+
+  lines.push("models");
+  const seen = new Set<string>();
+  for (const tier of config.table) {
+    for (const spec of [tier.models.direct, tier.models.exploration]) {
+      const name = `${spec.provider}/${spec.modelId}`;
+      if (seen.has(name)) continue;
+      seen.add(name);
+      const status = modelStatus(spec, lookup);
+      if (status === "ok") ok(`${name} (${tier.id})`);
+      else if (status === "no-auth") warn(`${name} (${tier.id}): no credentials in Pi (/login, or the provider's API key variable)`);
+      else fail(`${name} (${tier.id}): not in Pi's model registry (check the id with pi --list-models)`);
+    }
+  }
+  return lines;
+}
+
+function describeLaunch(launch: WorkerLaunch): string {
+  switch (launch.source) {
+    case "config":
+      return `the configured command (${[launch.command, ...launch.args].join(" ")})`;
+    case "env":
+      return `LAYA_PYTHON (${launch.command})`;
+    case "checkout":
+      return `the source checkout (${launch.cwd})`;
+    case "path":
+      return launch.command;
+  }
+}
+
+async function checkDecider(
+  decider: Decider,
+  config: RouterConfig,
+  env: NodeJS.ProcessEnv,
+  layaStatus: typeof layaRuntimeStatus,
+  report: { ok: (t: string) => void; warn: (t: string) => void; fail: (t: string) => void },
+): Promise<void> {
+  const name = `${decider.id}${decider.remote ? " (remote)" : ""}`;
+  if (decider.id === "laya-local") {
+    const spec = config.deciders?.find((d) => d.type === "laya-local");
+    const command = spec?.type === "laya-local" ? spec.command : undefined;
+    const status = layaStatus(env, process.platform, process.arch, command);
+    if (!status.ok) return report.fail(`${name}: ${status.reason}`);
+    if (status.launch) report.ok(`${name}: worker from ${describeLaunch(status.launch)}`);
+  }
+  if (!decider.isReady) {
+    if (decider.id === "laya-local") {
+      // Loading takes seconds to minutes (first run downloads the model): start it, don't wait.
+      decider.warmup().catch(() => {});
+      return report.warn(`${name}: model not loaded yet; loading now, run /pignon doctor again in a moment`);
+    }
+    try {
+      await decider.warmup();
+    } catch (err) {
+      return report.fail(`${name}: ${failureOf(decider, err)}`);
+    }
+  }
+  try {
+    const result = await decider.decide({ text: PROBE_PROMPT, questions: buildQuestions(config) }, AbortSignal.timeout(15_000));
+    const cost = result.costUsd !== undefined ? ` · $${result.costUsd.toFixed(6)}` : "";
+    report.ok(`${name}: ${result.model} answered a test prompt in ${result.latencyMs} ms${cost}`);
+  } catch (err) {
+    report.fail(`${name}: ${failureOf(decider, err)}`);
+  }
+}
+
+/** An error's message without the decider id it starts with, since the line already names the decider. */
+function failureOf(decider: Decider, err: unknown): string {
+  const message = err instanceof DeciderError || err instanceof Error ? err.message : String(err);
+  return message.startsWith(`${decider.id}: `) ? message.slice(decider.id.length + 2) : message;
+}