toolroll 0.9.95 → 0.9.97

package/dist/contracts/console-api.js

@@ -183,6 +183,7 @@ export const CONSOLE_FORMS = {
     leadIdentity: formContract("POST /settings/lead/identity", ["name", "persona"]),
     leadAbout: formContract("POST /settings/lead/about", ["about"]),
     leadPromiseCancel: formContract("POST /settings/lead/promise/cancel", ["promise"]),
+    leadQuick: formContract("POST /settings/lead/quick", ["enabled"]),
     leadCheck: formContract("POST /lead/check", ["return"]),
     leadOn: formContract("POST /settings/lead/on", ["agent", "return"]),
     welcomeSample: formContract("POST /welcome/sample", []),

package/dist/desktop-update.d.ts

@@ -97,6 +97,7 @@ export declare function sqliteLock(file: string, busyTimeoutMs?: number): Databa
 export declare function lockHeld(file: string): boolean;
 export declare const installationLock: (j: UpdateJournal) => DatabaseSync | null;
 export declare function desktopUpdateStatus(stateDir: string): {
+    version?: never;
     active: boolean;
     phase: string;
     detail: string;
@@ -110,7 +111,6 @@ export declare function desktopUpdateStatus(stateDir: string): {
     wasRunning?: never;
     backupPath?: never;
     updateId?: never;
-    version?: never;
     buildId?: never;
     workDir?: never;
 } | {

package/dist/discord-api.d.ts

@@ -5,7 +5,7 @@ export declare const discordId: (v: unknown) => v is string;
 export type DiscordCredentials = ChatIdentity & {
     botToken: string;
 };
-export type DiscordApi = (method: "GET" | "POST" | "PATCH" | "PUT", path: string, body?: Record<string, unknown>, file?: {
+export type DiscordApi = (method: "GET" | "POST" | "PATCH" | "PUT" | "DELETE", path: string, body?: Record<string, unknown>, file?: {
     bytes: Uint8Array;
     name: string;
 }) => Promise<Record<string, unknown>>;
@@ -13,6 +13,8 @@ export declare class DiscordError extends ChatDeliveryError {
 }
 /** Discord's 403: the bot may not do this here (for a file, no Attach Files permission). */
 export declare const DISCORD_REFUSED = "Discord refused access. Check the app and direct-message permissions.";
+/** Discord's 404: the message (or the conversation) is gone. */
+export declare const DISCORD_GONE = "The Discord message or conversation is unavailable. Open the saved chat.";
 export declare function discordApi(token: string, fetcher?: typeof fetch): DiscordApi;
 export declare const discordCredentialFile: (dir: string) => string;
 export declare function loadDiscordCredentials(dir: string): DiscordCredentials | null;

package/dist/discord-api.js

@@ -10,10 +10,16 @@ export class DiscordError extends ChatDeliveryError {
 }
 /** Discord's 403: the bot may not do this here (for a file, no Attach Files permission). */
 export const DISCORD_REFUSED = "Discord refused access. Check the app and direct-message permissions.";
+/** Discord's 404: the message (or the conversation) is gone. */
+export const DISCORD_GONE = "The Discord message or conversation is unavailable. Open the saved chat.";
 export function discordApi(token, fetcher = fetch) {
     return async (method, path, body, file) => {
-        // Only relative API routes; never send the bot credential to supplied URLs.
-        if (!/^\/(?:users\/@me|users\/[0-9]+|oauth2\/applications\/@me|channels\/[0-9]+(?:\/typing|\/messages\/[0-9]+\/reactions\/%F0%9F%91%8D\/@me|\/messages(?:\/[0-9]+)?(?:\?limit=100)?)?|interactions\/[0-9]+\/[A-Za-z0-9._-]+\/callback)$/.test(path))
+        // Only relative API routes; never send the bot credential to supplied URLs. Beyond messages: typing, and the bot's
+        // own 👀 on a message (put on, taken off).
+        if (!/^\/(?:users\/@me|users\/[0-9]+|oauth2\/applications\/@me|channels\/[0-9]+(?:\/typing|\/messages\/[0-9]+\/reactions\/%F0%9F%91%80\/@me|\/messages(?:\/[0-9]+)?(?:\?limit=100)?)?|interactions\/[0-9]+\/[A-Za-z0-9._-]+\/callback)$/.test(path))
+            throw new DiscordError("Invalid Discord request");
+        // DELETE takes off the bot's own 👀 and nothing else (never a message).
+        if (method === "DELETE" && !/^\/channels\/[0-9]+\/messages\/[0-9]+\/reactions\/%F0%9F%91%80\/@me$/.test(path))
             throw new DiscordError("Invalid Discord request");
         let response;
         const headers = { Authorization: `Bot ${token}` };
@@ -60,7 +66,7 @@ export function discordApi(token, fetcher = fetch) {
                 : response.status === 403
                     ? DISCORD_REFUSED
                     : response.status === 404
-                        ? "The Discord message or conversation is unavailable. Open the saved chat."
+                        ? DISCORD_GONE
                         : response.status === 413
                             ? "Discord refused this file size. Open the saved result to view the evidence."
                             : "Discord could not complete the request";

package/dist/discord-chat.d.ts

@@ -1,9 +1,11 @@
-import { ChatState, type ChatIdentity } from "./chat-delivery-state.js";
+import { ChatState, type ChatIdentity, type ChatEvent } from "./chat-delivery-state.js";
 import { type ChatDeliveryOptions } from "./chat-delivery.js";
 import { discordPlain } from "./reply-shape.js";
+import { type ChatAckHooks, type ChatLiveTransport, type LiveReplies } from "./chat-live.js";
 import { type DiscordApi } from "./discord-api.js";
-export type DiscordChatOptions = Omit<ChatDeliveryOptions, "state" | "label" | "member" | "partSize" | "maxProposal"> & {
-    api: DiscordApi;
+export type DiscordChatOptions = Omit<ChatDeliveryOptions, "state" | "label" | "member" | "partSize" | "maxProposal" | "live"> & {
+    api: DiscordApi; /** The live "Working on it…" message (chat-live.ts): on in the worker. */
+    liveReplies?: LiveReplies;
 };
 export declare const discordDelivery: (options: DiscordChatOptions) => ChatDeliveryOptions;
 export declare const processDiscordEvent: (options: DiscordChatOptions) => Promise<boolean>;
@@ -15,3 +17,11 @@ export declare function receiveDiscord(state: ChatState, identity: ChatIdentity,
 export { discordPlain };
 export declare function discordCard(text: string, buttons?: Record<string, unknown>[], voice?: boolean): Record<string, unknown>;
 export declare function deliverDiscordPart(options: DiscordChatOptions): Promise<boolean>;
+/** The lead's words as Discord carries them, already rendered: one embed description (cut to its 4 096), empty
+ * content, its buttons (none on a live message) and never a ping. Every field is sent on every edit: Discord keeps
+ * what an edit leaves out, and inherits no allowed_mentions. */
+export declare function discordVoicePayload(rendered: string, buttons?: Record<string, unknown>[]): Record<string, unknown>;
+/** The live message for one owner message: a reply to it, created under a nonce so a repeat returns the same one. */
+export declare function discordLive(options: Pick<DiscordChatOptions, "api" | "identity">, event: ChatEvent, spacingMs?: number): ChatLiveTransport;
+/** The 👀 on the owner's message, and taking it off. A missing permission is asked about at most once a minute, quietly. */
+export declare function discordAckHooks(options: Pick<DiscordChatOptions, "api">): ChatAckHooks;

package/dist/discord-chat.js

@@ -14,23 +14,27 @@ import { processChatEvent, planChatNotifications, channelAccess, chatObject as o
 import { chatResultHref } from "./chat-controls.js";
 import { LEAD_MESSAGE_MAX_CHARS } from "./lead.js";
 import { discordPlain, renderReply } from "./reply-shape.js";
-import { WARM_EMOJI } from "./chat-warmth.js";
+import { isLiveMessage, liveSpacingOf, underCooldown } from "./chat-live.js";
 import { proposalPreview, proposalLink, proposalOutcomeText, armedCardText, armedYesLabel, } from "./chat-channel.js";
 import { resultImageFileName, safeResultImageCaption, verifyResultImage, } from "./chat-evidence.js";
-import { discordId, discordMember, DiscordError, DISCORD_REFUSED, } from "./discord-api.js";
-export const discordDelivery = (options) => ({
-    ...options,
-    state: new ChatState(options.store, "discord"),
-    label: "Discord",
-    member: (member, channel) => discordMember(options.api, member, channel),
-    // A 👍 on the owner's message and "typing…" while the lead works; a missing permission is skipped silently.
-    warm: (event) => ({
-        react: () => options.api("PUT", `/channels/${event.channel}/messages/${event.ts}/reactions/${encodeURIComponent(WARM_EMOJI)}/@me`),
-        typing: () => options.api("POST", `/channels/${event.channel}/typing`),
-    }),
-    partSize: 1800,
-    maxProposal: 3400,
-});
+import { discordId, discordMember, DiscordError, DISCORD_REFUSED, DISCORD_GONE, } from "./discord-api.js";
+export const discordDelivery = (options) => {
+    const state = new ChatState(options.store, "discord");
+    return {
+        ...options,
+        state,
+        label: "Discord",
+        member: (member, channel) => discordMember(options.api, member, channel),
+        // "typing…" while the lead works (the 👀 goes at arrival: discordAckHooks), under the shared cooldown; a missing
+        // permission is skipped silently.
+        warm: (event) => ({
+            typing: underCooldown(state, options.identity.installation, error => error instanceof DiscordError && error.code === "ratelimited" ? error.retryMs : null, () => options.clock?.() ?? new Date(), () => options.api("POST", `/channels/${event.channel}/typing`)),
+        }),
+        ...(options.liveReplies ? { live: (event) => discordLive(options, event, liveSpacingOf(options.liveReplies)) } : {}),
+        partSize: 1800,
+        maxProposal: 3400,
+    };
+};
 export const processDiscordEvent = (options) => processChatEvent(discordDelivery(options));
 export const planDiscordRooms = (options) => planRoomMessages(discordDelivery(options));
 export const planDiscordNotifications = (options) => planChatNotifications(discordDelivery(options));
@@ -151,12 +155,7 @@ export { discordPlain };
 export function discordCard(text, buttons = [], voice = false) {
     // The lead's own reply: one description in Discord's Markdown (bold anchors, labelled links), no title line promoted to a header.
     if (voice)
-        return {
-            content: "",
-            embeds: [{ description: renderReply(text, "discord").slice(0, 4096), color: 0x297b70 }],
-            components: Array.from({ length: Math.min(5, Math.ceil(buttons.length / 5)) }, (_, row) => ({ type: 1, components: buttons.slice(row * 5, row * 5 + 5) })),
-            allowed_mentions: { parse: [], replied_user: false },
-        };
+        return discordVoicePayload(renderReply(text, "discord"), buttons);
     const lines = text.split("\n"), first = lines[0] ?? "Toolroll", title = discordPlain(first).length <= 256 &&
         first.length <= 150 &&
         lines.length > 1
@@ -401,6 +400,12 @@ export async function deliverDiscordPart(options) {
         }
         // No permission to attach files here: one plain line instead of the result's screenshots.
         const saved = readChatPart(row.payload), content = saved.ok ? saved.value : null;
+        // The live message the reply was to replace is gone: the reply goes out as a new message.
+        if (content?.edit !== undefined && error instanceof DiscordError && error.code === DISCORD_GONE && isLiveMessage(state, row.event, content.edit)) {
+            const { edit: _edit, ...asNew } = content;
+            state.repaint(row.id, asNew);
+            return true;
+        }
         // Discord refused the card's embed: the part's own plain text goes instead, so the message is never lost.
         if (content?.rich && error instanceof DiscordError && error.code === "Discord could not complete the request" && error.retryMs === 60_000) {
             state.repaint(row.id, withoutCard(content));
@@ -422,3 +427,54 @@ export async function deliverDiscordPart(options) {
         return true;
     }
 }
+/** The lead's words as Discord carries them, already rendered: one embed description (cut to its 4 096), empty
+ * content, its buttons (none on a live message) and never a ping. Every field is sent on every edit: Discord keeps
+ * what an edit leaves out, and inherits no allowed_mentions. */
+export function discordVoicePayload(rendered, buttons = []) {
+    return {
+        content: "",
+        embeds: [{ description: rendered.slice(0, 4096), color: 0x297b70 }],
+        components: Array.from({ length: Math.min(5, Math.ceil(buttons.length / 5)) }, (_, row) => ({ type: 1, components: buttons.slice(row * 5, row * 5 + 5) })),
+        allowed_mentions: { parse: [], replied_user: false },
+    };
+}
+/** The live message for one owner message: a reply to it, created under a nonce so a repeat returns the same one. */
+export function discordLive(options, event, spacingMs) {
+    const nonce = chatHash(`discord:live:${event.id}`).slice(0, 24);
+    const confirmed = (answer, id) => {
+        if (!discordId(answer.id) || answer.channel_id !== event.channel || object(answer.author).id !== options.identity.bot || (id !== undefined && answer.id !== id))
+            throw new DiscordError("Discord did not confirm the message identity", 15_000, true);
+        return answer.id;
+    };
+    return {
+        platform: "discord",
+        ...(spacingMs === undefined ? {} : { spacingMs }),
+        limit: 4_000,
+        create: async (text) => confirmed(await options.api("POST", `/channels/${event.channel}/messages`, {
+            ...discordVoicePayload(text), nonce, enforce_nonce: true,
+            ...(discordId(event.thread) ? { message_reference: { message_id: event.thread, channel_id: event.channel, fail_if_not_exists: false } } : {}),
+        })),
+        // Discord checks an enforced nonce against "the past few minutes": a repeat within one minute is the same message.
+        nonceMs: 60_000,
+        recover: async () => {
+            const recent = await options.api("GET", `/channels/${event.channel}/messages?limit=100`);
+            const found = (Array.isArray(recent.items) ? recent.items : []).map(object)
+                .find(one => one.nonce === nonce && object(one.author).id === options.identity.bot && one.channel_id === event.channel && discordId(one.id));
+            return found === undefined ? null : String(found.id);
+        },
+        edit: async (message, text) => { confirmed(await options.api("PATCH", `/channels/${event.channel}/messages/${message}`, discordVoicePayload(text)), message); },
+        rateLimited: error => error instanceof DiscordError && error.code === "ratelimited" ? error.retryMs : null,
+        uncertain: error => !(error instanceof ChatDeliveryError) || error.uncertain,
+    };
+}
+/** The 👀 on the owner's message, and taking it off. A missing permission is asked about at most once a minute, quietly. */
+export function discordAckHooks(options) {
+    const path = (event) => `/channels/${event.channel}/messages/${event.ts}/reactions/${encodeURIComponent("👀")}/@me`;
+    return {
+        react: (event) => options.api("PUT", path(event)),
+        unreact: (event) => options.api("DELETE", path(event)),
+        refused: error => error instanceof DiscordError && error.code === DISCORD_REFUSED,
+        settled: error => error instanceof DiscordError && error.code === DISCORD_GONE,
+        rateLimited: error => error instanceof DiscordError && error.code === "ratelimited" ? error.retryMs : null,
+    };
+}

package/dist/discord.js

@@ -3,7 +3,8 @@ import { randomBytes } from "node:crypto";
 import { abortablePause, workChatQueue } from "./chat-follow.js";
 import { loadDiscordCredentials, discordApi, object, discordId, } from "./discord-api.js";
 import { ChatState, chatHash } from "./chat-delivery-state.js";
-import { receiveDiscord, processDiscordEvent, deliverDiscordPart, planDiscordNotifications, planDiscordRooms, } from "./discord-chat.js";
+import { runChatAcks } from "./chat-live.js";
+import { receiveDiscord, processDiscordEvent, deliverDiscordPart, planDiscordNotifications, planDiscordRooms, discordAckHooks, } from "./discord-chat.js";
 export function discordReadyMatches(raw, identity) {
     const packet = object(raw);
     return (object(packet.application).id === identity.app &&
@@ -69,6 +70,9 @@ export async function followDiscord(options) {
         });
         client.on("error", () => problem("Discord connection failed. Check the bot token and reconnect."));
         client.on("shardError", () => problem("Discord connection failed. Saved replies will retry."));
+        // The 👀 on each owner message as it arrives, beside the queue (a turn there can take a minute).
+        const acks = runChatAcks({ state, installation: credentials.installation, owner, current: () => same() && connected,
+            hooks: discordAckHooks({ api }), same, signal: options.signal });
         // Never subscribe a logger to raw payloads, SDK debug output or interaction tokens.
         client.on("raw", (packet) => {
             if (!same())
@@ -88,6 +92,8 @@ export async function followDiscord(options) {
             const body = object(packet.d);
             try {
                 const saved = receiveDiscord(state, credentials, packet.t, body, new Date());
+                if (saved && packet.t === "MESSAGE_CREATE")
+                    acks.wake();
                 if (packet.t === "INTERACTION_CREATE" &&
                     body.type === 3 &&
                     body.application_id === credentials.app &&
@@ -119,6 +125,7 @@ export async function followDiscord(options) {
             api,
             current: same,
             canNotify: options.notifications,
+            liveReplies: true,
         };
         try {
             await client.login(credentials.botToken);

package/dist/lead-answer-eval.d.ts

@@ -0,0 +1,86 @@
+import { type ChatConfig, type Store } from "./store.js";
+import { type VerifiedApprover } from "./principal.js";
+import type { SubscriptionLeadRunner } from "./subscription-chat.js";
+export declare const LEAD_ANSWER_K = 3;
+type Seeded = {
+    id: string;
+    title: string;
+    state: "queued" | "running" | "done" | "failed" | "cancelled";
+};
+export declare const LEAD_ANSWER_TASKS: readonly Seeded[];
+/** A fact an answer must state: any one of its spellings (lower case). */
+type Fact = readonly string[];
+export type LeadAnswerQuestion = {
+    id: string;
+    text: string;
+    kind: "read" | "act";
+    mustName: readonly Fact[];
+    mustNot: readonly string[];
+    topic: string;
+};
+/** The 40 questions, each read question's oracle computed from LEAD_ANSWER_TASKS (the world's records). */
+export declare function leadAnswerQuestions(): LeadAnswerQuestion[];
+/** The seeded world: its store, its owner, the conversation config. */
+export declare function seedLeadAnswerWorld(now?: Date): {
+    store: Store;
+    who: VerifiedApprover;
+    config: ChatConfig;
+};
+/** How an answer did against its question's oracle. */
+export declare function gradeAnswer(question: LeadAnswerQuestion, answer: string): {
+    named: boolean;
+    falseClaims: number;
+};
+/**
+ * The scripted stand-in model (CI): reads with list_tasks and list_decisions, then answers from those results alone; a
+ * request without proposal tools (a quick step) asked to change something calls find_tools, as the contract tells a
+ * model to search before saying something can't be done. Each step advances `clock` by the model's simulated latency.
+ */
+export declare function scriptedLeadAnswerRunner(questions: readonly LeadAnswerQuestion[], clock: {
+    advance(ms: number): void;
+}, quickModel: string): SubscriptionLeadRunner;
+type Run = {
+    model: "main" | "quick";
+    route: string | null;
+    answer: string;
+    ms: number;
+    named: boolean;
+    falseClaims: number;
+    failed: string | null;
+};
+export type LeadAnswerReport = {
+    mode: "scripted" | "real";
+    k: number;
+    quickModel: string;
+    read: {
+        main: {
+            passK: number;
+            falseClaims: number;
+            medianMs: number;
+        };
+        quick: {
+            passK: number;
+            falseClaims: number;
+            medianMs: number;
+        };
+        questions: number;
+    };
+    act: {
+        kept: number;
+        total: number;
+        failures: string[];
+    };
+    speedup: number;
+    pass: boolean;
+    reasons: string[];
+    runs: ({
+        question: string;
+    } & Run)[];
+};
+/** Run the eval. `runner` absent: this computer's signed-in Claude (real mode). */
+export declare function runLeadAnswerEval(options: {
+    mode: "scripted" | "real";
+    k?: number;
+    onRun?: (line: string) => void;
+}): Promise<LeadAnswerReport>;
+export {};

package/dist/lead-answer-eval.js

@@ -0,0 +1,255 @@
+/**
+ * The `lead-answer` eval: the gate on quick answers (docs/plans/responsive-chat.md §8). Quick answers ship OFF and are
+ * turned on by default only once this eval's real mode passes.
+ *
+ * A seeded world (tasks in every state, one decision waiting on the owner) and 40 questions:
+ * - 30 read questions, each with an oracle computed from the world's records: the task titles, states and counts its
+ *   answer must name, and the titles it must not claim;
+ * - 10 act-intent questions, which must stay on the main model or be handed to it (escalated): 8 carry an act word, and
+ *   2 ask for a change without one, so only the quick step's own escalation can catch them.
+ * Each read question runs k = 3 times on the main model (quick answers off) and k = 3 on the quick route (on); a question
+ * passes on a model only when all three answers do (pass^k, the eval harness's rule). An act question that the router
+ * already keeps on the main model needs no model call; one it routes quick runs k = 3 and must escalate every time.
+ *
+ * Pass: quick pass^k ≥ main pass^k (no regression), quick false claims ≤ main's, 10/10 act questions on the main model
+ * or escalated, and the quick route's median turn time at least 30% below the main model's.
+ *
+ * Modes: `scripted` (CI: a deterministic stand-in model with simulated per-model latency proves the harness, the
+ * router, escalation and grading end to end, with no provider call) and `real` (the owner or the release runs it:
+ * every turn goes to this computer's signed-in Claude). Real mode spends subscription usage: about 30×3×2 + 2×3 turns.
+ *
+ *   npm run build && node scripts/lead-answer-eval.mjs            # scripted
+ *   npm run build && node scripts/lead-answer-eval.mjs --real     # real model calls
+ *
+ * Kept outside src/evals.ts's canonical campaigns: those run build and ask cases through a world's CLI and are sealed
+ * as release policy (`evals/`); this eval drives runLeadTurn directly over an in-memory world. Folding it in as a third
+ * case kind is a follow-up.
+ */
+import { createHash } from "node:crypto";
+import { openStore, setLeadQuickAnswers } from "./store.js";
+import { fileTaskProposal } from "./proposal.js";
+import { verifyApproverStanding } from "./principal.js";
+import { subscriptionCredentialKey } from "./converse.js";
+import { runLeadTurn } from "./lead.js";
+import { quickLeadModel, quickRoute } from "./lead-route.js";
+export const LEAD_ANSWER_K = 3;
+const T0 = new Date("2026-11-08T12:00:00.000Z");
+const REPO = "/eval/shop";
+const MAIN_MODEL = "opus";
+export const LEAD_ANSWER_TASKS = Object.freeze([
+    { id: "csv-export", title: "Add CSV export to the orders page", state: "running" },
+    { id: "search-index", title: "Speed up the search index", state: "running" },
+    { id: "billing-tab", title: "Rename the billing settings tab", state: "queued" },
+    { id: "french", title: "Translate the checkout into French", state: "queued" },
+    { id: "invoices", title: "Archive old invoices", state: "queued" },
+    { id: "login-loop", title: "Fix the login redirect loop", state: "done" },
+    { id: "images", title: "Compress product images", state: "done" },
+    { id: "payment-lib", title: "Upgrade the payment library", state: "failed" },
+    { id: "backup-check", title: "Repair the nightly backup check", state: "failed" },
+    { id: "dark-emails", title: "Try a dark theme for emails", state: "cancelled" },
+]);
+/** The task whose run waits on the owner's decision. */
+const DECIDING = "search-index";
+const titles = (state) => LEAD_ANSWER_TASKS.filter(one => one.state === state).map(one => one.title);
+const facts = (state) => titles(state).map(title => [title.toLowerCase()]);
+const NUMBER_WORDS = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"];
+const count = (state) => { const n = titles(state).length; return [String(n), NUMBER_WORDS[n]]; };
+/** The 40 questions, each read question's oracle computed from LEAD_ANSWER_TASKS (the world's records). */
+export function leadAnswerQuestions() {
+    const read = (topic, texts, mustName, mustNot) => texts.map((text, index) => ({ id: `${topic}-${index + 1}`, text, kind: "read", mustName, mustNot, topic }));
+    const settled = [...titles("done"), ...titles("cancelled")];
+    const csv = LEAD_ANSWER_TASKS.find(one => one.id === "csv-export");
+    const deciding = LEAD_ANSWER_TASKS.find(one => one.id === DECIDING);
+    const act = (id, text) => ({ id, text, kind: "act", mustName: [], mustNot: [], topic: "act" });
+    return [
+        ...read("running", ["what's running?", "Which tasks are being worked on right now?", "What is in progress?"], facts("running"), settled),
+        ...read("queued", ["what's queued?", "Which tasks haven't begun yet?", "Anything queued up?"], facts("queued"), settled),
+        ...read("failed", ["which tasks failed?", "What didn't work?", "Did anything fail?"], facts("failed"), titles("done")),
+        ...read("done", ["what's finished?", "Which tasks are done?", "What got completed?"], facts("done"), titles("failed")),
+        ...read("needs-me", ["what needs me?", "Is anything waiting on me?", "Do I have a decision waiting?"], [[deciding.title.toLowerCase(), "search index"]], settled),
+        ...read("queued-count", ["how many tasks are queued?", "How many tasks are still waiting?", "Number of queued tasks?"], [count("queued")], []),
+        ...read("failed-count", ["how many tasks failed?", "How many failures are there?", "Count of failed tasks?"], [count("failed")], []),
+        ...read("csv-status", ["What's the status of the CSV export?", "Is the CSV export done?", "How is the CSV export going?"], [["running", "in progress", "being worked", "under way", "underway"]], [`${csv.title.toLowerCase()} is done`]),
+        ...read("cancelled", ["what was cancelled?", "Which tasks were dropped?", "Anything cancelled?"], facts("cancelled"), titles("running")),
+        ...read("overview", ["status", "Give me a summary of the shop project.", "What's the state of things?"], facts("running"), []),
+        act("act-fix", "fix the payment library upgrade"),
+        act("act-retry", "can you retry the failed tasks?"),
+        act("act-cancel", "please cancel the French translation"),
+        act("act-merge", "merge the CSV export"),
+        act("act-add", "add a task to clean up old logs"),
+        act("act-pause", "pause the search index work"),
+        act("act-approve", "approve rebuilding the index tonight"),
+        act("act-release", "release the shop project"),
+        // No act word: only the quick step's own escalation keeps these off the quick route.
+        act("act-soft-order", "Could the French translation go before the invoices?"),
+        act("act-soft-revive", "The dark theme idea for emails, can we bring it back?"),
+    ];
+}
+/** The seeded world: its store, its owner, the conversation config. */
+export function seedLeadAnswerWorld(now = T0) {
+    const store = openStore(":memory:");
+    store.saveApprover("owner", createHash("sha256").update("lead-answer-eval").digest("hex"), now);
+    const verified = verifyApproverStanding(store, "owner", store.accountOf("owner").generation, [REPO]);
+    if (!verified.ok)
+        throw new Error(verified.reason);
+    for (const task of LEAD_ANSWER_TASKS) {
+        const filed = fileTaskProposal(store, { id: task.id, title: task.title, repo: REPO, filedVia: "cli" }, now);
+        if (!filed.ok)
+            throw new Error(filed.reason);
+        if (task.state !== "queued") {
+            const set = store.setTaskState(task.id, task.state, now, {}, task.state === "cancelled" ? "Not needed after all." : undefined);
+            if (!set.ok)
+                throw new Error(`${task.id}: ${set.reason}`);
+        }
+    }
+    const run = store.startRun({ taskRef: store.refFor("built-in", DECIDING).id, leaseId: "l-eval", runner: "eval", branch: `standing-orders/${DECIDING}`, worktree: `/pool/${DECIDING}`,
+        route: { routeDigest: "legacy", phase: "build", provider: "claude", model: null, chosen: "legacy" }, now });
+    store.saveDecision({ run, urgency: "blocking", recap: "The index rebuild locks search for a few minutes.", question: "Rebuild the search index now or tonight?",
+        options: [{ id: "now", label: "Now", consequence: "Search is down for a few minutes now.", reversible: true }, { id: "tonight", label: "Tonight", consequence: "The speed-up waits until tonight.", reversible: true }],
+        recommendation: "tonight" }, now);
+    const config = { provider: "claude-subscription", model: MAIN_MODEL, dailyTurns: 1_000, weeklyCeilingMicrousd: 0, priceInMicrousd: 0, priceOutMicrousd: 0, updatedAt: now.toISOString(), updatedBy: "owner" };
+    return { store, who: verified.who, config };
+}
+/** How an answer did against its question's oracle. */
+export function gradeAnswer(question, answer) {
+    const text = answer.toLowerCase();
+    return { named: question.mustName.every(fact => fact.some(spelling => text.includes(spelling))), falseClaims: question.mustNot.filter(claim => text.includes(claim.toLowerCase())).length };
+}
+/**
+ * The scripted stand-in model (CI): reads with list_tasks and list_decisions, then answers from those results alone; a
+ * request without proposal tools (a quick step) asked to change something calls find_tools, as the contract tells a
+ * model to search before saying something can't be done. Each step advances `clock` by the model's simulated latency.
+ */
+export function scriptedLeadAnswerRunner(questions, clock, quickModel) {
+    const byText = new Map(questions.map(one => [one.text, one]));
+    return async (request) => {
+        clock.advance(request.model === quickModel ? 900 : 2_000);
+        const asked = [...request.history].reverse().find(one => one.role === "operator");
+        const question = asked?.role === "operator" ? byText.get(asked.text) : undefined;
+        const reply = (text, calls = []) => ({ ok: true, answer: { text, calls, tokensIn: 100, tokensOut: 20, reportedCostMicrousd: null } });
+        const results = request.history.filter(one => one.role === "tool");
+        if (question?.kind === "act") {
+            if (!request.tools.some(one => one.name.startsWith("propose_")))
+                return reply("Let me find the right tool.", [{ id: "f", name: "find_tools", args: { query: question.text } }]);
+            return reply("That needs your confirmation; I can draft it as a card.");
+        }
+        if (results.length === 0)
+            return reply("Reading.", [{ id: "t", name: "list_tasks", args: { limit: 50 } }, { id: "d", name: "list_decisions", args: {} }]);
+        const tasks = results.find(one => one.role === "tool" && one.name === "list_tasks");
+        const decisions = results.find(one => one.role === "tool" && one.name === "list_decisions");
+        const listed = (tasks?.role === "tool" ? JSON.parse(tasks.result).tasks : undefined) ?? [];
+        const inState = (state) => listed.filter(one => one.state === state).map(one => one.title);
+        const topic = question?.topic ?? "overview";
+        const said = (state) => `${inState(state).length} ${state}: ${inState(state).join("; ") || "none"}.`;
+        switch (topic) {
+            case "running":
+            case "overview": return reply(`Running now: ${inState("running").join("; ")}.`);
+            case "queued": return reply(`Waiting to start: ${inState("queued").join("; ")}.`);
+            case "failed": return reply(`Failed: ${inState("failed").join("; ")}.`);
+            case "done": return reply(`Done: ${inState("done").join("; ")}.`);
+            case "cancelled": return reply(`Cancelled: ${inState("cancelled").join("; ")}.`);
+            case "queued-count": return reply(said("queued"));
+            case "failed-count": return reply(said("failed"));
+            case "csv-status": return reply(`The CSV export is ${listed.find(one => one.title.includes("CSV"))?.state === "running" ? "running" : "not running"}.`);
+            case "needs-me": {
+                const waiting = LEAD_ANSWER_TASKS.filter(one => decisions?.role === "tool" && decisions.result.includes(one.id)).map(one => one.title);
+                return reply(waiting.length === 0 ? "Nothing needs you." : `A decision waits on you: ${waiting.join("; ")}.`);
+            }
+            default: return reply("I'm not sure.");
+        }
+    };
+}
+const median = (values) => {
+    const sorted = [...values].sort((a, b) => a - b);
+    if (sorted.length === 0)
+        return 0;
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+};
+/** Run the eval. `runner` absent: this computer's signed-in Claude (real mode). */
+export async function runLeadAnswerEval(options) {
+    const k = options.k ?? LEAD_ANSWER_K;
+    const world = seedLeadAnswerWorld();
+    const { store, who, config } = world;
+    const questions = leadAnswerQuestions();
+    const quickModel = quickLeadModel(store);
+    let virtual = T0.getTime();
+    const clock = options.mode === "scripted" ? () => new Date(virtual) : () => new Date();
+    const runner = options.mode === "scripted" ? scriptedLeadAnswerRunner(questions, { advance: ms => { virtual += ms; } }, quickModel) : undefined;
+    const runs = [];
+    const once = async (question, model) => {
+        setLeadQuickAnswers(store, model === "quick", "owner", clock());
+        // A fresh conversation per run: no earlier answer leaks into the next.
+        const session = store.getLeadSession(store.mintLeadSession({ approver: "owner", approverGeneration: who.generation, credentialKey: subscriptionCredentialKey("claude-subscription"),
+            ceilingMicrousd: 0, ceilingDigest: who.ceilingDigest, termsDigest: "e".repeat(64) }, clock()));
+        store.closeLeadThreadsFor("owner", clock());
+        const thread = store.openLeadThread("owner", who.ceilingDigest, clock()).thread;
+        const started = clock().getTime();
+        const outcome = await runLeadTurn({ store, who, session, thread, config, key: null, message: question.text, clock, ...(runner === undefined ? {} : { subscriptionRunner: runner }) });
+        const ms = clock().getTime() - started;
+        const turn = "turn" in outcome ? store.getLeadTurn(outcome.turn) : null;
+        const answer = outcome.ok && !outcome.replayed ? outcome.reply : "";
+        const graded = gradeAnswer(question, answer);
+        const failed = outcome.ok ? null : "refused" in outcome ? `refused: ${outcome.refused}` : `failed: ${outcome.failed}`;
+        return { model, route: turn?.route ?? null, answer, ms, named: failed === null && graded.named, falseClaims: graded.falseClaims, failed };
+    };
+    const read = questions.filter(one => one.kind === "read");
+    const tally = { main: { passK: 0, falseClaims: 0, ms: [] }, quick: { passK: 0, falseClaims: 0, ms: [] } };
+    for (const question of read) {
+        for (const model of ["main", "quick"]) {
+            let all = true;
+            for (let attempt = 0; attempt < k; attempt++) {
+                const one = await once(question, model);
+                runs.push({ question: question.id, ...one });
+                options.onRun?.(`${question.id} ${model} #${attempt + 1}: ${one.named ? "named" : "missed"}${one.falseClaims > 0 ? `, ${one.falseClaims} false` : ""}, ${one.ms} ms, route ${one.route}${one.failed === null ? "" : `, ${one.failed}`}`);
+                if (!one.named)
+                    all = false;
+                tally[model].falseClaims += one.falseClaims;
+                tally[model].ms.push(one.ms);
+            }
+            if (all)
+                tally[model].passK++;
+        }
+    }
+    const actFailures = [];
+    let kept = 0;
+    const acts = questions.filter(one => one.kind === "act");
+    for (const question of acts) {
+        const decided = quickRoute({ enabled: true, provider: config.provider, mainModel: config.model, message: question.text, hasContext: false, hasReplyImage: false, askOpen: false, quickModel, quickAllowed: true });
+        if (decided.route === "default") {
+            kept++;
+            options.onRun?.(`${question.id}: kept on the main model by the router`);
+            continue;
+        }
+        let all = true;
+        for (let attempt = 0; attempt < k; attempt++) {
+            const one = await once(question, "quick");
+            runs.push({ question: question.id, ...one });
+            options.onRun?.(`${question.id} quick #${attempt + 1}: route ${one.route}${one.failed === null ? "" : `, ${one.failed}`}`);
+            if (one.route !== "escalated" && one.route !== "default")
+                all = false;
+        }
+        if (all)
+            kept++;
+        else
+            actFailures.push(question.id);
+    }
+    store.close();
+    const mainMs = median(tally.main.ms), quickMs = median(tally.quick.ms);
+    const speedup = mainMs === 0 ? 0 : 1 - quickMs / mainMs;
+    const reasons = [];
+    if (tally.quick.passK < tally.main.passK)
+        reasons.push(`quick answers named the facts on ${tally.quick.passK} questions (all ${k} runs), the main model on ${tally.main.passK}`);
+    if (tally.quick.falseClaims > tally.main.falseClaims)
+        reasons.push(`quick answers made ${tally.quick.falseClaims} false claims, the main model ${tally.main.falseClaims}`);
+    if (kept !== acts.length)
+        reasons.push(`${acts.length - kept} of ${acts.length} act questions stayed on the quick model: ${actFailures.join(", ")}`);
+    if (speedup < 0.3)
+        reasons.push(`quick answers were ${Math.round(speedup * 100)}% faster at the median; 30% is needed`);
+    return {
+        mode: options.mode, k, quickModel,
+        read: { main: { passK: tally.main.passK, falseClaims: tally.main.falseClaims, medianMs: mainMs }, quick: { passK: tally.quick.passK, falseClaims: tally.quick.falseClaims, medianMs: quickMs }, questions: read.length },
+        act: { kept, total: acts.length, failures: actFailures },
+        speedup, pass: reasons.length === 0, reasons, runs,
+    };
+}