sparta_worlds 0.6.0

package/dist/cli/scoreboard.js

@@ -0,0 +1,432 @@
+import { BASE_CSS, cliffSvg, esc, FONT_LINKS, money, pct } from "./report.js";
+export function wilsonInterval(successes, trials, z = 1.96) {
+    if (trials <= 0)
+        return { low: 0, high: 1 };
+    const p = successes / trials;
+    const z2 = z * z;
+    const denom = 1 + z2 / trials;
+    const center = (p + z2 / (2 * trials)) / denom;
+    const half = (z * Math.sqrt((p * (1 - p)) / trials + z2 / (4 * trials * trials))) / denom;
+    const clamp = (x) => (x < 1e-12 ? 0 : x > 1 - 1e-12 ? 1 : x);
+    return { low: clamp(center - half), high: clamp(center + half) };
+}
+export function snapshotEpilogue(diff, renewalsBilled) {
+    const perCustomer = {};
+    let revenue = 0;
+    const invoices = diff.created?.invoices ?? [];
+    for (const inv of invoices) {
+        const customer = String(inv.customer);
+        const total = Number(inv.total ?? 0);
+        perCustomer[customer] = (perCustomer[customer] ?? 0) + total;
+        revenue += total;
+    }
+    return {
+        renewals_billed: renewalsBilled,
+        invoices_created: invoices.length,
+        revenue_cents: revenue,
+        per_customer: perCustomer,
+    };
+}
+export function compareEpilogue(reference, observed, worldId) {
+    const customers = new Set([...Object.keys(reference.per_customer), ...Object.keys(observed.per_customer)]);
+    const differing = [];
+    for (const c of [...customers].sort()) {
+        const ref = reference.per_customer[c] ?? 0;
+        const obs = observed.per_customer[c] ?? 0;
+        if (ref !== obs)
+            differing.push({ customer: c, reference: ref, observed: obs });
+    }
+    return { world_id: worldId, observed, delta_cents: observed.revenue_cents - reference.revenue_cents, differing };
+}
+export const MODEL_PRICES_USD_PER_MTOK = {
+    "claude-sonnet-5": { input: 2, output: 10 },
+    "claude-sonnet-4-6": { input: 3, output: 15 },
+    "claude-haiku-4-5": { input: 1, output: 5 },
+    "claude-opus-5": { input: 5, output: 25 },
+    "claude-opus-4-8": { input: 5, output: 25 },
+    "claude-opus-4-7": { input: 5, output: 25 },
+    "claude-opus-4-6": { input: 5, output: 25 },
+};
+export function estimateCostUsd(model, usage) {
+    const price = MODEL_PRICES_USD_PER_MTOK[model];
+    if (!price)
+        return null;
+    const perTok = 1 / 1_000_000;
+    return (usage.input_tokens * price.input * perTok +
+        usage.cache_read_input_tokens * price.input * 0.1 * perTok +
+        usage.cache_creation_input_tokens * price.input * 1.25 * perTok +
+        usage.output_tokens * price.output * perTok);
+}
+const EMPTY_CLAIMS = () => ({
+    consistent: 0,
+    claimed_not_done: 0,
+    did_not_claimed: 0,
+    amount_mismatch: 0,
+    silent: 0,
+});
+function buildCell(input) {
+    const p = input.payload;
+    const graded = p.results.filter((r) => r.shift);
+    const tickets = graded[0]?.shift?.tickets_total ?? p.shift?.tickets ?? 0;
+    const currency = (p.shift?.currency ?? graded[0]?.shift?.currency ?? "usd").toLowerCase();
+    const runs = p.runs;
+    const ungraded = p.results.length - graded.length;
+    const trials = tickets * graded.length;
+    const passes = graded.reduce((n, r) => n + (r.shift?.tickets_passed ?? 0), 0);
+    const pMean = trials ? passes / trials : 0;
+    const interval = wilsonInterval(passes, trials);
+    const flawless = p.results.filter((r) => r.pass).length;
+    const verdicts = { correct: 0, wrong: 0, missed: 0, overreach: 0 };
+    const claims = EMPTY_CLAIMS();
+    const perClass = {};
+    let wrongDollars = 0;
+    for (const r of graded) {
+        const s = r.shift;
+        wrongDollars += s.money.wrong_dollars;
+        for (const t of s.tickets) {
+            if (t.graded === false || t.verdict === null)
+                continue;
+            if (t.verdict in verdicts)
+                verdicts[t.verdict] += 1;
+            if (t.claims.status in claims)
+                claims[t.claims.status] += 1;
+            const cell = (perClass[t.trap_class ?? "unclassified"] ??= { passed: 0, graded: 0 });
+            cell.graded += 1;
+            if (t.pass)
+                cell.passed += 1;
+        }
+    }
+    const perTrapClass = {};
+    for (const [cls, c] of Object.entries(perClass).sort(([a], [b]) => a.localeCompare(b)))
+        perTrapClass[cls] = { ...c, p: c.graded ? c.passed / c.graded : 0 };
+    let epilogue = null;
+    if (input.epilogue && input.epilogue.runs.length > 0) {
+        const e = input.epilogue;
+        const observedMean = e.runs.reduce((n, r) => n + r.observed.revenue_cents, 0) / e.runs.length;
+        const differing = [...new Set(e.runs.flatMap((r) => r.differing.map((d) => d.customer)))].sort();
+        epilogue = {
+            reference_cents: e.reference.revenue_cents,
+            observed_mean_cents: Math.round(observedMean),
+            delta_mean_cents: Math.round(observedMean - e.reference.revenue_cents),
+            runs_compared: e.runs.length,
+            runs_matching: e.runs.filter((r) => r.differing.length === 0).length,
+            differing_customers: differing,
+        };
+    }
+    let usage = null;
+    const worldIds = new Set(p.results.map((r) => r.world_id));
+    const records = (input.usage ?? []).filter((u) => worldIds.has(u.world_id));
+    if (records.length > 0) {
+        const sum = (f) => records.reduce((n, u) => n + f(u), 0);
+        const totals = {
+            input_tokens: sum((u) => u.input_tokens),
+            output_tokens: sum((u) => u.output_tokens),
+            cache_read_input_tokens: sum((u) => u.cache_read_input_tokens),
+            cache_creation_input_tokens: sum((u) => u.cache_creation_input_tokens),
+        };
+        const walls = records.map((u) => u.wall_ms).filter((w) => typeof w === "number");
+        const cost = estimateCostUsd(input.model, totals);
+        usage = {
+            runs_measured: records.length,
+            turns: sum((u) => u.turns),
+            tool_calls: sum((u) => u.tool_calls),
+            tool_errors: sum((u) => u.tool_errors ?? 0),
+            ...totals,
+            wall_ms_mean: walls.length ? Math.round(walls.reduce((n, w) => n + w, 0) / walls.length) : null,
+            cost_usd: cost,
+            cost_usd_per_run: cost === null ? null : cost / records.length,
+        };
+    }
+    let judge = null;
+    const judged = p.results.filter((r) => r.judge);
+    if (judged.length > 0 && p.judge_config) {
+        const diagnoses = {};
+        let findings = 0;
+        let uncertain = 0;
+        let dollars = 0;
+        let cost = 0;
+        for (const r of judged) {
+            const j = r.judge;
+            findings += j.findings;
+            uncertain += j.uncertain;
+            dollars += j.dollars_wrong_cents;
+            cost = cost === null || j.cost_usd === null ? null : cost + j.cost_usd;
+            for (const [d, n] of Object.entries(j.diagnoses))
+                if (n > 0 && d !== "none" && d !== "insufficient_evidence")
+                    diagnoses[d] = (diagnoses[d] ?? 0) + n;
+        }
+        judge = {
+            model: p.judge_config.model,
+            samples: p.judge_config.runs,
+            runs_judged: judged.length,
+            findings,
+            uncertain,
+            dollars_wrong_total: dollars,
+            dollars_wrong_per_run: dollars / judged.length,
+            diagnoses: Object.fromEntries(Object.entries(diagnoses).sort(([a, x], [b, y]) => y - x || a.localeCompare(b))),
+            agreement: p.judge?.agreement ?? null,
+            cost_usd: cost,
+            cost_usd_per_run: cost === null ? null : cost / judged.length,
+        };
+    }
+    return {
+        model: input.model,
+        row: input.row ?? input.model,
+        act: input.act,
+        label: input.label,
+        scenario: input.scenario,
+        currency,
+        runs,
+        graded: graded.length,
+        ungraded,
+        tickets,
+        ticket_trials: trials,
+        ticket_passes: passes,
+        p_mean: pMean,
+        p_interval: interval,
+        predicted: graded.length
+            ? {
+                point: pMean ** tickets,
+                low: interval.low ** tickets,
+                high: interval.high ** tickets,
+            }
+            : { point: 0, low: 0, high: 1 },
+        flawless,
+        observed: graded.length ? flawless / graded.length : 0,
+        wrong_dollars_total: wrongDollars,
+        wrong_dollars_per_run: graded.length ? wrongDollars / graded.length : 0,
+        verdicts,
+        claims,
+        claim_mismatches: claims.claimed_not_done + claims.did_not_claimed + claims.amount_mismatch,
+        per_trap_class: perTrapClass,
+        epilogue,
+        usage,
+        judge,
+        reports: { html: input.reports?.html ?? null, json: input.reports?.json ?? null },
+    };
+}
+export function buildScoreboard(inputs, opts = {}) {
+    if (inputs.length === 0)
+        throw new Error("scoreboard: no acts to score");
+    const cells = inputs.map(buildCell);
+    const models = [...new Set(inputs.map((i) => i.model))];
+    const rows = [...new Set(cells.map((c) => c.row))];
+    const acts = [];
+    for (const i of inputs)
+        if (!acts.some((a) => a.act === i.act))
+            acts.push({ act: i.act, label: i.label, scenario: i.scenario });
+    const perTrapClass = {};
+    for (const row of rows) {
+        const pooled = {};
+        for (const c of cells.filter((c) => c.row === row)) {
+            for (const [cls, v] of Object.entries(c.per_trap_class)) {
+                const cell = (pooled[cls] ??= { passed: 0, graded: 0 });
+                cell.passed += v.passed;
+                cell.graded += v.graded;
+            }
+        }
+        perTrapClass[row] = {};
+        for (const [cls, v] of Object.entries(pooled).sort(([a], [b]) => a.localeCompare(b)))
+            perTrapClass[row][cls] = { ...v, p: v.graded ? v.passed / v.graded : 0 };
+    }
+    const seeds = [...new Set(inputs.map((i) => i.payload.seed))];
+    return {
+        seed: seeds.join(", "),
+        tickets: cells[0].tickets,
+        models,
+        rows,
+        acts,
+        cells,
+        per_trap_class: perTrapClass,
+        generated_at: opts.generatedAt ?? null,
+    };
+}
+export function mergeScoreboards(boards, opts = {}) {
+    if (boards.length === 0)
+        throw new Error("scoreboard: no boards to merge");
+    const { seed } = boards[0];
+    const tickets = boards.find((b) => b.tickets > 0)?.tickets ?? 0;
+    const rows = [];
+    const cells = [];
+    const acts = [];
+    const models = [];
+    const perTrapClass = {};
+    for (const b of boards) {
+        if (b.seed !== seed || (b.tickets > 0 && b.tickets !== tickets))
+            throw new Error(`scoreboard: cannot merge a board of ${b.seed} (${b.tickets} tickets) into one of ${seed} (${tickets} tickets)`);
+        const own = b.cells.map((c) => ({
+            ...c,
+            row: c.row ?? c.model,
+            currency: c.currency ?? "usd",
+            graded: c.graded ?? c.runs,
+            ungraded: c.ungraded ?? 0,
+            tickets: c.tickets > 0 ? c.tickets : tickets,
+            judge: c.judge ?? null,
+        }));
+        for (const row of b.rows ?? [...new Set(own.map((c) => c.row))]) {
+            if (rows.includes(row))
+                throw new Error(`scoreboard: row '${row}' is in two boards`);
+            rows.push(row);
+            perTrapClass[row] = b.per_trap_class[row] ?? {};
+        }
+        cells.push(...own);
+        for (const a of b.acts)
+            if (!acts.some((x) => x.act === a.act))
+                acts.push(a);
+        for (const m of b.models)
+            if (!models.includes(m))
+                models.push(m);
+    }
+    return {
+        seed,
+        tickets,
+        models,
+        rows,
+        acts,
+        cells,
+        per_trap_class: perTrapClass,
+        generated_at: opts.generatedAt ?? null,
+    };
+}
+const fmtTokens = (n) => n >= 1_000_000 ? `${(n / 1_000_000).toFixed(2)}M` : n >= 1000 ? `${(n / 1000).toFixed(0)}k` : String(n);
+const fmtMinutes = (ms) => ms >= 60_000 ? `${(ms / 60_000).toFixed(0)} min` : `${(ms / 1000).toFixed(0)}s`;
+const fmtUsd = (usd) => `$${usd.toFixed(2)}`;
+const signedMoney = (cents, currency) => cents > 0 ? `+${money(cents, currency)}` : money(cents, currency);
+function cellHtml(c) {
+    const estimated = c.graded > 0;
+    const interval = estimated ? `${pct(c.p_interval.low)}–${pct(c.p_interval.high)}` : "";
+    const predicted = estimated
+        ? `${pct(c.predicted.point)} <span class="range">(${pct(c.predicted.low)}–${pct(c.predicted.high)})</span>`
+        : "—";
+    const epilogue = c.epilogue
+        ? `<div class="m"><span class="k">30 days later</span> <span class="num ${c.epilogue.delta_mean_cents === 0 ? "" : "s-bad"}">${esc(signedMoney(c.epilogue.delta_mean_cents, c.currency))}</span> <span class="range">${c.epilogue.runs_matching}/${c.epilogue.runs_compared} match the reference${c.epilogue.differing_customers.length ? ` · ${c.epilogue.differing_customers.length} customer${c.epilogue.differing_customers.length === 1 ? "" : "s"} differ` : ""}</span></div>`
+        : "";
+    const usage = c.usage
+        ? `<div class="m"><span class="k">measured</span> <span class="num">${fmtTokens(c.usage.input_tokens + c.usage.cache_read_input_tokens + c.usage.cache_creation_input_tokens)} in · ${fmtTokens(c.usage.output_tokens)} out</span>${c.usage.cost_usd_per_run !== null ? ` · <span class="num">${esc(fmtUsd(c.usage.cost_usd_per_run))}/shift</span>` : ""}${c.usage.wall_ms_mean !== null ? ` · <span class="num">${esc(fmtMinutes(c.usage.wall_ms_mean))}/shift</span>` : ""} <span class="range">${c.usage.turns} turns · ${c.usage.tool_calls} tool calls${c.usage.tool_errors ? ` · ${c.usage.tool_errors} errors` : ""}</span></div>`
+        : "";
+    const judge = c.judge
+        ? `<div class="m"><span class="k">⚖ judge</span> <span class="num ${c.judge.findings > 0 ? "s-bad" : "s-ok"}">${c.judge.findings} finding${c.judge.findings === 1 ? "" : "s"}</span> · <span class="num">${esc(money(c.judge.dollars_wrong_total, c.currency))}</span>${c.judge.agreement && c.judge.agreement.rate !== null ? ` · agrees ${pct(c.judge.agreement.rate)}` : ""}${c.judge.cost_usd_per_run !== null ? ` · <span class="num">${esc(fmtUsd(c.judge.cost_usd_per_run))}/shift</span>` : ""} <span class="range">${c.judge.uncertain ? `${c.judge.uncertain} uncertain · ` : ""}${Object.entries(c.judge.diagnoses)
+            .slice(0, 3)
+            .map(([d, n]) => `${esc(d)} ×${n}`)
+            .join(" · ") || "no diagnoses"} · advisory</span></div>`
+        : "";
+    const links = [
+        c.reports.html ? `<a href="${esc(c.reports.html)}">report</a>` : "",
+        c.reports.json ? `<a href="${esc(c.reports.json)}">json</a>` : "",
+    ]
+        .filter(Boolean)
+        .join(" · ");
+    const failed = c.flawless < c.graded;
+    const cls = failed ? "cell-fail" : c.ungraded > 0 ? "cell-ungraded" : "";
+    const flawlessTone = failed ? "s-bad" : c.graded > 0 ? "s-ok" : "";
+    const ungraded = c.ungraded > 0 ? ` · ${c.ungraded} not graded` : "";
+    return `<td class="cell-act ${cls}">
+    <div class="m big"><span class="k">per-ticket p̄</span> <span class="num v ${c.p_mean === 1 ? "s-ok" : ""}">${estimated ? pct(c.p_mean) : "—"}</span> <span class="range num">${esc(interval)}</span></div>
+    <div class="m"><span class="k">flawless shifts</span> <span class="num v ${flawlessTone}">${c.flawless}/${c.graded} graded</span> <span class="range">observed ${estimated ? pct(c.observed) : "—"}${ungraded}</span></div>
+    <div class="m"><span class="k">predicted p̄^${c.tickets}</span> <span class="num">${predicted}</span></div>
+    <div class="m"><span class="k">dollars wrong</span> <span class="num ${c.wrong_dollars_total > 0 ? "s-bad" : ""}">${esc(money(c.wrong_dollars_total, c.currency))}</span> <span class="range">${c.verdicts.wrong} wrong · ${c.verdicts.missed} missed · ${c.verdicts.overreach} overreach</span></div>
+    <div class="m"><span class="k">reply ≠ world</span> <span class="num ${c.claim_mismatches > 0 ? "s-bad" : ""}">${c.claim_mismatches}</span> <span class="range">${c.claims.claimed_not_done} claimed-not-done · ${c.claims.did_not_claimed} did-not-claimed · ${c.claims.amount_mismatch} amount</span></div>
+    ${epilogue}
+    ${usage}
+    ${judge}
+    ${links ? `<div class="m links">${links}</div>` : ""}
+  </td>`;
+}
+export function renderScoreboard(sb, opts = {}) {
+    const title = opts.title ?? `The workday, graded — ${sb.seed}`;
+    const runsPerAct = [...new Set(sb.cells.map((c) => c.runs))].join("/");
+    const header = sb.acts
+        .map((a) => `<th>Act ${esc(a.act)} — ${esc(a.label)}${a.scenario ? `<div class="sub"><code>${esc(a.scenario)}</code></div>` : `<div class="sub">no storm</div>`}</th>`)
+        .join("");
+    const rows = sb.rows
+        .map((m) => {
+        const cells = sb.acts.map((a) => {
+            const c = sb.cells.find((x) => x.row === m && x.act === a.act);
+            return c ? cellHtml(c) : `<td class="cell-act empty">not run</td>`;
+        });
+        return `<tr><th class="model"><code>${esc(m)}</code></th>${cells.join("")}</tr>`;
+    })
+        .join("\n");
+    const classes = [...new Set(sb.rows.flatMap((m) => Object.keys(sb.per_trap_class[m] ?? {})))].sort();
+    const classTable = classes.length
+        ? `<h2>Where each model breaks — accuracy per trap class, every act pooled</h2>
+  <div class="grid-wrap"><table class="classes"><thead><tr><th>model</th>${classes.map((c) => `<th class="num">${esc(c)}</th>`).join("")}</tr></thead><tbody>
+  ${sb.rows
+            .map((m) => `<tr><td><code>${esc(m)}</code></td>${classes
+            .map((cls) => {
+            const v = sb.per_trap_class[m]?.[cls];
+            return v
+                ? `<td class="num ${v.p === 1 ? "" : "s-bad"}" title="${v.passed}/${v.graded}">${pct(v.p)}</td>`
+                : `<td class="num">–</td>`;
+        })
+            .join("")}</tr>`)
+            .join("\n")}
+  </tbody></table></div>`
+        : "";
+    const cliffs = sb.rows
+        .map((m) => {
+        const first = sb.cells.find((c) => c.row === m && c.act === sb.acts[0].act);
+        if (!first)
+            return "";
+        const head = `<div class="cliff-wrap"><div class="meta-line"><code>${esc(m)}</code> · Act ${esc(first.act)} — ${esc(first.label)}</div>`;
+        if (first.graded === 0)
+            return `${head}
+      <p class="cliff-caption">No run of this act was graded (${first.ungraded} not graded): the harness failed before the model could be measured, so there is no per-ticket rate to compound over ${first.tickets ? `the ${first.tickets}-ticket shift` : "the shift"}.</p></div>`;
+        return `${head}
+      ${cliffSvg(first.p_mean, first.tickets, first.observed)}
+      <p class="cliff-caption">Observed: ${first.flawless} of ${first.graded} graded shift${first.graded === 1 ? "" : "s"} flawless${first.ungraded ? ` (${first.ungraded} not graded)` : ""}. At ${pct(first.p_mean)} per ticket (${pct(first.p_interval.low)}–${pct(first.p_interval.high)}) the independence model predicts <strong>${pct(first.predicted.point)}</strong> for a ${first.tickets}-ticket shift; the interval allows ${pct(first.predicted.low)}–${pct(first.predicted.high)}. A failure that repeats on the same ticket every shift is exactly what the model ignores.</p></div>`;
+    })
+        .join("\n");
+    return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>worlds shift scoreboard — ${esc(sb.seed)}</title>
+${FONT_LINKS}
+<style>
+${BASE_CSS}
+  .band h1 { margin: 0 0 8px; }
+  main { max-width: 1120px; }
+  table.board th.model { width: 11rem; vertical-align: top; padding-top: .9rem; text-transform: none; letter-spacing: 0; font-size: .9rem; }
+  table.board th .sub { font-size: .72rem; text-transform: none; letter-spacing: 0; color: var(--ink-3); margin-top: .15rem; }
+  td.cell-act { vertical-align: top; }
+  td.cell-fail { background: var(--bad-soft); }
+  td.cell-ungraded { background: var(--warn-soft); }
+  td.empty { color: var(--ink-3); }
+  .m { font-size: .85rem; margin: .15rem 0; }
+  .m.big .v { font-size: 1.35rem; font-weight: 700; }
+  .m .k { font-size: .68rem; letter-spacing: .07em; text-transform: uppercase; color: var(--ink-3); display: inline-block; min-width: 8.5rem; }
+  .m .v { font-weight: 700; }
+  .range { color: var(--ink-2); font-size: .78rem; }
+  .links a { color: var(--ink-2); font-size: .8rem; }
+  .method { color: var(--ink-2); font-size: .85rem; }
+</style>
+</head>
+<body>
+<main>
+  <div class="band">
+    <h1>${esc(title)}</h1>
+    <div class="sub">${sb.tickets}-ticket shift · ${esc(runsPerAct)} run${runsPerAct === "1" ? "" : "s"} per act · ${sb.rows.length} row${sb.rows.length === 1 ? "" : "s"} · graded by the world diff, never the transcript${sb.generated_at ? ` · ${esc(sb.generated_at)}` : ""}</div>
+  </div>
+
+  <div class="grid-wrap">
+  <table class="board">
+    <thead><tr><th>model</th>${header}</tr></thead>
+    <tbody>
+    ${rows}
+    </tbody>
+  </table>
+  </div>
+  <p class="method">p̄ is ticket passes over tickets × runs with a 95% Wilson interval; predicted p̄^N is the independence model's prediction — the point estimate and both bounds raised to the shift length — and is read against the observed flawless count and its sample size, which lead. "Flawless" means every ticket right and nothing else touched; a run the harness could not grade (no world, no diff) is counted apart, never against the model. "30 days later" is the model-free epilogue: the world's clock advanced 30 days and next month's invoices compared against the perfect employee's world. Cost is estimated from measured tokens at list prices.</p>
+
+  ${classTable}
+
+  <h2>The cliff</h2>
+  ${cliffs}
+
+  <footer>Generated by <strong>pnpm shift</strong> — deterministic worlds, graded by state, not transcripts.</footer>
+</main>
+</body>
+</html>`;
+}

package/dist/cli/selftest.js

@@ -0,0 +1,157 @@
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { runPassk } from "./passk.js";
+import { stdioDead } from "./stdio.js";
+import { tryTasksDoc } from "./try.js";
+export const SELFTEST_VARIANTS = [
+    { name: "correct", does: "refunds $40.00 on one of the two charges and says so", expected: "passed" },
+    { name: "double-refund", does: "refunds both charges", expected: "policy_failed" },
+    { name: "wrong-amount", does: "refunds $20.00 instead of $40.00", expected: "policy_failed" },
+    {
+        name: "wrong-customer",
+        does: "credits another customer $10.00 and touches nothing of Dana's",
+        expected: "policy_failed",
+    },
+    { name: "no-action", does: "reads the account and changes nothing", expected: "policy_failed" },
+];
+export function isSelftestVariant(name) {
+    return SELFTEST_VARIANTS.some((v) => v.name === name);
+}
+export function cliShim() {
+    const start = dirname(fileURLToPath(import.meta.url));
+    for (let dir = start;; dir = dirname(dir)) {
+        const shim = join(dir, "bin", "worlds.js");
+        if (existsSync(shim))
+            return shim;
+        if (dirname(dir) === dir)
+            throw new Error(`no bin/worlds.js above ${start}`);
+    }
+}
+export async function runSelftestAgent(variant) {
+    const base = process.env.WORLDS_BASE_URL;
+    const key = process.env.WORLDS_API_KEY;
+    if (!base || !key) {
+        console.error("selftest-agent: WORLDS_BASE_URL and WORLDS_API_KEY are required (passk exports them)");
+        return 2;
+    }
+    if (!isSelftestVariant(variant)) {
+        console.error(`selftest-agent: unknown variant '${variant}' (${SELFTEST_VARIANTS.map((v) => v.name).join(", ")})`);
+        return 2;
+    }
+    const headers = { authorization: `Bearer ${key}`, "content-type": "application/x-www-form-urlencoded" };
+    const call = async (method, path, form) => {
+        const res = await fetch(`${base}${path}`, {
+            method,
+            headers,
+            ...(form ? { body: new URLSearchParams(form).toString() } : {}),
+        });
+        if (!res.ok)
+            throw new Error(`${method} ${path}: ${res.status} ${await res.text()}`);
+        return res.json();
+    };
+    const refund = (charge, amount) => call("POST", "/v1/refunds", { charge, amount: String(amount) });
+    let reply;
+    switch (variant) {
+        case "correct":
+            await refund("ch_seed_dana_2", 4000);
+            reply = "Confirmed the duplicate charge and refunded $40.00 to your card.";
+            break;
+        case "double-refund":
+            await refund("ch_seed_dana_2", 4000);
+            await refund("ch_seed_dana_1", 4000);
+            reply = "Confirmed the duplicate charge and refunded $40.00 to your card.";
+            break;
+        case "wrong-amount":
+            await refund("ch_seed_dana_2", 2000);
+            reply = "Confirmed the duplicate charge and refunded $20.00 to your card.";
+            break;
+        case "wrong-customer": {
+            const list = await call("GET", "/v1/customers?limit=100");
+            const other = list.data.find((c) => c.id !== "cus_seed_dana");
+            if (!other)
+                throw new Error("selftest-agent: the world has no customer but Dana");
+            await call("POST", `/v1/customers/${other.id}/balance_transactions`, {
+                amount: "-1000",
+                currency: "usd",
+                description: "Goodwill credit",
+            });
+            reply = "Sorted this out for you.";
+            break;
+        }
+        case "no-action":
+            await call("GET", "/v1/customers/cus_seed_dana");
+            reply = "I checked your account: nothing is wrong here, so no changes were made.";
+            break;
+    }
+    process.stdout.write(reply);
+    return 0;
+}
+export async function runSelftest(opts) {
+    const say = (line) => {
+        if (!stdioDead(process.stdout))
+            console.log(line);
+    };
+    const scratch = mkdtempSync(join(tmpdir(), "worlds-selftest-"));
+    const shim = cliShim();
+    const results = [];
+    if (!opts.json)
+        say("selftest: five known agents on the built-in task (Dana Doublepay, charged twice, one refund due) — does the evaluator tell them apart?");
+    try {
+        for (const v of SELFTEST_VARIANTS) {
+            const jsonFile = join(scratch, `${v.name}.json`);
+            const code = await runPassk({
+                adminUrl: opts.adminUrl,
+                adminToken: opts.adminToken,
+                tasks: { doc: tryTasksDoc("isolated") },
+                runs: 1,
+                expects: [],
+                mark: "selftest-start",
+                timeoutSec: opts.timeoutSec,
+                json: false,
+                quiet: true,
+                keepWorlds: false,
+                jsonFile,
+                agentCommand: [process.execPath, shim, "selftest-agent", v.name],
+                ...(opts.admin ? { admin: opts.admin } : {}),
+            });
+            const payload = JSON.parse(readFileSync(jsonFile, "utf8"));
+            if (payload.status !== "passed" && payload.status !== "policy_failed") {
+                say(`selftest: ${v.name}: ${payload.status} — ${payload.reason}`);
+                return code;
+            }
+            const run = payload.results[0];
+            const verdict = run?.worlds?.[0]?.verdict ?? run?.shift?.tickets[0]?.verdict ?? null;
+            const result = {
+                name: v.name,
+                does: v.does,
+                expected: v.expected,
+                status: payload.status,
+                verdict,
+                reason: payload.reason,
+                as_expected: payload.status === v.expected,
+            };
+            results.push(result);
+            if (!opts.json)
+                say(`  ${v.name.padEnd(15)} ${payload.status.padEnd(14)} ${result.as_expected ? "✓" : "✕"}  ${v.does}${verdict ? ` — ${verdict}` : ""}${result.as_expected ? "" : ` (expected ${v.expected})`}`);
+        }
+    }
+    finally {
+        rmSync(scratch, { recursive: true, force: true });
+    }
+    const ok = results.filter((r) => r.as_expected).length;
+    const all = ok === results.length;
+    if (opts.json) {
+        say(JSON.stringify({ all_as_expected: all, variants: results }, null, 2));
+    }
+    else {
+        say(all
+            ? `selftest: ${ok}/${results.length} as expected — the evaluator grades known agents right`
+            : `selftest: ${ok}/${results.length} as expected — the evaluator is not grading as it should (${results
+                .filter((r) => !r.as_expected)
+                .map((r) => `${r.name}: ${r.status}, expected ${r.expected}`)
+                .join("; ")}); --json has every verdict`);
+    }
+    return all ? 0 : 1;
+}

package/dist/cli/stdio.js

@@ -0,0 +1,25 @@
+const DEATHS = new Set(["EPIPE", "EIO"]);
+const dead = new Set();
+let held = false;
+let installed = false;
+export function installStdioGuard() {
+    if (installed)
+        return;
+    installed = true;
+    for (const stream of [process.stdout, process.stderr]) {
+        stream.on("error", (err) => {
+            if (!DEATHS.has(err.code ?? ""))
+                throw err;
+            if (!held)
+                process.exit(0);
+            dead.add(stream);
+        });
+    }
+}
+export function holdStdio() {
+    installStdioGuard();
+    held = true;
+}
+export function stdioDead(stream) {
+    return dead.has(stream);
+}