wicked-crew 0.16.4 → 0.16.6

package/dist/evals/routes.js

@@ -0,0 +1,121 @@
+/**
+ * `/api/v1/testing/default-evals*` (DES-DEFAULT-EVALS-001 §7): run the default evals over this daemon's own run
+ * store, per project or across projects, and read the history.
+ *
+ *   GET  /testing/default-evals/suites  what each suite checks, and whether crew can replay it
+ *   POST /testing/default-evals/run     {project?: string | "all", suites?: SuiteId[], baseline?: boolean}
+ *   GET  /testing/default-evals         the history, newest first
+ *   GET  /testing/default-evals/:id     one run's report and its cases
+ *
+ * Everything is read from the run store (sessions, events, decisions, QE ledgers, the latest steering eval); the
+ * only write is the report itself, under `<home>/evals/default/`.
+ */
+import { randomUUID } from 'node:crypto';
+import { z } from 'zod';
+import { API_PREFIX } from '../api/api-prefix.js';
+import { readRunVerdicts, runQeLedgerRoot } from '../qe/ledger.js';
+import { captureAll } from './capture.js';
+import { runDefaultEvals, SUITE_TITLES } from './runner.js';
+import { needsYouSeed } from './seed-needs-you.js';
+import { RIG_SEED } from './seed-rig.js';
+import { steeringCasesFrom } from './steering.js';
+import { syntheticSuites } from './synthetic.js';
+import { SUITE_IDS } from './types.js';
+const V = API_PREFIX;
+const RunBody = z
+    .object({
+    project: z.string().trim().min(1).max(200).optional(),
+    suites: z.array(z.enum(SUITE_IDS)).min(1).optional(),
+    baseline: z.boolean().optional(),
+})
+    .strict();
+/** The cases this daemon's own store yields right now (and the baseline, when asked for). */
+export async function gatherCases(deps, opts) {
+    const runs = await deps.runs();
+    const events = new Map();
+    const qeVerdicts = [];
+    for (const run of runs) {
+        const id = typeof run.session['id'] === 'string' ? run.session['id'] : null;
+        if (id === null)
+            continue;
+        events.set(id, (await deps.events(id)) ?? []);
+        const root = runQeLedgerRoot(run.session, null);
+        if (root === null)
+            continue;
+        try {
+            for (const v of readRunVerdicts(root, id))
+                qeVerdicts.push({ runId: id, ...v });
+        }
+        catch {
+            // an unreadable ledger is that run's QE cases missing, never the eval failing
+        }
+    }
+    const input = { runs, events: (id) => events.get(id) ?? [], decisions: deps.decisions(), qeVerdicts };
+    const live = captureAll(input);
+    let steering = [];
+    if (deps.steeringEvals !== undefined) {
+        const latest = (await deps.steeringEvals.list())[0];
+        if (latest !== undefined)
+            steering = steeringCasesFrom(await deps.steeringEvals.get(latest.id));
+    }
+    const baseline = opts.baseline ? [...RIG_SEED, ...needsYouSeed()] : [];
+    // Live cases come last: a live capture of a baseline case replaces it (same id), so nothing counts twice.
+    const liveIds = new Set(live.map((c) => c.id));
+    return [...baseline.filter((c) => !liveIds.has(c.id)), ...live, ...steering];
+}
+/** Run the default evals over this daemon's store and keep the report: the one path the route, the `run-evals`
+ *  workflow's launch and the schedule share. */
+export async function runAndRecord(deps, opts) {
+    const cases = await gatherCases(deps, { baseline: opts.baseline });
+    const { report, cases: scored } = await runDefaultEvals({
+        id: opts.id,
+        now: opts.now,
+        project: opts.project,
+        cases,
+        replay: deps.replay,
+        previous: deps.store !== undefined ? await deps.store.previous(opts.project) : {},
+        ...(opts.suites !== undefined ? { suites: opts.suites } : {}),
+    });
+    if (deps.store !== undefined)
+        await deps.store.record({ report, cases: scored });
+    return report;
+}
+export function registerDefaultEvalRoutes(app, deps) {
+    const now = deps.now ?? (() => new Date());
+    const newId = deps.newId ?? (() => randomUUID());
+    app.get(`${V}/testing/default-evals/suites`, { config: { manifest: { responseType: 'DefaultEvalSuitesResponse', statusCodes: [200] } } }, async () => {
+        const synth = new Set(syntheticSuites());
+        return {
+            suites: SUITE_IDS.map((id) => ({
+                id,
+                title: SUITE_TITLES[id],
+                replay: deps.replay.adapters[id] !== undefined || id === 'steering',
+                synthetic: synth.has(id),
+                note: deps.replay.missing[id] ?? null,
+            })),
+        };
+    });
+    app.post(`${V}/testing/default-evals/run`, { config: { manifest: { requestType: 'RunDefaultEvalsBody', responseType: 'DefaultEvalReport', statusCodes: [200, 400] } } }, async (req, reply) => {
+        const parsed = RunBody.safeParse(req.body ?? {});
+        if (!parsed.success)
+            return reply.code(400).send({ error: 'invalid default-evals run body', details: parsed.error.issues });
+        const project = parsed.data.project === undefined || parsed.data.project === 'all' ? null : parsed.data.project;
+        const report = await runAndRecord(deps, {
+            project,
+            baseline: parsed.data.baseline === true,
+            ...(parsed.data.suites !== undefined ? { suites: parsed.data.suites } : {}),
+            id: newId(),
+            now: now(),
+        });
+        return report;
+    });
+    app.get(`${V}/testing/default-evals`, { config: { manifest: { responseType: 'ListDefaultEvalRunsResponse', statusCodes: [200] } } }, async () => ({ runs: deps.store !== undefined ? await deps.store.list() : [] }));
+    app.get(`${V}/testing/default-evals/:id`, { config: { manifest: { responseType: 'DefaultEvalDetail', statusCodes: [200, 404] } } }, async (req, reply) => {
+        const { id } = req.params;
+        const detail = deps.store !== undefined ? await deps.store.get(id) : null;
+        if (detail === null)
+            return reply.code(404).send({ error: `no default-eval run ${id}` });
+        return detail;
+    });
+}
+//# sourceMappingURL=routes.js.map

package/dist/evals/routes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"routes.js","sourceRoot":"","sources":["../../src/evals/routes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,UAAU,EAA0F,MAAM,cAAc,CAAC;AAElI,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAuD,MAAM,YAAY,CAAC;AAE5F,MAAM,CAAC,GAAG,UAAU,CAAC;AAkBrB,MAAM,OAAO,GAAG,CAAC;KACd,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACrD,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACpD,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAqB,EAAE,IAA2B;IAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,EAAE,GAAG,OAAO,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5E,IAAI,EAAE,KAAK,IAAI;YAAE,SAAS;QAC1B,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAChD,IAAI,IAAI,KAAK,IAAI;YAAE,SAAS;QAC5B,IAAI,CAAC;YACH,KAAK,MAAM,CAAC,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;gBAAE,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClF,CAAC;QAAC,MAAM,CAAC;YACP,8EAA8E;QAChF,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,CAAC;IACpH,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,QAAQ,GAAe,EAAE,CAAC;IAC9B,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,KAAK,SAAS;YAAE,QAAQ,GAAG,iBAAiB,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAClG,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,0GAA0G;IAC1G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC,CAAC;AAC/E,CAAC;AAED;gDACgD;AAChD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAqB,EACrB,IAA8F;IAE9F,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,eAAe,CAAC;QACtD,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,KAAK;QACL,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,QAAQ,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;QACjF,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACjF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,GAAoB,EAAE,IAAqB;IACnF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;IAEjD,GAAG,CAAC,GAAG,CACL,GAAG,CAAC,+BAA+B,EACnC,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,2BAA2B,EAAE,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAC3F,KAAK,IAAI,EAAE;QACT,MAAM,KAAK,GAAG,IAAI,GAAG,CAAU,eAAe,EAAE,CAAC,CAAC;QAClD,OAAO;YACL,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC7B,EAAE;gBACF,KAAK,EAAE,YAAY,CAAC,EAAE,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,SAAS,IAAI,EAAE,KAAK,UAAU;gBACnE,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,IAAI;aACtC,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,GAAG,CAAC,IAAI,CACN,GAAG,CAAC,4BAA4B,EAChC,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,qBAAqB,EAAE,YAAY,EAAE,mBAAmB,EAAE,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,EAC5H,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACnB,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,gCAAgC,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5H,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QAChH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE;YACtC,OAAO;YACP,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI;YACvC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3E,EAAE,EAAE,KAAK,EAAE;YACX,GAAG,EAAE,GAAG,EAAE;SACX,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC,CACF,CAAC;IAEF,GAAG,CAAC,GAAG,CACL,GAAG,CAAC,wBAAwB,EAC5B,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,6BAA6B,EAAE,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAC7F,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAChF,CAAC;IAEF,GAAG,CAAC,GAAG,CACL,GAAG,CAAC,4BAA4B,EAChC,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,mBAAmB,EAAE,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,EACxF,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACnB,MAAM,EAAE,EAAE,EAAE,GAAG,GAAG,CAAC,MAAwB,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,EAAE,EAAE,CAAC,CAAC;QACzF,OAAO,MAAM,CAAC;IAChB,CAAC,CACF,CAAC;AACJ,CAAC"}

package/dist/evals/run-cli.d.ts

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+/**
+ * `run-evals` (DES-DEFAULT-EVALS-001 §7): the Tool step of the `run-evals` workflow, and a plain command.
+ *
+ *   node dist/evals/run-cli.js [--daemon <url>] [--project <id>|all] [--baseline] [--out <file>]
+ *   node dist/evals/run-cli.js --runs-dir <dir>      (the workflow's Tool step)
+ *
+ * From a shell, the daemon is `--daemon`, else `$WICKED_CREW_URL`; with neither it refuses (it never guesses a port).
+ * As the `run-evals` workflow's Tool step it calls nothing: the daemon ran the evals when it launched the run and
+ * wrote `<dir>/$WICKED_RUN_ID/report.json`; the step reads that back and prints its summary.
+ *
+ * It asks the daemon to run the default evals (`POST /api/v1/testing/default-evals/run`), so the daemon's own store
+ * is what gets scored and the report is stored where studio reads it. It prints a short plain summary, writes the
+ * full report to `--out` when given, and exits 0 on a completed run (a regression is a finding, not a failure),
+ * 1 when the daemon refused or could not be reached.
+ */
+import type { DefaultEvalReport } from './types.js';
+export interface CliArgs {
+    /** `null` until named: the CLI never guesses a port, since a default could reach another daemon. */
+    daemon: string | null;
+    project: string | null;
+    baseline: boolean;
+    out: string | null;
+    /** The workflow's mode: read `<runsDir>/$WICKED_RUN_ID/report.json` back instead of calling a daemon. */
+    runsDir: string | null;
+}
+export declare function parseArgs(argv: readonly string[], env?: NodeJS.ProcessEnv): CliArgs;
+/** The plain summary: one line per suite, too-few marked, then the drift keys. */
+export declare function summaryLines(r: DefaultEvalReport): string[];
+export declare function main(argv: readonly string[], fetchImpl?: typeof fetch, env?: NodeJS.ProcessEnv): Promise<number>;
+//# sourceMappingURL=run-cli.d.ts.map

package/dist/evals/run-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-cli.d.ts","sourceRoot":"","sources":["../../src/evals/run-cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,WAAW,OAAO;IACtB,oGAAoG;IACpG,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,yGAAyG;IACzG,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,OAAO,CA0BhG;AAMD,kFAAkF;AAClF,wBAAgB,YAAY,CAAC,CAAC,EAAE,iBAAiB,GAAG,MAAM,EAAE,CAc3D;AAED,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,SAAS,GAAE,OAAO,KAAa,EAAE,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,OAAO,CAAC,MAAM,CAAC,CAgD1I"}

package/dist/evals/run-cli.js

@@ -0,0 +1,128 @@
+#!/usr/bin/env node
+/**
+ * `run-evals` (DES-DEFAULT-EVALS-001 §7): the Tool step of the `run-evals` workflow, and a plain command.
+ *
+ *   node dist/evals/run-cli.js [--daemon <url>] [--project <id>|all] [--baseline] [--out <file>]
+ *   node dist/evals/run-cli.js --runs-dir <dir>      (the workflow's Tool step)
+ *
+ * From a shell, the daemon is `--daemon`, else `$WICKED_CREW_URL`; with neither it refuses (it never guesses a port).
+ * As the `run-evals` workflow's Tool step it calls nothing: the daemon ran the evals when it launched the run and
+ * wrote `<dir>/$WICKED_RUN_ID/report.json`; the step reads that back and prints its summary.
+ *
+ * It asks the daemon to run the default evals (`POST /api/v1/testing/default-evals/run`), so the daemon's own store
+ * is what gets scored and the report is stored where studio reads it. It prints a short plain summary, writes the
+ * full report to `--out` when given, and exits 0 on a completed run (a regression is a finding, not a failure),
+ * 1 when the daemon refused or could not be reached.
+ */
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+export function parseArgs(argv, env = process.env) {
+    const args = {
+        daemon: env['WICKED_CREW_URL'] !== undefined && env['WICKED_CREW_URL'] !== '' ? env['WICKED_CREW_URL'].replace(/\/+$/, '') : null,
+        project: null,
+        baseline: false,
+        out: null,
+        runsDir: null,
+    };
+    for (let i = 0; i < argv.length; i += 1) {
+        const a = argv[i];
+        const next = () => {
+            const v = argv[i + 1];
+            if (v === undefined)
+                throw new Error(`${a} needs a value`);
+            i += 1;
+            return v;
+        };
+        if (a === '--daemon')
+            args.daemon = next().replace(/\/+$/, '');
+        else if (a === '--project') {
+            const p = next();
+            args.project = p === 'all' ? null : p;
+        }
+        else if (a === '--baseline')
+            args.baseline = true;
+        else if (a === '--out')
+            args.out = next();
+        else if (a === '--runs-dir')
+            args.runsDir = next();
+        else
+            throw new Error(`unknown argument ${String(a)}`);
+    }
+    return args;
+}
+function pct(v) {
+    return v === null ? '—' : `${Math.round(v * 100)}%`;
+}
+/** The plain summary: one line per suite, too-few marked, then the drift keys. */
+export function summaryLines(r) {
+    const lines = [
+        `default evals ${r.id} (${r.project ?? 'every project'}): ${r.counts.real} real, ${r.counts.synthetic} synthetic` +
+            `${r.counts.baseline > 0 ? `, ${r.counts.baseline} from the shipped baseline` : ''}, ${r.counts.unscored} not scored`,
+    ];
+    for (const s of r.suites) {
+        const a = s.combined.agreement;
+        const figure = a.n === 0 ? 'no cases' : a.tooFew ? `too few cases (n=${a.n})` : `${pct(a.value)} agree (${pct(a.lo)}–${pct(a.hi)}, n=${a.n})`;
+        lines.push(`  ${s.suite}: ${figure}; trend ${s.trend.word}`);
+    }
+    for (const d of r.drift) {
+        lines.push(`  drift ${d.suite} ${d.inputKey} by ${d.axis}: ${Object.entries(d.observed).map(([p, v]) => `${p}=${v.join('|')}`).join(', ')}`);
+    }
+    return lines;
+}
+export async function main(argv, fetchImpl = fetch, env = process.env) {
+    let args;
+    try {
+        args = parseArgs(argv, env);
+    }
+    catch (err) {
+        process.stderr.write(`run-evals: ${err instanceof Error ? err.message : String(err)}\n`);
+        return 1;
+    }
+    if (args.runsDir !== null) {
+        const runId = env['WICKED_RUN_ID'];
+        if (runId === undefined || runId === '' || !/^[A-Za-z0-9_.-]+$/.test(runId)) {
+            process.stderr.write('run-evals: --runs-dir needs the run id in WICKED_RUN_ID (the engine hands it to every Tool step)\n');
+            return 1;
+        }
+        let report;
+        try {
+            report = JSON.parse(readFileSync(join(args.runsDir, runId, 'report.json'), 'utf8'));
+        }
+        catch (err) {
+            process.stderr.write(`run-evals: the daemon prepared no report for run ${runId}: ${err instanceof Error ? err.message : String(err)}\n`);
+            return 1;
+        }
+        process.stdout.write(`${summaryLines(report).join('\n')}\n`);
+        return 0;
+    }
+    if (args.daemon === null) {
+        process.stderr.write('run-evals: no daemon named: pass --daemon <url> or set WICKED_CREW_URL (it never guesses a port)\n');
+        return 1;
+    }
+    const body = { ...(args.project !== null ? { project: args.project } : {}), ...(args.baseline ? { baseline: true } : {}) };
+    let res;
+    try {
+        res = await fetchImpl(`${args.daemon}/api/v1/testing/default-evals/run`, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify(body),
+        });
+    }
+    catch (err) {
+        process.stderr.write(`run-evals: the daemon at ${args.daemon} did not answer: ${err instanceof Error ? err.message : String(err)}\n`);
+        return 1;
+    }
+    if (!res.ok) {
+        process.stderr.write(`run-evals: the daemon refused (${res.status}): ${(await res.text()).slice(0, 300)}\n`);
+        return 1;
+    }
+    const report = (await res.json());
+    if (args.out !== null)
+        writeFileSync(args.out, JSON.stringify(report, null, 2));
+    process.stdout.write(`${summaryLines(report).join('\n')}\n`);
+    return 0;
+}
+if (process.argv[1] !== undefined && /run-cli\.(js|ts)$/.test(process.argv[1])) {
+    void main(process.argv.slice(2)).then((code) => process.exit(code));
+}
+//# sourceMappingURL=run-cli.js.map

package/dist/evals/run-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-cli.js","sourceRoot":"","sources":["../../src/evals/run-cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAcjC,MAAM,UAAU,SAAS,CAAC,IAAuB,EAAE,MAAyB,OAAO,CAAC,GAAG;IACrF,MAAM,IAAI,GAAY;QACpB,MAAM,EAAE,GAAG,CAAC,iBAAiB,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;QACjI,OAAO,EAAE,IAAI;QACb,QAAQ,EAAE,KAAK;QACf,GAAG,EAAE,IAAI;QACT,OAAO,EAAE,IAAI;KACd,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,GAAW,EAAE;YACxB,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtB,IAAI,CAAC,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC3D,CAAC,IAAI,CAAC,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC,CAAC;QACF,IAAI,CAAC,KAAK,UAAU;YAAE,IAAI,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;aAC1D,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;aAAM,IAAI,CAAC,KAAK,YAAY;YAAE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;aAC/C,IAAI,CAAC,KAAK,OAAO;YAAE,IAAI,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC;aACrC,IAAI,CAAC,KAAK,YAAY;YAAE,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,CAAC;;YAC9C,MAAM,IAAI,KAAK,CAAC,oBAAoB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,GAAG,CAAC,CAAgB;IAC3B,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;AACtD,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,YAAY,CAAC,CAAoB;IAC/C,MAAM,KAAK,GAAG;QACZ,iBAAiB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,OAAO,IAAI,eAAe,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,MAAM,CAAC,SAAS,YAAY;YAC/G,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,4BAA4B,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,aAAa;KACxH,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9I,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,KAAK,MAAM,WAAW,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,QAAQ,OAAO,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/I,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAuB,EAAE,YAA0B,KAAK,EAAE,MAAyB,OAAO,CAAC,GAAG;IACvH,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,eAAe,CAAC,CAAC;QACnC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oGAAoG,CAAC,CAAC;YAC3H,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,MAAyB,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAsB,CAAC;QAC3G,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oDAAoD,KAAK,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACzI,OAAO,CAAC,CAAC;QACX,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oGAAoG,CAAC,CAAC;QAC3H,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,IAAI,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAC3H,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,mCAAmC,EAAE;YACvE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtI,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7G,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAsB,CAAC;IACvD,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI;QAAE,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,OAAO,CAAC,CAAC;AACX,CAAC;AAED,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/E,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACtE,CAAC"}

package/dist/evals/runner.d.ts

@@ -0,0 +1,34 @@
+/**
+ * The default-evals runner (DES-DEFAULT-EVALS-001 §5): one project, or every project, scored suite by suite, with
+ * synthetic cases filling the gaps and the normalization drift a cross-project run surfaces. Deterministic given its
+ * inputs: the clock and the run id are passed in, and the only side effect is the replay adapters' subprocesses.
+ */
+import type { ReplaySet } from './replay.js';
+import { type DefaultEvalReport, type DefaultEvalSummary, type EvalCase, type SuiteId, type SuiteScore } from './types.js';
+export declare const SUITE_TITLES: Record<SuiteId, string>;
+export type PreviousScores = Partial<Record<SuiteId, {
+    runId: string;
+    score: SuiteScore;
+}>>;
+/** The most recent score of each suite across stored reports, newest first. */
+export declare function previousScores(reports: readonly DefaultEvalReport[]): PreviousScores;
+export interface RunInput {
+    id: string;
+    now: Date;
+    /** `null` = every project. */
+    project: string | null;
+    /** Real cases (captured and seeded) and the steering eval's cases. Synthetic ones are generated here. */
+    cases: EvalCase[];
+    replay: ReplaySet;
+    /** Per suite, the most recent stored score of the same scope and its run id (a run that skipped the suite is
+     *  passed over), for the trend. */
+    previous: PreviousScores;
+    /** Limit to these suites (all when absent). */
+    suites?: SuiteId[];
+}
+export declare function runDefaultEvals(input: RunInput): Promise<{
+    report: DefaultEvalReport;
+    cases: EvalCase[];
+}>;
+export declare function summarize(report: DefaultEvalReport): DefaultEvalSummary;
+//# sourceMappingURL=runner.d.ts.map

package/dist/evals/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAa,KAAK,iBAAiB,EAAE,KAAK,kBAAkB,EAAE,KAAK,QAAQ,EAAmB,KAAK,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,YAAY,CAAC;AAEvJ,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAShD,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,CAAC,CAAC,CAAC;AAE5F,+EAA+E;AAC/E,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,GAAG,cAAc,CAIpF;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,IAAI,CAAC;IACV,8BAA8B;IAC9B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,yGAAyG;IACzG,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,MAAM,EAAE,SAAS,CAAC;IAClB;uCACmC;IACnC,QAAQ,EAAE,cAAc,CAAC;IACzB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;CACpB;AAyBD,wBAAsB,eAAe,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,iBAAiB,CAAC;IAAC,KAAK,EAAE,QAAQ,EAAE,CAAA;CAAE,CAAC,CA0GhH;AAED,wBAAgB,SAAS,CAAC,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,CAevE"}

package/dist/evals/runner.js

@@ -0,0 +1,181 @@
+/**
+ * The default-evals runner (DES-DEFAULT-EVALS-001 §5): one project, or every project, scored suite by suite, with
+ * synthetic cases filling the gaps and the normalization drift a cross-project run surfaces. Deterministic given its
+ * inputs: the clock and the run id are passed in, and the only side effect is the replay adapters' subprocesses.
+ */
+import { driftKeys, normalizationRate, scoreSlice, slices } from './score.js';
+import { rate, trendOf, MIN_CALIBRATION, MIN_CASES } from './stats.js';
+import { generateSynthetic } from './synthetic.js';
+import { SUITE_IDS } from './types.js';
+export const SUITE_TITLES = {
+    'needs-you': 'What needs you: what counts, and what is one item',
+    'floor-class': 'Floor decisions: pass, real failure, base red, inconclusive, boundary deny',
+    'qe-verdict': 'QE verdicts: pass, fail, inconclusive, and one answer per tree',
+    attribution: 'Who built it: authors and the creator a review checks',
+    steering: 'Steering rules: block the bad, leave the good alone',
+    recall: 'Memory recall: the memories a step needs surface',
+    archetype: 'Prompt routing: which kind of work a prompt asks for',
+    normalization: 'Normalizing: the same input judged the same in every project and surface',
+};
+/** The most recent score of each suite across stored reports, newest first. */
+export function previousScores(reports) {
+    const out = {};
+    for (const r of reports)
+        for (const sc of r.suites)
+            if (out[sc.suite] === undefined)
+                out[sc.suite] = { runId: r.id, score: sc };
+    return out;
+}
+function dedupe(cases) {
+    const by = new Map();
+    for (const c of cases)
+        by.set(c.id, c);
+    return [...by.values()].sort((a, b) => a.id.localeCompare(b.id));
+}
+function inScope(c, project) {
+    return project === null || c.project === project || (c.source === 'synthetic' && c.project === null);
+}
+function group(cases, keyOf) {
+    const by = new Map();
+    for (const c of cases) {
+        const k = keyOf(c);
+        if (k === null)
+            continue;
+        by.set(k, [...(by.get(k) ?? []), c]);
+    }
+    if (by.size === 0)
+        return undefined;
+    const out = {};
+    for (const [k, cs] of [...by.entries()].sort(([a], [b]) => a.localeCompare(b)))
+        out[k] = scoreSlice(cs);
+    return out;
+}
+export async function runDefaultEvals(input) {
+    const wanted = new Set(input.suites ?? SUITE_IDS);
+    // Normalization is computed over every other suite's cases, so asking for it keeps them all (codex/agy r1);
+    // otherwise only the asked-for suites' cases are read.
+    const reads = (c) => wanted.has('normalization') || wanted.has(c.suite);
+    const given = dedupe(input.cases).filter((c) => inScope(c, input.project) && reads(c));
+    const givenIds = new Set(given.map((c) => c.id));
+    const synthetic = generateSynthetic(given).filter((c) => reads(c) && !givenIds.has(c.id));
+    let cases = [...given, ...synthetic];
+    // Replay: the current classifier answers every case that has no recorded answer.
+    const notes = new Map();
+    const note = (s, line) => {
+        notes.set(s, [...(notes.get(s) ?? []), line]);
+    };
+    for (const suite of SUITE_IDS) {
+        const waiting = cases.filter((c) => c.suite === suite && c.observed === null);
+        if (waiting.length === 0)
+            continue;
+        const adapter = input.replay.adapters[suite];
+        if (adapter === undefined) {
+            note(suite, `${waiting.length} case(s) not scored: ${input.replay.missing[suite] ?? 'no replay adapter'}`);
+            continue;
+        }
+        try {
+            const answers = await adapter.replay(waiting);
+            cases = cases.map((c) => (c.observed === null && answers.has(c.id) ? { ...c, observed: answers.get(c.id) } : c));
+            const left = waiting.length - answers.size;
+            if (left > 0)
+                note(suite, `${left} case(s) the replay adapter could not answer`);
+        }
+        catch (err) {
+            note(suite, `replay failed, ${waiting.length} case(s) not scored: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    const drift = driftKeys(cases);
+    const suites = [];
+    for (const suite of SUITE_IDS) {
+        if (!wanted.has(suite))
+            continue;
+        const prevEntry = input.previous[suite];
+        const prev = prevEntry?.score ?? null;
+        const prevId = prevEntry?.runId ?? null;
+        if (suite === 'normalization') {
+            const { compared, consistent } = normalizationRate(cases, drift);
+            const real = scoreSlice([]);
+            real.cases = compared;
+            real.agreement = rate(consistent, compared);
+            const empty = scoreSlice([]);
+            suites.push({
+                suite,
+                title: SUITE_TITLES[suite],
+                real,
+                synthetic: empty,
+                combined: real,
+                model: empty,
+                trend: { word: trendOf(real.agreement, prev ? { value: prev.combined.agreement.value, n: prev.combined.agreement.n } : null), previous: prev?.combined.agreement.value ?? null, previousRunId: prevId },
+                notes: [
+                    `${compared} input(s) were judged in two or more places; ${drift.length} were judged differently`,
+                    ...(input.project !== null ? ['a single-project run compares nothing across places; run every project to compare projects and surfaces'] : []),
+                ],
+            });
+            continue;
+        }
+        const mine = cases.filter((c) => c.suite === suite);
+        const s = slices(mine);
+        const extra = notes.get(suite) ?? [];
+        if (mine.length === 0)
+            extra.push('no cases yet');
+        if (suite === 'needs-you' && input.project !== null)
+            extra.push('what needs you is daemon-wide: run every project to score it');
+        if (suite === 'recall' && mine.length === 0) {
+            extra.push('no labelled recall cases: run the steering eval with a memory store and samples that name expected_memories');
+        }
+        const score = {
+            suite,
+            title: SUITE_TITLES[suite],
+            ...s,
+            trend: {
+                word: trendOf(s.combined.agreement, prev ? { value: prev.combined.agreement.value, n: prev.combined.agreement.n } : null),
+                previous: prev?.combined.agreement.value ?? null,
+                previousRunId: prevId,
+            },
+            notes: extra,
+        };
+        if (input.project === null) {
+            const byProject = group(mine.filter((c) => !c.expected.by.startsWith('model:')), (c) => c.project);
+            if (byProject !== undefined && Object.keys(byProject).length > 1)
+                score.byProject = byProject;
+        }
+        const bySurface = group(mine.filter((c) => !c.expected.by.startsWith('model:')), (c) => c.surface ?? null);
+        if (bySurface !== undefined)
+            score.bySurface = bySurface;
+        suites.push(score);
+    }
+    const people = cases.filter((c) => !c.expected.by.startsWith('model:'));
+    const report = {
+        id: input.id,
+        createdAt: input.now.toISOString(),
+        project: input.project,
+        suites,
+        drift,
+        counts: {
+            real: people.filter((c) => c.source === 'real').length,
+            synthetic: people.filter((c) => c.source === 'synthetic').length,
+            modelLabelled: cases.length - people.length,
+            unscored: cases.filter((c) => c.observed === null).length,
+            baseline: cases.filter((c) => c.baseline === true).length,
+        },
+        minimums: { rate: MIN_CASES, calibration: MIN_CALIBRATION },
+    };
+    return { report, cases };
+}
+export function summarize(report) {
+    return {
+        id: report.id,
+        createdAt: report.createdAt,
+        project: report.project,
+        suites: report.suites.map((s) => ({
+            suite: s.suite,
+            agreement: s.combined.agreement.value,
+            n: s.combined.agreement.n,
+            tooFew: s.combined.agreement.tooFew,
+            trend: s.trend.word,
+        })),
+        drift: report.drift.length,
+        counts: report.counts,
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/evals/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAkH,MAAM,YAAY,CAAC;AAEvJ,MAAM,CAAC,MAAM,YAAY,GAA4B;IACnD,WAAW,EAAE,mDAAmD;IAChE,aAAa,EAAE,4EAA4E;IAC3F,YAAY,EAAE,gEAAgE;IAC9E,WAAW,EAAE,uDAAuD;IACpE,QAAQ,EAAE,qDAAqD;IAC/D,MAAM,EAAE,kDAAkD;IAC1D,SAAS,EAAE,sDAAsD;IACjE,aAAa,EAAE,0EAA0E;CAC1F,CAAC;AAIF,+EAA+E;AAC/E,MAAM,UAAU,cAAc,CAAC,OAAqC;IAClE,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM;YAAE,IAAI,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,SAAS;gBAAE,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAChI,OAAO,GAAG,CAAC;AACb,CAAC;AAiBD,SAAS,MAAM,CAAC,KAA0B;IACxC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAoB,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,OAAO,CAAC,CAAW,EAAE,OAAsB;IAClD,OAAO,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,KAAK,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,KAAK,CAAmB,KAA0B,EAAE,KAAgC;IAC3F,MAAM,EAAE,GAAG,IAAI,GAAG,EAAsB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,CAAC,KAAK,IAAI;YAAE,SAAS;QACzB,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACpC,MAAM,GAAG,GAA+B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QAAE,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IACxG,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAe;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAU,KAAK,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;IAC3D,4GAA4G;IAC5G,uDAAuD;IACvD,MAAM,KAAK,GAAG,CAAC,CAAW,EAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1F,IAAI,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC,CAAC;IAErC,iFAAiF;IACjF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC3C,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,IAAY,EAAQ,EAAE;QAC9C,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAC9E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACnC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,mBAAmB,EAAE,CAAC,CAAC;YAC3G,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC9C,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClH,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAC3C,IAAI,IAAI,GAAG,CAAC;gBAAE,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,8CAA8C,CAAC,CAAC;QACnF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,EAAE,kBAAkB,OAAO,CAAC,MAAM,wBAAwB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAS;QACjC,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,SAAS,EAAE,KAAK,IAAI,IAAI,CAAC;QACtC,MAAM,MAAM,GAAG,SAAS,EAAE,KAAK,IAAI,IAAI,CAAC;QACxC,IAAI,KAAK,KAAK,eAAe,EAAE,CAAC;YAC9B,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACjE,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC;YACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK;gBACL,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;gBAC1B,IAAI;gBACJ,SAAS,EAAE,KAAK;gBAChB,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE;gBACvM,KAAK,EAAE;oBACL,GAAG,QAAQ,gDAAgD,KAAK,CAAC,MAAM,0BAA0B;oBACjG,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,yGAAyG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC/I;aACF,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QACpD,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAClD,IAAI,KAAK,KAAK,WAAW,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;QAChI,IAAI,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5C,KAAK,CAAC,IAAI,CAAC,6GAA6G,CAAC,CAAC;QAC5H,CAAC;QACD,MAAM,KAAK,GAAe;YACxB,KAAK;YACL,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;YAC1B,GAAG,CAAC;YACJ,KAAK,EAAE;gBACL,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBACzH,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI;gBAChD,aAAa,EAAE,MAAM;aACtB;YACD,KAAK,EAAE,KAAK;SACb,CAAC;QACF,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACnG,IAAI,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;QAChG,CAAC;QACD,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;QAC3G,IAAI,SAAS,KAAK,SAAS;YAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAsB;QAChC,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM;QACN,KAAK;QACL,MAAM,EAAE;YACN,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;YACtD,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM;YAChE,aAAa,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YAC3C,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,MAAM;YACzD,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,MAAM;SAC1D;QACD,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE;KAC5D,CAAC;IACF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAyB;IACjD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,EAAE;QACb,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK;YACrC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACzB,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM;YACnC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI;SACpB,CAAC,CAAC;QACH,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;QAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC"}

package/dist/evals/score.d.ts

@@ -0,0 +1,31 @@
+/**
+ * The scorer (DES-DEFAULT-EVALS-001 §3, §6): cases in, figures out. Deterministic code only: no model, no clock, no
+ * I/O. A case with no observation is `unscored` and is never counted as right or wrong.
+ */
+import type { DriftKey, EvalCase, SliceScore } from './types.js';
+/** Did the system get this case right? `null` when it gave no answer. */
+export declare function isCorrect(c: EvalCase): boolean | null;
+export declare function isModelLabelled(c: EvalCase): boolean;
+export declare function scoreSlice(cases: readonly EvalCase[]): SliceScore;
+/** The four slices every suite reports (DES §4): real, synthetic, combined (both, no model labels), model. */
+export declare function slices(cases: readonly EvalCase[]): {
+    real: SliceScore;
+    synthetic: SliceScore;
+    combined: SliceScore;
+    model: SliceScore;
+};
+/**
+ * The normalization figure (S8): of the inputs observed in two or more places (projects, or surfaces), the share
+ * judged the same everywhere. Computed from the same grouping as {@link driftKeys}.
+ */
+export declare function normalizationRate(cases: readonly EvalCase[], drift: readonly DriftKey[]): {
+    compared: number;
+    consistent: number;
+};
+/**
+ * Normalization drift (DES §5): the same input judged differently by place. Cases are grouped by suite and
+ * `inputKey`; a key whose observed answers differ across projects (or across surfaces, when the cases carry one)
+ * is drift. A place where the answers already differ among themselves still counts once per distinct answer.
+ */
+export declare function driftKeys(cases: readonly EvalCase[]): DriftKey[];
+//# sourceMappingURL=score.d.ts.map

package/dist/evals/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/evals/score.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAc,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAW,MAAM,YAAY,CAAC;AAQtF,yEAAyE;AACzE,wBAAgB,SAAS,CAAC,CAAC,EAAE,QAAQ,GAAG,OAAO,GAAG,IAAI,CAMrD;AAED,wBAAgB,eAAe,CAAC,CAAC,EAAE,QAAQ,GAAG,OAAO,CAEpD;AAkDD,wBAAgB,UAAU,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,GAAG,UAAU,CAgBjE;AAED,8GAA8G;AAC9G,wBAAgB,MAAM,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,GAAG;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,SAAS,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,CAQvI;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,QAAQ,EAAE,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAalI;AASD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAyChE"}