wicked-crew 0.16.4 → 0.16.6

package/dist/evals/score.js

@@ -0,0 +1,191 @@
+/**
+ * The scorer (DES-DEFAULT-EVALS-001 §3, §6): cases in, figures out. Deterministic code only: no model, no clock, no
+ * I/O. A case with no observation is `unscored` and is never counted as right or wrong.
+ */
+import { calibration, rate } from './stats.js';
+function sameSet(a, b) {
+    const sa = new Set(a);
+    const sb = new Set(b);
+    return sa.size === sb.size && [...sa].every((x) => sb.has(x));
+}
+/** Did the system get this case right? `null` when it gave no answer. */
+export function isCorrect(c) {
+    const o = c.observed;
+    if (o === null)
+        return null;
+    const e = c.expected;
+    if (e.labels !== undefined)
+        return o.labels !== undefined && sameSet(o.labels, e.labels);
+    return o.label !== undefined && o.label === e.label;
+}
+export function isModelLabelled(c) {
+    return c.expected.by.startsWith('model:');
+}
+function perClass(scored) {
+    const labels = new Set();
+    for (const c of scored) {
+        if (c.expected.label !== undefined)
+            labels.add(c.expected.label);
+        if (c.observed?.label !== undefined)
+            labels.add(c.observed.label);
+    }
+    return [...labels].sort().map((label) => {
+        const said = scored.filter((c) => c.observed?.label === label);
+        const truth = scored.filter((c) => c.expected.label === label);
+        const both = scored.filter((c) => c.observed?.label === label && c.expected.label === label).length;
+        return { label, precision: rate(both, said.length), recall: rate(both, truth.length) };
+    });
+}
+function members(scored) {
+    let hit = 0;
+    let said = 0;
+    let truth = 0;
+    for (const c of scored) {
+        const o = new Set(c.observed?.labels ?? []);
+        const e = new Set(c.expected.labels ?? []);
+        said += o.size;
+        truth += e.size;
+        for (const x of o)
+            if (e.has(x))
+                hit += 1;
+    }
+    return { precision: rate(hit, said), recall: rate(hit, truth) };
+}
+/** Pairwise merge agreement: over every pair of cases from the same answerer (surface), "these two are one item" in the observation vs the label. */
+function pairs(scored) {
+    let both = 0;
+    let said = 0;
+    let truth = 0;
+    for (let i = 0; i < scored.length; i += 1) {
+        for (let j = i + 1; j < scored.length; j += 1) {
+            const a = scored[i];
+            const b = scored[j];
+            if ((a.surface ?? '') !== (b.surface ?? ''))
+                continue; // each answerer's merge is judged on its own
+            const o = a.observed?.cluster !== undefined && a.observed.cluster === b.observed?.cluster;
+            const e = a.expected.cluster !== undefined && a.expected.cluster === b.expected.cluster;
+            if (o)
+                said += 1;
+            if (e)
+                truth += 1;
+            if (o && e)
+                both += 1;
+        }
+    }
+    return { precision: rate(both, said), recall: rate(both, truth) };
+}
+export function scoreSlice(cases) {
+    const scored = cases.filter((c) => c.observed !== null);
+    const right = scored.filter((c) => isCorrect(c) === true).length;
+    const out = {
+        cases: cases.length,
+        unscored: cases.length - scored.length,
+        agreement: rate(right, scored.length),
+        perClass: perClass(scored),
+    };
+    if (scored.some((c) => c.expected.labels !== undefined))
+        out.members = members(scored);
+    if (scored.some((c) => c.expected.cluster !== undefined || c.observed?.cluster !== undefined))
+        out.pairs = pairs(scored);
+    const confident = scored.filter((c) => typeof c.observed?.confidence === 'number');
+    if (confident.length > 0) {
+        out.calibration = calibration(confident.map((c) => ({ confidence: c.observed.confidence, correct: isCorrect(c) === true })));
+    }
+    return out;
+}
+/** The four slices every suite reports (DES §4): real, synthetic, combined (both, no model labels), model. */
+export function slices(cases) {
+    const people = cases.filter((c) => !isModelLabelled(c));
+    return {
+        real: scoreSlice(people.filter((c) => c.source === 'real')),
+        synthetic: scoreSlice(people.filter((c) => c.source === 'synthetic')),
+        combined: scoreSlice(people),
+        model: scoreSlice(cases.filter(isModelLabelled)),
+    };
+}
+/**
+ * The normalization figure (S8): of the inputs observed in two or more places (projects, or surfaces), the share
+ * judged the same everywhere. Computed from the same grouping as {@link driftKeys}.
+ */
+export function normalizationRate(cases, drift) {
+    const places = new Map();
+    for (const c of cases) {
+        if (c.observed === null)
+            continue;
+        const k = `${c.suite}\u0000${c.inputKey}`;
+        const p = places.get(k) ?? { surfaces: new Set(), projects: new Set() };
+        if (c.surface !== undefined)
+            p.surfaces.add(c.surface);
+        if (c.project !== null)
+            p.projects.add(c.project);
+        places.set(k, p);
+    }
+    let compared = 0;
+    for (const p of places.values())
+        if (p.surfaces.size >= 2 || p.projects.size >= 2)
+            compared += 1;
+    return { compared, consistent: compared - drift.length };
+}
+function answerWord(c) {
+    const o = c.observed;
+    if (o === null)
+        return null;
+    if (o.labels !== undefined)
+        return [...new Set(o.labels)].sort().join('+') || '(none)';
+    return o.label ?? null;
+}
+/**
+ * Normalization drift (DES §5): the same input judged differently by place. Cases are grouped by suite and
+ * `inputKey`; a key whose observed answers differ across projects (or across surfaces, when the cases carry one)
+ * is drift. A place where the answers already differ among themselves still counts once per distinct answer.
+ */
+export function driftKeys(cases) {
+    const groups = new Map();
+    for (const c of cases) {
+        if (c.observed === null)
+            continue;
+        const k = `${c.suite}\u0000${c.inputKey}`;
+        const g = groups.get(k);
+        if (g === undefined)
+            groups.set(k, [c]);
+        else
+            g.push(c);
+    }
+    const out = [];
+    for (const g of groups.values()) {
+        const axes = [
+            ['surface', (c) => c.surface ?? null],
+            ['project', (c) => c.project],
+        ];
+        for (const [axis, placeOf] of axes) {
+            const by = new Map();
+            for (const c of g) {
+                const place = placeOf(c);
+                const word = answerWord(c);
+                if (place === null || word === null)
+                    continue;
+                const s = by.get(place) ?? new Set();
+                s.add(word);
+                by.set(place, s);
+            }
+            if (by.size < 2)
+                continue;
+            const distinct = new Set([...by.values()].map((s) => [...s].sort().join('|')));
+            if (distinct.size < 2)
+                continue;
+            const observed = {};
+            for (const [place, s] of [...by.entries()].sort(([a], [b]) => a.localeCompare(b)))
+                observed[place] = [...s].sort();
+            out.push({
+                inputKey: g[0].inputKey,
+                suite: g[0].suite,
+                axis,
+                observed,
+                caseIds: g.map((c) => c.id).sort(),
+            });
+            break; // one drift row per key: the surface axis, when present, already names the disagreement
+        }
+    }
+    return out.sort((a, b) => a.suite.localeCompare(b.suite) || a.inputKey.localeCompare(b.inputKey));
+}
+//# sourceMappingURL=score.js.map

package/dist/evals/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/evals/score.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAG/C,SAAS,OAAO,CAAC,CAAoB,EAAE,CAAoB;IACzD,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACtB,OAAO,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,SAAS,CAAC,CAAW;IACnC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACrB,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACrB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzF,OAAO,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,CAAW;IACzC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,QAAQ,CAAC,MAA2B;IAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACjE,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACpG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;IACzF,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,MAA2B;IAC1C,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC;QACf,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,CAAC;YAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,GAAG,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,qJAAqJ;AACrJ,SAAS,KAAK,CAAC,MAA2B;IACxC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YACrB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YACrB,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;gBAAE,SAAS,CAAC,6CAA6C;YACpG,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAK,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC;YAC1F,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;YACxF,IAAI,CAAC;gBAAE,IAAI,IAAI,CAAC,CAAC;YACjB,IAAI,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC;gBAAE,IAAI,IAAI,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAA0B;IACnD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,GAAG,GAAe;QACtB,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,QAAQ,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QACtC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC;QACrC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC;KAC3B,CAAC;IACF,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC;QAAE,GAAG,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvF,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,KAAK,SAAS,CAAC;QAAE,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IACzH,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,UAAU,KAAK,QAAQ,CAAC,CAAC;IACnF,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,QAAS,CAAC,UAAW,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACjI,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8GAA8G;AAC9G,MAAM,UAAU,MAAM,CAAC,KAA0B;IAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QAC3D,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QACrE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;KACjD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAA0B,EAAE,KAA0B;IACtF,MAAM,MAAM,GAAG,IAAI,GAAG,EAA4D,CAAC;IACnF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI;YAAE,SAAS;QAClC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAU,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAU,EAAE,CAAC;QACxF,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;YAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI;YAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnB,CAAC;IACD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;QAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;YAAE,QAAQ,IAAI,CAAC,CAAC;IACjG,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;AAC3D,CAAC;AAED,SAAS,UAAU,CAAC,CAAW;IAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACrB,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;IACvF,OAAO,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;AACzB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,KAA0B;IAClD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI;YAAE,SAAS;QAClC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;;YACnC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,GAAmE;YAC3E,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC;YACrC,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;SAC9B,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;YACnC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAuB,CAAC;YAC1C,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC3B,IAAI,KAAK,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;oBAAE,SAAS;gBAC9C,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;gBAC7C,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACZ,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACnB,CAAC;YACD,IAAI,EAAE,CAAC,IAAI,GAAG,CAAC;gBAAE,SAAS;YAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/E,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC;gBAAE,SAAS;YAChC,MAAM,QAAQ,GAA6B,EAAE,CAAC;YAC9C,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;gBAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnH,GAAG,CAAC,IAAI,CAAC;gBACP,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,QAAQ;gBACxB,KAAK,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,KAAgB;gBAC7B,IAAI;gBACJ,QAAQ;gBACR,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE;aACnC,CAAC,CAAC;YACH,MAAM,CAAC,wFAAwF;QACjG,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AACpG,CAAC"}

package/dist/evals/seed-needs-you.d.ts

@@ -0,0 +1,17 @@
+/**
+ * S1 needs-you, seeded from the operator's ruling on one live snapshot (rig crew 0.16.3, 2026-10-27T04:11:50Z,
+ * GET only): "the number is how many distinct causes are holding work up right now" (the needs-you DEFINITION).
+ * The ruled answer is 5 items over 6 runs. Each item below is asked of four answerers:
+ *
+ *  - `home`: studio's Home badge (`attentionBadge`: gate rows minus ask turns, plus failed runs) said 21;
+ *  - `needs-page`: studio's /needs headline (`needCount` over every fold row, proposals included) said 51;
+ *  - `nerve-center`: crew `GET /system` `attention[]` said 12 at the dump (11 on the screenshot);
+ *  - `codex`: the codex CLI, given the same dump read-only, said 7 (it would not merge the core red-base runs).
+ *
+ * The surfaces never merge, so each counted row is its own cluster there. These are real cases labelled by the
+ * operator; a live capture replaces them once needs-one's Resolve records operator answers.
+ */
+import type { EvalCase } from './types.js';
+export declare const NEEDS_YOU_SNAPSHOT = "2026-10-27T04:11:50Z";
+export declare function needsYouSeed(): EvalCase[];
+//# sourceMappingURL=seed-needs-you.d.ts.map

package/dist/evals/seed-needs-you.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"seed-needs-you.d.ts","sourceRoot":"","sources":["../../src/evals/seed-needs-you.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAoC3C,eAAO,MAAM,kBAAkB,yBAAyB,CAAC;AAEzD,wBAAgB,YAAY,IAAI,QAAQ,EAAE,CAsBzC"}

package/dist/evals/seed-needs-you.js

@@ -0,0 +1,57 @@
+/**
+ * S1 needs-you, seeded from the operator's ruling on one live snapshot (rig crew 0.16.3, 2026-10-27T04:11:50Z,
+ * GET only): "the number is how many distinct causes are holding work up right now" (the needs-you DEFINITION).
+ * The ruled answer is 5 items over 6 runs. Each item below is asked of four answerers:
+ *
+ *  - `home`: studio's Home badge (`attentionBadge`: gate rows minus ask turns, plus failed runs) said 21;
+ *  - `needs-page`: studio's /needs headline (`needCount` over every fold row, proposals included) said 51;
+ *  - `nerve-center`: crew `GET /system` `attention[]` said 12 at the dump (11 on the screenshot);
+ *  - `codex`: the codex CLI, given the same dump read-only, said 7 (it would not merge the core red-base runs).
+ *
+ * The surfaces never merge, so each counted row is its own cluster there. These are real cases labelled by the
+ * operator; a live capture replaces them once needs-one's Resolve records operator answers.
+ */
+const SURFACES = ['home', 'needs-page', 'nerve-center', 'codex'];
+const TINY_MATH_FAILED = ['89da69fb', '1ff74953', 'c2bc93a0', 'dcc4547b', 'e2c2cd4d', '6dd64a52', 'b979f42c', '7bbaab3a', 'bb505ad8', '449f566e', '4b517e33', '78cf6482', '11b1b4ea'];
+const ITEMS = [
+    { id: 'infra:load', kind: 'infra', cluster: 'host-load', why: 'the host is overloaded: system slowing', countedBy: ['nerve-center', 'codex'], codexCluster: 'host-load' },
+    { id: '29bcace4', kind: 'deliver gate', cluster: 'core-red-base', why: 'the fix for core’s red base waits at its deliver gate', countedBy: ['home', 'needs-page', 'nerve-center', 'codex'], codexCluster: '29bcace4' },
+    { id: '1940f173', kind: 'escalation', cluster: 'core-red-base', why: 'held by core’s red base', countedBy: ['home', 'needs-page', 'nerve-center', 'codex'], codexCluster: '1940f173' },
+    { id: '2ebd9da2', kind: 'escalation', cluster: 'core-red-base', why: 'held by core’s red base', countedBy: ['home', 'needs-page', 'nerve-center', 'codex'], codexCluster: '2ebd9da2' },
+    { id: '64f05d3d', kind: 'escalation', cluster: 'crew-red-base', why: 'crew’s base is red and cannot be separated', countedBy: ['home', 'needs-page', 'nerve-center', 'codex'], codexCluster: '64f05d3d' },
+    { id: '290a7fee', kind: 'escalation', cluster: 'dead-seat', why: 'a dead seat with no failover', countedBy: ['home', 'needs-page', 'nerve-center', 'codex'], codexCluster: '290a7fee' },
+    { id: '68c26704', kind: 'escalation (ask)', cluster: 'ask-refused', why: 'an ask refused by input governance waits on the operator', countedBy: ['nerve-center', 'codex'], codexCluster: '68c26704' },
+    { id: '7f4e149f', kind: 'deliver gate', cluster: null, why: 'superseded: the same repo and title was delivered later', countedBy: ['home', 'needs-page', 'nerve-center'] },
+    { id: '9bd0f728', kind: 'escalation', cluster: null, why: 'superseded: a retry names it and was delivered', countedBy: ['home', 'needs-page', 'nerve-center'] },
+    { id: 'b87f42fb', kind: 'answered ask', cluster: null, why: 'an answered ask parked at its turn', countedBy: ['nerve-center'] },
+    { id: 'e43c4560', kind: 'answered ask', cluster: null, why: 'an answered ask parked at its turn', countedBy: ['nerve-center'] },
+    ...TINY_MATH_FAILED.map((id) => ({ id, kind: 'failed run', cluster: null, why: 'an old ad-hoc probe, failed over 7 days ago', countedBy: ['home', 'needs-page'] })),
+    { id: '910e3e0b', kind: 'failed run', cluster: null, why: 'superseded by a delivered run', countedBy: ['home', 'needs-page'] },
+    { id: 'cli:copilot', kind: 'benched seat', cluster: null, why: 'a self-expiring bench while a distinct judge is still available (info)', countedBy: ['nerve-center'] },
+    { id: 'proposals', kind: 'pending proposals', cluster: null, why: 'reviews: nothing waits on them (to review, not in the number)', countedBy: ['needs-page'] },
+];
+export const NEEDS_YOU_SNAPSHOT = '2026-10-27T04:11:50Z';
+export function needsYouSeed() {
+    const out = [];
+    for (const it of ITEMS) {
+        for (const s of SURFACES) {
+            const counted = it.countedBy.includes(s);
+            const cluster = counted ? (s === 'codex' ? (it.codexCluster ?? it.id) : it.id) : undefined;
+            out.push({
+                id: `needs-you:${NEEDS_YOU_SNAPSHOT}:${s}:${it.id}`,
+                suite: 'needs-you',
+                source: 'real',
+                project: null,
+                surface: s,
+                inputKey: `needs:${it.id}`,
+                input: { item: it.id, kind: it.kind, snapshot: NEEDS_YOU_SNAPSHOT },
+                observed: { label: counted ? 'count' : 'not', ...(cluster !== undefined ? { cluster } : {}), by: 'recorded' },
+                expected: { label: it.cluster !== null ? 'count' : 'not', ...(it.cluster !== null ? { cluster: it.cluster } : {}), by: 'operator', why: it.why },
+                evidence: { file: 'the needs-you DEFINITION ruling, 2026-10-27' },
+                baseline: true,
+            });
+        }
+    }
+    return out;
+}
+//# sourceMappingURL=seed-needs-you.js.map

package/dist/evals/seed-needs-you.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"seed-needs-you.js","sourceRoot":"","sources":["../../src/evals/seed-needs-you.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,MAAM,QAAQ,GAAc,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;AAa5E,MAAM,gBAAgB,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAEtL,MAAM,KAAK,GAAW;IACpB,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,wCAAwC,EAAE,SAAS,EAAE,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE;IACzK,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,EAAE,uDAAuD,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE;IACtN,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,EAAE,yBAAyB,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE;IACtL,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,EAAE,yBAAyB,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE;IACtL,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,EAAE,4CAA4C,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE;IACzM,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,8BAA8B,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE;IACvL,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,0DAA0D,EAAE,SAAS,EAAE,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE;IACrM,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,yDAAyD,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,CAAC,EAAE;IAC1K,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,gDAAgD,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,CAAC,EAAE;IAC/J,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,oCAAoC,EAAE,SAAS,EAAE,CAAC,cAAc,CAAC,EAAE;IAC/H,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,oCAAoC,EAAE,SAAS,EAAE,CAAC,cAAc,CAAC,EAAE;IAC/H,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,6CAA6C,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;IACzK,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,+BAA+B,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE;IAC9H,EAAE,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,wEAAwE,EAAE,SAAS,EAAE,CAAC,cAAc,CAAC,EAAE;IACtK,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,mBAAmB,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,+DAA+D,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,EAAE;CAC/J,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAEzD,MAAM,UAAU,YAAY;IAC1B,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC3F,GAAG,CAAC,IAAI,CAAC;gBACP,EAAE,EAAE,aAAa,kBAAkB,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;gBACnD,KAAK,EAAE,WAAW;gBAClB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,CAAC;gBACV,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE;gBAC1B,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE;gBACnE,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE;gBAC7G,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE;gBAChJ,QAAQ,EAAE,EAAE,IAAI,EAAE,6CAA6C,EAAE;gBACjE,QAAQ,EAAE,IAAI;aACf,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/evals/seed-rig.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Real cases captured from the test rig's run store on 2026-10-27 (crew 0.16.3, GET only), by this module's own
+ * capture code (`capture.ts`), so a fresh daemon has evidence on day one. Live capture yields the same ids and
+ * replaces these. Inputs are scrubbed: no paths, no operator words, titles only.
+ */
+import type { EvalCase } from './types.js';
+export declare const RIG_SEED_AT = "2026-10-27";
+export declare const RIG_SEED: EvalCase[];
+//# sourceMappingURL=seed-rig.d.ts.map

package/dist/evals/seed-rig.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"seed-rig.d.ts","sourceRoot":"","sources":["../../src/evals/seed-rig.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,eAAO,MAAM,WAAW,eAAe,CAAC;AAGxC,eAAO,MAAM,QAAQ,EAAE,QAAQ,EAmJwb,CAAC"}