spawnfile 0.1.24 → 0.1.27

package/dist/training/graders/calibration/escalation.js

@@ -0,0 +1,145 @@
+import { planCallAdmission, escalationCallKey as callKey } from '../admission.js';
+import { deriveConsensus } from '../consensus.js';
+import { evidenceCounts, teacherAccuracyFailures, wilson90 } from './metrics.js';
+export function inEscalationBand(p, band) { return p >= band[0] && p <= band[1]; }
+/** Retained group means a lineage with at least one retained (non-escalated) instance. */
+function retainedCoverage(rows, band) {
+    const total = new Set(rows.map(row => row.group)).size;
+    return total ? new Set(rows.filter(row => !row.missing && !inEscalationBand(row.p, band)).map(row => row.group)).size / total : 0;
+}
+function retainedAccuracy(rows, band) {
+    const retained = rows.filter(row => !row.missing && !inEscalationBand(row.p, band));
+    return retained.length ? retained.filter(row => (row.p >= 0.5) === row.good).length / retained.length : 0;
+}
+/** Remove the escalation teacher before deriving strict option consensus, never binary majority. */
+function independentGood(row, teacherId) {
+    if (row.votes && row.labelers && row.expected && teacherId !== undefined) {
+        const votes = Object.fromEntries(Object.entries(row.votes).filter(([name]) => name !== teacherId));
+        const asked = [...row.asked ?? Object.keys(row.votes)].filter(name => name !== teacherId);
+        const label = { id: 'cascade', lineage: row.group, output: null, context: {}, votes: { answer: votes }, asked, provenance: {} };
+        const answer = deriveConsensus(label, row.labelers).answer.answer;
+        return answer === undefined ? undefined : row.expected.includes(answer);
+    }
+    // Human/editor truth is independent; model-derived truth requires original votes.
+    return row.source === 'human' || row.source === 'editor' ? row.good : undefined;
+}
+function teacherAnswer(row, teacherId) {
+    if (row.votes && row.expected && teacherId !== undefined) {
+        const answer = row.votes[teacherId];
+        return answer === undefined || answer === 'abstain' ? undefined : row.expected.includes(answer);
+    }
+    return row.teacherGood;
+}
+export function escalationTeacherReport(heldOut, teacherId) {
+    const independent = heldOut.flatMap(row => {
+        const good = independentGood(row, teacherId);
+        if (good === undefined)
+            return [];
+        let correct = teacherAnswer(row, teacherId) === good;
+        if (row.votes && row.labelers && row.expected) {
+            const votes = Object.fromEntries(Object.entries(row.votes).filter(([name]) => name !== teacherId));
+            const asked = [...row.asked ?? Object.keys(row.votes)].filter(name => name !== teacherId);
+            const answer = deriveConsensus({ id: 'teacher', lineage: row.group, output: null, context: {},
+                votes: { answer: votes }, asked, provenance: {} }, row.labelers).answer.answer;
+            // Exact options must agree; two different acceptable options are not agreement.
+            correct = row.votes[teacherId] === answer;
+        }
+        return [{ ...row, good, correct }];
+    });
+    const counts = evidenceCounts(independent);
+    const agreement = independent.length ? independent.filter(row => row.correct).length / independent.length : 0;
+    const source = independent.length && independent.every(row => row.source === 'human') ? 'human'
+        : independent.length && independent.every(row => row.source === 'editor') ? 'editor' : 'independent';
+    return { ...counts, source, agreement, failures: teacherAccuracyFailures({ ...counts, agreement }),
+        warnings: independent.length < heldOut.length ? ['Instances without independent golden answers are excluded from cascade verification.'] : [] };
+}
+function callFor(row) {
+    return row.outputId !== undefined && row.projectionDigest !== undefined
+        ? { outputId: row.outputId, projectionDigest: row.projectionDigest } : undefined;
+}
+export function planEscalation(fit, heldOut, policy, options = {}) {
+    if ([...fit, ...heldOut].some(row => !row.missing && (!Number.isFinite(row.p) || row.p < 0 || row.p > 1))) {
+        throw new Error('p: expected finite probability');
+    }
+    const limit = options.maxCallsPerTrial ?? Infinity;
+    if (options.maxCallsPerTrial !== undefined && (!Number.isInteger(limit) || limit < 1)) {
+        throw new Error('maxCallsPerTrial: expected positive integer');
+    }
+    fit = fit.filter(row => !row.disputed);
+    const report = options.teacherId === undefined ? {} : { teacherReport: escalationTeacherReport(heldOut, options.teacherId) };
+    let band;
+    if ('between' in policy) {
+        const [lo, hi] = policy.between;
+        if (!(Number.isFinite(lo) && Number.isFinite(hi) && lo >= 0 && lo < hi && hi <= 1)) {
+            throw new Error('between: expected 0 <= lo < hi <= 1');
+        }
+        band = policy.between;
+    }
+    else {
+        const target = policy.target_accuracy;
+        if (!(Number.isFinite(target) && target > 0.5 && target <= 1))
+            throw new Error('target_accuracy: expected (0.5, 1]');
+        // Width-first traversal is deterministic; ties choose the lowest lower endpoint.
+        for (let width = 1; width <= 20 && !band; width++) {
+            for (let lower = 0; lower + width <= 20; lower++) {
+                const candidate = [lower / 20, (lower + width) / 20];
+                if (retainedCoverage(fit, candidate) >= 0.30 && retainedAccuracy(fit, candidate) >= target) {
+                    band = candidate;
+                    break;
+                }
+            }
+        }
+    }
+    if (!band && options.forceTeacher)
+        band = [0, 1];
+    if (!band)
+        return { status: 'escalation-infeasible', fitAccuracy: 0, heldOutAccuracy: 0, cascadeInterval: [0, 1],
+            independentGroups: 0, escalationRate: 0, coverage: 0, failures: ['fit.target_accuracy and retainedCoverage >= 0.30'], ...report };
+    const chosen = band;
+    const escalated = heldOut.filter(row => options.forceTeacher || !row.missing && inEscalationBand(row.p, chosen));
+    const calls = options.calls ?? escalated.flatMap(row => { const call = callFor(row); return call ? [call] : []; });
+    const allowed = new Set(planCallAdmission(calls, limit).filter(call => call.admitted).map(callKey));
+    const independent = heldOut.flatMap(row => {
+        const good = independentGood(row, options.teacherId);
+        if (good === undefined)
+            return [];
+        const call = callFor(row);
+        const permitted = limit === Infinity || (call !== undefined && allowed.has(callKey(call)));
+        const answer = options.forceTeacher || !row.missing && inEscalationBand(row.p, chosen)
+            ? (permitted ? teacherAnswer(row, options.teacherId) : undefined) : row.missing ? undefined : row.p >= 0.5;
+        return [{ ...row, good, answer }];
+    });
+    const independentGroups = new Set(independent.map(row => row.group)).size;
+    const correct = independent.filter(row => row.answer !== undefined && row.answer === row.good).length;
+    const heldOutAccuracy = independent.length ? correct / independent.length : 0;
+    const cascadeInterval = wilson90(correct, independent.length);
+    const coverage = options.forceTeacher ? 0 : retainedCoverage(heldOut, chosen);
+    const failures = [];
+    if (independentGroups < 15)
+        failures.push('heldOut.independentGroups >= 15');
+    if (options.forceTeacher && report.teacherReport)
+        failures.push(...report.teacherReport.failures);
+    if ('target_accuracy' in policy) {
+        if (heldOutAccuracy < policy.target_accuracy)
+            failures.push('heldOut.cascadeAccuracy >= target_accuracy');
+        if (cascadeInterval[0] < policy.target_accuracy - 0.10)
+            failures.push('heldOut.cascade90Lower >= target_accuracy - 0.10');
+    }
+    if (!options.forceTeacher && coverage < 0.30)
+        failures.push('heldOut.retainedCoverage >= 0.30');
+    const bad = independent.filter(row => !row.good);
+    const falsePassCount = bad.filter(row => row.answer === true).length;
+    const falsePassRate = bad.length ? falsePassCount / bad.length : 0;
+    const falsePassInterval = wilson90(falsePassCount, bad.length);
+    if (options.gate) {
+        if (falsePassRate > 0.10)
+            failures.push('gate.falsePassRate <= 0.10');
+        if (falsePassInterval[1] > 0.20)
+            failures.push('gate.falsePass90Upper <= 0.20');
+    }
+    return { status: independentGroups < 15 ? 'unverified' : failures.length ? 'escalation-infeasible' : 'calibrated', band: chosen,
+        fitAccuracy: retainedAccuracy(fit, chosen), heldOutAccuracy, cascadeInterval, independentGroups,
+        ...(options.gate ? { falsePassRate, falsePassInterval } : {}),
+        escalationRate: heldOut.length ? escalated.length / heldOut.length : 0, coverage, failures, ...report };
+}
+//# sourceMappingURL=escalation.js.map

package/dist/training/graders/calibration/escalation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"escalation.js","sourceRoot":"","sources":["../../../../src/training/graders/calibration/escalation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,IAAI,OAAO,EAAuB,MAAM,iBAAiB,CAAC;AAEvG,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,OAAO,EAAE,cAAc,EAAE,uBAAuB,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAqCjF,MAAM,UAAU,gBAAgB,CAAC,CAAS,EAAE,IAAoB,IAAa,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEnH,0FAA0F;AAC1F,SAAS,gBAAgB,CAAC,IAA8B,EAAE,IAAoB;IAC5E,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACpI,CAAC;AAED,SAAS,gBAAgB,CAAC,IAA8B,EAAE,IAAoB;IAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACpF,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5G,CAAC;AAED,oGAAoG;AACpG,SAAS,eAAe,CAAC,GAAkB,EAAE,SAAkB;IAC7D,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC;QACnG,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QAChI,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAO,CAAC,MAAM,CAAC;QACnE,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC;IACD,kFAAkF;IAClF,OAAO,GAAG,CAAC,MAAM,KAAK,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAClF,CAAC;AAED,SAAS,aAAa,CAAC,GAAkB,EAAE,SAAkB;IAC3D,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACpC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,GAAG,CAAC,WAAW,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAiC,EAAE,SAAiB;IAC1F,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QACxC,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QAClC,IAAI,OAAO,GAAG,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,KAAK,IAAI,CAAC;QACrD,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC;YACnG,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;YAC1F,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;gBAC3F,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAO,CAAC,MAAM,CAAC;YAClF,gFAAgF;YAChF,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,MAAM,CAAC;QAC5C,CAAC;QACD,OAAO,CAAC,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9G,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;QAC7F,CAAC,CAAC,WAAW,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC;IACvG,OAAO,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,uBAAuB,CAAC,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,CAAC;QAChG,QAAQ,EAAE,WAAW,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,sFAAsF,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACpJ,CAAC;AAED,SAAS,OAAO,CAAC,GAAkB;IACjC,OAAO,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,gBAAgB,KAAK,SAAS;QACrE,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAA6B,EAAE,OAAiC,EAAE,MAAkB,EACjH,UAAwI,EAAE;IAC1I,IAAI,CAAC,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1G,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC;IACnD,IAAI,OAAO,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,uBAAuB,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;IAC7H,IAAI,IAAgC,CAAC;IACrC,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;QACxB,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC;QACtC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACrH,iFAAiF;QACjF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAClD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;gBACjD,MAAM,SAAS,GAAmB,CAAC,KAAK,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;gBACrE,IAAI,gBAAgB,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,IAAI,IAAI,gBAAgB,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,MAAM,EAAE,CAAC;oBAC3F,IAAI,GAAG,SAAS,CAAC;oBAAC,MAAM;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,YAAY;QAAE,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9G,iBAAiB,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,kDAAkD,CAAC,EAAE,GAAG,MAAM,EAAE,CAAC;IACpI,MAAM,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IACjH,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACnH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IACpG,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QACxC,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1B,MAAM,SAAS,GAAG,KAAK,KAAK,QAAQ,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3F,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC;YACpF,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC;QAC7G,OAAO,CAAC,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1E,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACtG,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,iBAAiB,GAAG,EAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC7E,IAAI,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,aAAa;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAClG,IAAI,iBAAiB,IAAI,MAAM,EAAE,CAAC;QAChC,IAAI,eAAe,GAAG,MAAM,CAAC,eAAe;YAAE,QAAQ,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;QAC1G,IAAI,eAAe,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,eAAe,GAAG,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;IAC5H,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,QAAQ,GAAG,IAAI;QAAE,QAAQ,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAChG,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjD,MAAM,cAAc,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACrE,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/D,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,IAAI,aAAa,GAAG,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACtE,IAAI,iBAAiB,CAAC,CAAC,CAAC,GAAG,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,iBAAiB,GAAG,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,YAAY,EAAE,IAAI,EAAE,MAAM;QAC7H,WAAW,EAAE,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,eAAe,EAAE,eAAe,EAAE,iBAAiB;QAC/F,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,EAAE,CAAC;AAC5G,CAAC"}

package/dist/training/graders/calibration/fitting.d.ts

@@ -0,0 +1,28 @@
+import type { CalibrationRecord, Exclusions, PlattFit, PlattParams, QuestionCalibration } from './types.js';
+/** Hash rank gives a balanced 50/50 group split, invariant to input row ordering. */
+export declare function splitCalibration(records: readonly CalibrationRecord[], seed: string, exclusions?: Exclusions): {
+    fit: CalibrationRecord[];
+    heldOut: CalibrationRecord[];
+    excluded: CalibrationRecord[];
+};
+export declare function applyPlatt(rawPGood: number, params: PlattParams): number;
+/** Mean binary NLL + lambda * ((a-1)^2 + b^2), lambda=0.01.
+ * This fixed regularizer is intentionally not weakened by larger instance counts.
+ */
+export declare function fitPlatt(records: readonly Pick<CalibrationRecord, 'rawPGood' | 'good'>[], options?: {
+    maxIterations?: number;
+    tolerance?: number;
+}): PlattFit;
+/** This seam accepts fit rows only; candidate selection has no held-out input. */
+export declare function fitQuestion(fitRows: readonly CalibrationRecord[], questionId: string, type: CalibrationRecord['type']): {
+    questionSplit: CalibrationRecord[];
+    questionFit: CalibrationRecord[];
+    fitGroups: number;
+    fitSource: "type" | "question";
+    params: PlattFit;
+};
+export declare function calibrateQuestions(records: readonly CalibrationRecord[], options: {
+    seed: string;
+    exclusions?: Exclusions;
+    gates?: readonly string[];
+}): QuestionCalibration[];

package/dist/training/graders/calibration/fitting.js

@@ -0,0 +1,121 @@
+import { createHash } from 'node:crypto';
+import { calibrationBars, calibrationMetrics } from './metrics.js';
+/** Hash rank gives a balanced 50/50 group split, invariant to input row ordering. */
+export function splitCalibration(records, seed, exclusions = {}) {
+    const lineages = new Set(exclusions.lineages);
+    const examples = new Set(exclusions.examples?.map(example => JSON.stringify([example.dataset, example.id])));
+    // Excluding an example excludes all revisions in its lineage, including revisions without example metadata.
+    for (const record of records) {
+        if (record.example && examples.has(JSON.stringify([record.example.dataset, record.example.id])))
+            lineages.add(record.group);
+    }
+    const excluded = records.filter(record => lineages.has(record.group));
+    const eligible = records.filter(record => !lineages.has(record.group));
+    const groups = [...new Set(eligible.map(record => record.group))].map(group => ({ group,
+        hash: createHash('sha256').update(JSON.stringify([seed, group])).digest('hex'),
+    })).sort((a, b) => a.hash.localeCompare(b.hash) || a.group.localeCompare(b.group));
+    const fitGroups = new Set(groups.slice(0, Math.floor(groups.length / 2)).map(item => item.group));
+    return { fit: eligible.filter(record => fitGroups.has(record.group)),
+        heldOut: eligible.filter(record => !fitGroups.has(record.group)), excluded };
+}
+function logit(p) {
+    if (!Number.isFinite(p) || p < 0 || p > 1)
+        throw new Error('rawPGood: expected finite probability in [0, 1]');
+    const clipped = Math.max(1e-6, Math.min(1 - 1e-6, p));
+    return Math.log(clipped / (1 - clipped));
+}
+function sigmoid(value) { return 1 / (1 + Math.exp(-value)); }
+export function applyPlatt(rawPGood, params) {
+    if (!Number.isFinite(params.a) || !Number.isFinite(params.b))
+        throw new Error('params: expected finite a and b');
+    return sigmoid(params.a * logit(rawPGood) + params.b);
+}
+/** Mean binary NLL + lambda * ((a-1)^2 + b^2), lambda=0.01.
+ * This fixed regularizer is intentionally not weakened by larger instance counts.
+ */
+export function fitPlatt(records, options = {}) {
+    const maxIterations = options.maxIterations ?? 100;
+    const tolerance = options.tolerance ?? 1e-9;
+    if (!Number.isInteger(maxIterations) || maxIterations < 1 || !Number.isFinite(tolerance) || tolerance <= 0) {
+        throw new Error('fit options: maxIterations must be positive integer and tolerance positive finite');
+    }
+    const rows = records.map(record => ({ x: logit(record.rawPGood), y: Number(record.good) }));
+    let a = 1, b = 0;
+    if (!rows.length)
+        return { a, b, converged: false, iterations: 0 };
+    const loss = (nextA, nextB) => rows.reduce((sum, row) => {
+        const z = nextA * row.x + nextB;
+        return sum + Math.max(z, 0) + Math.log1p(Math.exp(-Math.abs(z))) - row.y * z;
+    }, 0) / rows.length + 0.01 * ((nextA - 1) ** 2 + nextB ** 2);
+    for (let iteration = 1; iteration <= maxIterations; iteration++) {
+        let ga = 0.02 * (a - 1), gb = 0.02 * b, aa = 0.02, ab = 0, bb = 0.02;
+        for (const row of rows) {
+            const p = sigmoid(a * row.x + b), residual = (p - row.y) / rows.length;
+            const curvature = p * (1 - p) / rows.length;
+            ga += residual * row.x;
+            gb += residual;
+            aa += curvature * row.x * row.x;
+            ab += curvature * row.x;
+            bb += curvature;
+        }
+        if (Math.hypot(ga, gb) < tolerance)
+            return { a, b, converged: true, iterations: iteration };
+        const determinant = aa * bb - ab * ab;
+        const da = (bb * ga - ab * gb) / determinant, db = (aa * gb - ab * ga) / determinant;
+        let step = 1;
+        const currentLoss = loss(a, b);
+        // Summed NLL has O(n * epsilon) roundoff; do not stall a converged Newton step on that noise.
+        const roundoff = Number.EPSILON * rows.length * Math.max(1, currentLoss);
+        while (step > 1e-8 && loss(a - step * da, b - step * db) > currentLoss + roundoff)
+            step /= 2;
+        a -= step * da;
+        b -= step * db;
+    }
+    return { a, b, converged: false, iterations: maxIterations };
+}
+/** This seam accepts fit rows only; candidate selection has no held-out input. */
+export function fitQuestion(fitRows, questionId, type) {
+    const questionSplit = fitRows.filter(record => record.questionId === questionId && !record.disputed);
+    const questionFit = questionSplit.filter(record => !record.missing);
+    const fitGroups = new Set(questionFit.map(record => record.group)).size;
+    const fitSource = fitGroups >= 20 ? 'question' : 'type';
+    const fit = fitSource === 'question' ? questionFit : fitRows.filter(record => record.type === type && !record.missing && !record.disputed);
+    return { questionSplit, questionFit, fitGroups, fitSource, params: fitPlatt(fit) };
+}
+export function calibrateQuestions(records, options) {
+    const population = splitCalibration(records, options.seed, options.exclusions);
+    const split = { fit: population.fit.filter(record => !record.disputed),
+        heldOut: population.heldOut.filter(record => !record.disputed) };
+    return [...new Set(records.map(record => record.questionId))].map(questionId => {
+        const question = records.find(record => record.questionId === questionId);
+        if (records.some(record => record.questionId === questionId && record.type !== question.type)) {
+            throw new Error(`${questionId}.type: inconsistent question types`);
+        }
+        const { questionSplit, questionFit, fitGroups, fitSource, params } = fitQuestion(split.fit, questionId, question.type);
+        const baseRate = questionSplit.length ? questionSplit.filter(record => record.good).length / questionSplit.length : 0;
+        const heldOut = split.heldOut.filter(record => record.questionId === questionId)
+            .map(record => ({ ...record, p: record.missing ? 0 : applyPlatt(record.rawPGood, params) }));
+        const gate = options.gates?.includes(questionId) ?? false;
+        const metrics = calibrationMetrics(heldOut, baseRate, `${options.seed}:${questionId}`, gate);
+        const bars = calibrationBars(metrics);
+        if (!params.converged) {
+            bars.status = 'insufficient-evidence';
+            bars.failures.push('fit.convergence');
+        }
+        if (!questionFit.length) {
+            bars.status = 'insufficient-evidence';
+            bars.failures.push('fit.questionInstances > 0');
+        }
+        const bySource = {};
+        for (const source of ['human', 'editor', 'consensus', 'single-teacher']) {
+            const subset = heldOut.filter(record => record.source === source);
+            if (subset.length)
+                bySource[source] = calibrationMetrics(subset, baseRate, `${options.seed}:${questionId}:${source}`, gate);
+        }
+        return { questionId, params: { a: params.a, b: params.b }, fitSource, fitGroups, metrics, bySource, ...bars,
+            interpretation: heldOut.some(record => record.source === 'human')
+                ? 'human-adjudicated and teacher agreement' : heldOut.some(record => record.source === 'editor')
+                ? 'editor-adjudicated and teacher agreement' : 'teacher agreement, not validated correctness' };
+    });
+}
+//# sourceMappingURL=fitting.js.map

package/dist/training/graders/calibration/fitting.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fitting.js","sourceRoot":"","sources":["../../../../src/training/graders/calibration/fitting.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAGnE,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAAC,OAAqC,EAAE,IAAY,EAAE,aAAyB,EAAE;IAG/G,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7G,4GAA4G;IAC5G,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9H,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK;QACrF,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;KAC/E,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAClG,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;AACjF,CAAC;AAED,SAAS,KAAK,CAAC,CAAS;IACtB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IAC9G,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,OAAO,CAAC,KAAa,IAAY,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAE9E,MAAM,UAAU,UAAU,CAAC,QAAgB,EAAE,MAAmB;IAC9D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACjH,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,OAAgE,EACvF,UAA0D,EAAE;IAC5D,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC;IACnD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QAC3G,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAC;IACvG,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5F,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACjB,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACnE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,KAAa,EAAU,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC9E,MAAM,CAAC,GAAG,KAAK,GAAG,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QAChC,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/E,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC;IAC7D,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,aAAa,EAAE,SAAS,EAAE,EAAE,CAAC;QAChE,IAAI,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC;QACrE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YACvE,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YAC5C,EAAE,IAAI,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC;YAAC,EAAE,IAAI,QAAQ,CAAC;YACvC,EAAE,IAAI,SAAS,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YAAC,EAAE,IAAI,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC;YAAC,EAAE,IAAI,SAAS,CAAC;QAC5E,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,SAAS;YAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QAC5F,MAAM,WAAW,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,WAAW,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,WAAW,CAAC;QACrF,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/B,8FAA8F;QAC9F,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QACzE,OAAO,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC,GAAG,WAAW,GAAG,QAAQ;YAAE,IAAI,IAAI,CAAC,CAAC;QAC7F,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;QAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;IACjC,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;AAC/D,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,WAAW,CAAC,OAAqC,EAAE,UAAkB,EAAE,IAA+B;IACpH,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACrG,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACpE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IACxE,MAAM,SAAS,GAAG,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,UAAmB,CAAC,CAAC,CAAC,MAAe,CAAC;IAC1E,MAAM,GAAG,GAAG,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3I,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAqC,EAAE,OAEzE;IACC,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;QACpE,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;IACnE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;QAC7E,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAE,CAAC;QAC3E,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9F,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,oCAAoC,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,EAAE,aAAa,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvH,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACtH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC;aAC7E,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/F,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;QAC1D,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,IAAI,UAAU,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7F,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAAC,IAAI,CAAC,MAAM,GAAG,uBAAuB,CAAC;YAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAAC,CAAC;QACxG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAAC,IAAI,CAAC,MAAM,GAAG,uBAAuB,CAAC;YAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAAC,CAAC;QACpH,MAAM,QAAQ,GAAoC,EAAE,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,CAAU,EAAE,CAAC;YACjF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;YAClE,IAAI,MAAM,CAAC,MAAM;gBAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,kBAAkB,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,IAAI,UAAU,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9H,CAAC;QACD,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI;YACzG,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC;gBAC/D,CAAC,CAAC,yCAAyC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC;gBAC9F,CAAC,CAAC,0CAA0C,CAAC,CAAC,CAAC,8CAA8C,EAAE,CAAC;IACxG,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/training/graders/calibration/index.d.ts

@@ -0,0 +1,7 @@
+export { splitCalibration, fitPlatt, applyPlatt, calibrateQuestions } from './fitting.js';
+export { seededRandom, evidenceCounts, expectedCalibrationError, groupBootstrap, calibrationMetrics, evidenceFailures, calibrationBars, teacherAgreement, teacherAccuracyFailures, wilson90 } from './metrics.js';
+export { inEscalationBand, planEscalation, escalationTeacherReport } from './escalation.js';
+export type { QuestionType, GoldenSource, CalibrationRecord, Exclusions, PlattParams, PlattFit, ScoredRecord, EvidenceCounts, Interval, CalibrationMetrics, CalibrationStatus, BarResult, QuestionCalibration } from './types.js';
+export type { TeacherAgreementRecord } from './metrics.js';
+export type { EscalationBand, BandPolicy, CascadeRecord, BandPlan, EscalationTeacherReport, EscalationCall } from './escalation.js';
+export { selectQuestionModel, fitNll } from './selection.js';

package/dist/training/graders/calibration/index.js

@@ -0,0 +1,5 @@
+export { splitCalibration, fitPlatt, applyPlatt, calibrateQuestions } from './fitting.js';
+export { seededRandom, evidenceCounts, expectedCalibrationError, groupBootstrap, calibrationMetrics, evidenceFailures, calibrationBars, teacherAgreement, teacherAccuracyFailures, wilson90 } from './metrics.js';
+export { inEscalationBand, planEscalation, escalationTeacherReport } from './escalation.js';
+export { selectQuestionModel, fitNll } from './selection.js';
+//# sourceMappingURL=index.js.map

package/dist/training/graders/calibration/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/training/graders/calibration/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAC1F,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,wBAAwB,EAAE,cAAc,EAC7E,kBAAkB,EAAE,gBAAgB,EAAE,eAAe,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACnI,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAO5F,OAAO,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/training/graders/calibration/metrics.d.ts

@@ -0,0 +1,31 @@
+import type { BarResult, CalibrationMetrics, EvidenceCounts, Interval, ScoredRecord } from './types.js';
+/** Two-sided 90% Wilson score interval; no observations cannot establish a bound. */
+export declare function wilson90(successes: number, total: number): Interval;
+export declare function seededRandom(seed: string): () => number;
+export declare function evidenceCounts(records: readonly Pick<ScoredRecord, 'group' | 'good'>[]): EvidenceCounts;
+/** Equal-mass bins; fewer than 10 instances cannot form a valid ECE bin. */
+export declare function expectedCalibrationError(records: readonly Pick<ScoredRecord, 'p' | 'good' | 'missing'>[]): number | null;
+/** Every selected lineage contributes all its instances, including repeated draws. */
+export declare function groupBootstrap(records: readonly ScoredRecord[], seed: string): {
+    eceInterval: Interval | null;
+    brierInterval: Interval;
+};
+export declare function calibrationMetrics(records: readonly ScoredRecord[], baseRate: number, seed: string, gate?: boolean): CalibrationMetrics;
+export declare function evidenceFailures(counts: EvidenceCounts): string[];
+export declare function calibrationBars(metrics: CalibrationMetrics): BarResult;
+/** The same independent evidence/accuracy bar applies to every forced teacher route. */
+export declare function teacherAccuracyFailures(metrics: EvidenceCounts & {
+    agreement: number;
+}): string[];
+export interface TeacherAgreementRecord {
+    group: string;
+    good: boolean;
+    goldenAnswer: string | number;
+    teacherAnswer?: string | number;
+    /** Labeler identities contributing to the golden answer, excluding disputed labels. */
+    goldenLabelers: readonly string[];
+}
+export declare function teacherAgreement(records: readonly TeacherAgreementRecord[], teacherId: string): BarResult & EvidenceCounts & {
+    agreement: number;
+    excludedSelfLabels: number;
+};

package/dist/training/graders/calibration/metrics.js

@@ -0,0 +1,145 @@
+import { createHash } from 'node:crypto';
+/** Two-sided 90% Wilson score interval; no observations cannot establish a bound. */
+export function wilson90(successes, total) {
+    if (!Number.isInteger(total) || total < 0 || !Number.isInteger(successes) || successes < 0 || successes > total) {
+        throw new Error('Wilson counts: expected 0 <= successes <= total integers');
+    }
+    if (!total)
+        return [0, 1];
+    const z = 1.6448536269514722, p = successes / total, denominator = 1 + z * z / total;
+    const centre = (p + z * z / (2 * total)) / denominator;
+    const radius = z * Math.sqrt(p * (1 - p) / total + z * z / (4 * total * total)) / denominator;
+    return [Math.max(0, centre - radius), Math.min(1, centre + radius)];
+}
+export function seededRandom(seed) {
+    let state = createHash('sha256').update(seed).digest().readUInt32LE(0);
+    return () => {
+        state = (state + 0x6D2B79F5) | 0;
+        let value = Math.imul(state ^ (state >>> 15), 1 | state);
+        value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value;
+        return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
+    };
+}
+export function evidenceCounts(records) {
+    const good = records.filter(record => record.good).length;
+    return { groups: new Set(records.map(record => record.group)).size, instances: records.length,
+        good, bad: records.length - good };
+}
+/** Equal-mass bins; fewer than 10 instances cannot form a valid ECE bin. */
+export function expectedCalibrationError(records) {
+    const observed = records.filter(row => !row.missing);
+    const missing = records.length - observed.length;
+    if (observed.length < 10)
+        return null;
+    const sorted = [...observed].sort((a, b) => a.p - b.p);
+    const bins = Math.min(10, Math.floor(sorted.length / 10));
+    let total = missing;
+    for (let index = 0; index < bins; index++) {
+        const bin = sorted.slice(Math.floor(index * sorted.length / bins), Math.floor((index + 1) * sorted.length / bins));
+        total += Math.abs(bin.reduce((sum, row) => sum + row.p - Number(row.good), 0));
+    }
+    return total / records.length;
+}
+function brier(records) {
+    return records.length ? records.reduce((sum, row) => sum + (row.missing ? 1 : (row.p - Number(row.good)) ** 2), 0) / records.length : 1;
+}
+function percentile(values, fraction) {
+    values.sort((a, b) => a - b);
+    const position = (values.length - 1) * fraction;
+    const low = Math.floor(position), high = Math.ceil(position);
+    return values[low] + (values[high] - values[low]) * (position - low);
+}
+/** Every selected lineage contributes all its instances, including repeated draws. */
+export function groupBootstrap(records, seed) {
+    if (!records.length)
+        return { eceInterval: null, brierInterval: [1, 1] };
+    const byGroup = new Map();
+    for (const record of records) {
+        const group = byGroup.get(record.group) ?? [];
+        group.push(record);
+        byGroup.set(record.group, group);
+    }
+    const groups = [...byGroup.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([, rows]) => rows);
+    const random = seededRandom(seed), eces = [], briers = [];
+    for (let iteration = 0; iteration < 1000; iteration++) {
+        const sample = [];
+        for (let draw = 0; draw < groups.length; draw++)
+            sample.push(...groups[Math.floor(random() * groups.length)]);
+        const ece = expectedCalibrationError(sample);
+        if (ece !== null)
+            eces.push(ece);
+        briers.push(brier(sample));
+    }
+    return { eceInterval: expectedCalibrationError(records) === null || eces.length !== 1000 ? null
+            : [percentile(eces, 0.05), percentile(eces, 0.95)],
+        brierInterval: [percentile(briers, 0.05), percentile(briers, 0.95)] };
+}
+export function calibrationMetrics(records, baseRate, seed, gate = false) {
+    if (!Number.isFinite(baseRate) || baseRate < 0 || baseRate > 1)
+        throw new Error('baseRate: expected probability');
+    if (records.some(record => !record.missing && (!Number.isFinite(record.p) || record.p < 0 || record.p > 1))) {
+        throw new Error('p: expected finite probability');
+    }
+    const counts = evidenceCounts(records);
+    return { ...counts, baseRate,
+        accuracy: records.length ? records.filter(row => !row.missing && (row.p >= 0.5) === row.good).length / records.length : 0,
+        brier: brier(records),
+        baseRateBrier: brier(records.map(row => ({ ...row, p: baseRate, missing: false }))),
+        ece: expectedCalibrationError(records), ...groupBootstrap(records, seed),
+        ...(gate ? { falsePassRate: counts.bad ? records.filter(row => !row.missing && row.p >= 0.5 && !row.good).length / counts.bad : 0,
+            falsePassInterval: wilson90(records.filter(row => !row.missing && row.p >= 0.5 && !row.good).length, counts.bad) } : {}),
+    };
+}
+export function evidenceFailures(counts) {
+    const failures = [];
+    if (counts.groups < 15)
+        failures.push('evidence.groups >= 15');
+    if (counts.instances < 30)
+        failures.push('evidence.instances >= 30');
+    if (counts.good < 5)
+        failures.push('evidence.good >= 5');
+    if (counts.bad < 5)
+        failures.push('evidence.bad >= 5');
+    return failures;
+}
+export function calibrationBars(metrics) {
+    const evidence = evidenceFailures(metrics);
+    if (metrics.ece === null || metrics.eceInterval === null)
+        evidence.push('evidence.eceBins >= 1 (including bootstrap resamples)');
+    if (evidence.length)
+        return { status: 'insufficient-evidence', failures: evidence };
+    const failures = [];
+    if (!(metrics.ece <= 0.10))
+        failures.push('calibration.ece <= 0.10');
+    if (!(metrics.eceInterval[1] <= 0.15))
+        failures.push('calibration.ece90Upper <= 0.15');
+    if (!(metrics.brier <= 0.20))
+        failures.push('calibration.brier <= 0.20');
+    if (!(metrics.brier < metrics.baseRateBrier))
+        failures.push('calibration.brier < baseRateBrier');
+    if (metrics.falsePassRate !== undefined && !(metrics.falsePassRate <= 0.10))
+        failures.push('gate.falsePassRate <= 0.10');
+    if (metrics.falsePassRate !== undefined && !(metrics.falsePassInterval?.[1] !== undefined && metrics.falsePassInterval[1] <= 0.20)) {
+        failures.push('gate.falsePass90Upper <= 0.20');
+    }
+    return { status: failures.length ? 'uncalibrated' : 'calibrated', failures };
+}
+/** The same independent evidence/accuracy bar applies to every forced teacher route. */
+export function teacherAccuracyFailures(metrics) {
+    const failures = evidenceFailures(metrics).map(failure => `teacher.${failure}`);
+    if (metrics.agreement < 0.85)
+        failures.push('teacher.agreement >= 0.85');
+    return failures;
+}
+export function teacherAgreement(records, teacherId) {
+    // A consensus containing this teacher must be rederived from other labelers by the caller.
+    const independent = records.filter(row => row.goldenLabelers.length > 0 && !row.goldenLabelers.includes(teacherId));
+    const counts = evidenceCounts(independent);
+    const agreement = independent.length
+        ? independent.filter(row => row.teacherAnswer !== undefined && row.teacherAnswer === row.goldenAnswer).length / independent.length : 0;
+    const failures = teacherAccuracyFailures({ ...counts, agreement });
+    const insufficient = evidenceFailures(counts).length > 0;
+    return { ...counts, agreement, excludedSelfLabels: records.length - independent.length, failures,
+        status: insufficient ? 'insufficient-evidence' : failures.length ? 'uncalibrated' : 'calibrated' };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/training/graders/calibration/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../../src/training/graders/calibration/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,qFAAqF;AACrF,MAAM,UAAU,QAAQ,CAAC,SAAiB,EAAE,KAAa;IACvD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,KAAK,EAAE,CAAC;QAChH,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1B,MAAM,CAAC,GAAG,kBAAkB,EAAE,CAAC,GAAG,SAAS,GAAG,KAAK,EAAE,WAAW,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACrF,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,WAAW,CAAC;IACvD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,WAAW,CAAC;IAC9F,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,GAAG,EAAE;QACV,KAAK,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;QACzD,KAAK,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC;QACvE,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IACvD,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAwD;IACrF,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC1D,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM;QAC3F,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,wBAAwB,CAAC,OAAgE;IACvG,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IACjD,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;IAC1D,IAAI,KAAK,GAAG,OAAO,CAAC;IACpB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;QACnH,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;AAChC,CAAC;AAED,SAAS,KAAK,CAAC,OAAgE;IAC7E,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1I,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB,EAAE,QAAgB;IACpD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC,GAAG,CAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAE,GAAG,MAAM,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;AAC1E,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,cAAc,CAAC,OAAgC,EAAE,IAAY;IAG3E,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IACzE,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IACrG,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,GAAa,EAAE,EAAE,MAAM,GAAa,EAAE,CAAC;IAC9E,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QACtD,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAE,CAAC,CAAC;QAC/G,MAAM,GAAG,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,GAAG,KAAK,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,wBAAwB,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI;YAC7F,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClD,aAAa,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;AAC1E,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAgC,EAAE,QAAgB,EAAE,IAAY,EACjG,IAAI,GAAG,KAAK;IACZ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAClH,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5G,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACvC,OAAO,EAAE,GAAG,MAAM,EAAE,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACzH,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC;QACrB,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACnF,GAAG,EAAE,wBAAwB,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC;QACxE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/H,iBAAiB,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3H,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAsB;IACrD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/D,IAAI,MAAM,CAAC,SAAS,GAAG,EAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACrE,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACzD,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACvD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAA2B;IACzD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,OAAO,CAAC,GAAG,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI;QAAE,QAAQ,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACjI,IAAI,QAAQ,CAAC,MAAM;QAAE,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACpF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,CAAC,CAAC,OAAO,CAAC,GAAI,IAAI,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACtE,IAAI,CAAC,CAAC,OAAO,CAAC,WAAY,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACxF,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACzE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IACjG,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACzH,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QACnI,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,EAAE,QAAQ,EAAE,CAAC;AAC/E,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,uBAAuB,CAAC,OAA+C;IACrF,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC;IAChF,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI;QAAE,QAAQ,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACzE,OAAO,QAAQ,CAAC;AAClB,CAAC;AAWD,MAAM,UAAU,gBAAgB,CAAC,OAA0C,EAAE,SAAiB;IAE5F,2FAA2F;IAC3F,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IACpH,MAAM,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM;QAClC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACzI,MAAM,QAAQ,GAAG,uBAAuB,CAAC,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IACnE,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACzD,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,QAAQ;QAC9F,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;AACvG,CAAC"}

package/dist/training/graders/calibration/selection.d.ts

@@ -0,0 +1,13 @@
+import type { CalibrationRecord, PlattParams } from './types.js';
+export interface ModelFitRows {
+    id: string;
+    teacher: boolean;
+    fit: readonly CalibrationRecord[];
+}
+/** Calibrated mean binary NLL, with the entire golden fit denominator retained.
+ * A missing answer has maximum binary error, as in held-out metrics: truth
+ * probability zero, clipped to 1e-6 for finite NLL. Never impute p(good)=0.
+ */
+export declare function fitNll(rows: readonly CalibrationRecord[], params: PlattParams): number;
+/** Strict improvement preserves authored order on ties, including no fit evidence. */
+export declare function selectQuestionModel(questionId: string, type: CalibrationRecord['type'], candidates: readonly ModelFitRows[]): string;

package/dist/training/graders/calibration/selection.js

@@ -0,0 +1,30 @@
+import { applyPlatt, fitQuestion } from './fitting.js';
+/** Calibrated mean binary NLL, with the entire golden fit denominator retained.
+ * A missing answer has maximum binary error, as in held-out metrics: truth
+ * probability zero, clipped to 1e-6 for finite NLL. Never impute p(good)=0.
+ */
+export function fitNll(rows, params) {
+    if (!rows.length)
+        return Infinity;
+    return rows.reduce((sum, row) => {
+        const p = row.missing ? 0 : applyPlatt(row.rawPGood, params);
+        const truth = row.missing ? 0 : row.good ? p : 1 - p;
+        return sum - Math.log(Math.max(1e-6, Math.min(1 - 1e-6, truth)));
+    }, 0) / rows.length;
+}
+/** Strict improvement preserves authored order on ties, including no fit evidence. */
+export function selectQuestionModel(questionId, type, candidates) {
+    if (!candidates.length)
+        throw new Error('Model selection requires candidates');
+    let selected = candidates[0].id, lowest = Infinity;
+    for (const candidate of candidates) {
+        const fit = fitQuestion(candidate.fit, questionId, type);
+        const loss = fitNll(fit.questionSplit, candidate.teacher ? { a: 1, b: 0 } : fit.params);
+        if (loss < lowest) {
+            selected = candidate.id;
+            lowest = loss;
+        }
+    }
+    return selected;
+}
+//# sourceMappingURL=selection.js.map

package/dist/training/graders/calibration/selection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"selection.js","sourceRoot":"","sources":["../../../../src/training/graders/calibration/selection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AASvD;;;GAGG;AACH,MAAM,UAAU,MAAM,CAAC,IAAkC,EAAE,MAAmB;IAC5E,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,QAAQ,CAAC;IAClC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACrD,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;AACtB,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,mBAAmB,CAAC,UAAkB,EAAE,IAA+B,EAAE,UAAmC;IAC1H,IAAI,CAAC,UAAU,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC/E,IAAI,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,CAAC;IACpD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,WAAW,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACxF,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;YAAC,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC;QAAC,CAAC;IAChE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}