spawnfile 0.1.25 → 0.1.27

package/dist/training/cli/grader/calibrate.d.ts

@@ -0,0 +1,19 @@
+import type { DecisionAccounting } from '../../contracts/grading.js';
+import { type CalibrationFile } from '../../graders/index.js';
+import { type GraderDependencies } from './backend.js';
+export interface CalibrateOptions {
+    graderFile: string;
+    excludeDatasets?: string[];
+    seed?: string;
+    reportFile?: string;
+    concurrency?: number;
+    signal?: AbortSignal;
+    maxSpendUsd?: number;
+}
+export declare function runCalibrate(options: CalibrateOptions, dependencies?: GraderDependencies): Promise<{
+    calibrationFile: string;
+    reportFile: string;
+    calibration: CalibrationFile;
+    code: 0 | 1;
+    decisionAccounting?: DecisionAccounting;
+}>;

package/dist/training/cli/grader/calibrate.js

@@ -0,0 +1,237 @@
+import { mkdir, writeFile, rename } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { withDecisionFailureAccounting, summarizeDecisionAccounting, allQuestions, applyPlatt, calibrateQuestions, calibrationMetrics, createCalibrationIdentity, decisionModels, graderForModel, selectQuestionModel, stableDigest, evidenceFailures, expectedOptions, inEscalationBand, loadGrader, loadLabels, parseCalibration, planEscalation, splitCalibration, teacherAgreement } from '../../graders/index.js';
+import { validateStoredVotes } from '../../graders/labeling/index.js';
+import { DECISION_PROMPT_VERSION } from '../../inference/decision/index.js';
+import { backendIdentity, graderBackend, withDecisionBudget } from './backend.js';
+import { estimateEscalationCost } from './cost.js';
+import { cachedScores } from './cache.js';
+import { excludeLabels, readExclusions } from './exclusions.js';
+import { goldenInstances, indexDecisionResults, rawGood, calibrationRecord, cascadeRecord } from './records.js';
+import { calibrationReport } from './report.js';
+export async function runCalibrate(options, dependencies = {}) {
+    const backends = new Set();
+    const receipts = [];
+    return withDecisionFailureAccounting(backends, receipts, () => calibrate(options, dependencies, backends, receipts));
+}
+async function calibrate(options, dependencies, backends, receipts) {
+    const grader = await loadGrader(options.graderFile);
+    if (grader.calibration === 'identity')
+        throw new Error('Identity mode has no fitted calibration; select a calibration file first');
+    dependencies = withDecisionBudget(grader, { ...dependencies, maxSpendUsd: options.maxSpendUsd ?? dependencies.maxSpendUsd });
+    const loaded = await loadLabels(grader.golden, grader);
+    validateStoredVotes(grader, loaded.labels, { resolved: true });
+    const exclusions = await readExclusions(options.excludeDatasets ?? []);
+    const labels = { ...loaded.labels, labels: excludeLabels(loaded.labels.labels, exclusions) };
+    const seed = options.seed ?? '0';
+    const gates = grader.gates.map(gate => gate.id);
+    const models = decisionModels(grader);
+    const commonGolden = goldenInstances(grader, labels);
+    // Candidates share device capacity: finish one backend before starting the next.
+    // Each still owns its identity, cache and worker bound, in authored order.
+    const scoreCandidate = async (model) => {
+        const definition = graderForModel(grader, model);
+        try {
+            options.signal?.throwIfAborted();
+            const backend = graderBackend(definition, dependencies);
+            backends.add(backend);
+            const identity = await backendIdentity(definition, backend, options.signal, receipts);
+            const golden = goldenInstances(definition, labels);
+            const results = await cachedScores(dirname(options.graderFile), backend, stableDigest({ model: model.id, identity }), golden.rows.map(row => row.request), options.concurrency ?? 2, options.signal, receipts);
+            indexDecisionResults(golden.rows.map(row => row.request), results);
+            const valid = golden.rows.map((row, index) => {
+                const raw = rawGood(row, results[index]);
+                return { row, record: { ...calibrationRecord(row, raw ?? 0), ...(raw === undefined ? { missing: true } : {}) } };
+            });
+            const records = valid.map(item => item.record);
+            const split = splitCalibration(records, seed);
+            // Selection shares the full-panel denominator. Teacher-only independent
+            // disputes are charged missing, never removed to improve a candidate's loss.
+            const selectionRecords = commonGolden.rows.map((row, index) => ({
+                ...calibrationRecord(row, valid[index].record.rawPGood),
+                ...(valid[index].record.missing || valid[index].record.disputed ? { missing: true } : {}),
+            }));
+            return { model, definition, backend, identity, golden, results, valid, split,
+                selection: { id: model.id, teacher: model.backend === 'teacher', fit: splitCalibration(selectionRecords, seed).fit },
+                fitted: model.backend === 'teacher' ? [] : calibrateQuestions(records, { seed, gates }) };
+        }
+        catch (cause) {
+            throw new Error(`Decision model ${model.id}: ${cause instanceof Error ? cause.message : 'calibration failed'}`, { cause });
+        }
+    };
+    const candidates = [];
+    for (const model of models)
+        candidates.push(await scoreCandidate(model));
+    const selected = new Map(allQuestions(grader).map(question => {
+        const id = selectQuestionModel(question.id, question.type, candidates.map(candidate => candidate.selection));
+        return [question.id, candidates.find(candidate => candidate.model.id === id)];
+    }));
+    const valid = commonGolden.rows.map((row, index) => selected.get(row.question.id).valid[index]);
+    const golden = { ...commonGolden, rows: valid.map(item => item.row) };
+    const observations = golden.rows.map(row => ({ ...row, outputId: row.label.id }));
+    const results = golden.rows.map((row, index) => selected.get(row.question.id).results[index]);
+    const byKey = indexDecisionResults(observations.map(row => row.request), results);
+    const records = valid.map(item => item.record);
+    const split = splitCalibration(records, seed);
+    const fitted = allQuestions(grader).flatMap(question => selected.get(question.id).fitted.filter(fit => fit.questionId === question.id));
+    const localQuestions = allQuestions(grader).filter(question => selected.get(question.id).model.backend !== 'teacher');
+    const questions = {};
+    const diagnostics = {};
+    const cascades = new Map();
+    for (const item of valid) {
+        const fit = fitted.find(fit => fit.questionId === item.record.questionId);
+        const p = fit && !item.record.missing ? applyPlatt(item.record.rawPGood, fit.params) : item.record.rawPGood;
+        const cascade = cascadeRecord(item.row, item.record, p, labels.labelers);
+        const vote = item.row.label.votes[item.row.instance.key]?.[grader.escalate.to];
+        if (vote !== undefined && vote !== 'abstain')
+            cascade.teacherGood = expectedOptions(item.row.question).includes(vote);
+        cascades.set(item.record, cascade);
+    }
+    const plans = new Map();
+    let costCalls = [];
+    const forced = new Set();
+    const plan = (id, calls) => planEscalation(split.fit.filter(row => row.questionId === id).map(row => cascades.get(row)), split.heldOut.filter(row => row.questionId === id).map(row => cascades.get(row)), grader.escalate, { teacherId: grader.escalate.to, maxCallsPerTrial: grader.escalate.max_calls_per_trial, gate: gates.includes(id), calls, forceTeacher: forced.has(id) });
+    if (localQuestions.length) {
+        for (const question of localQuestions)
+            plans.set(question.id, plan(question.id));
+        const heldOutputs = new Set(valid.filter(item => split.heldOut.includes(item.record)).map(item => item.row.label.id));
+        const failedPlans = new Map();
+        // A newly failing cascade becomes forced-teacher and can consume earlier calls.
+        // The set only grows, so at most one round per question is needed to settle routing.
+        for (let round = 0; round <= plans.size; round++) {
+            const before = forced.size;
+            if (grader.uncalibrated === 'teacher') {
+                for (const [id, candidate] of plans) {
+                    if (fitted.find(fit => fit.questionId === id)?.status !== 'calibrated' || candidate.status !== 'calibrated') {
+                        forced.add(id);
+                        if (!failedPlans.has(id))
+                            failedPlans.set(id, candidate);
+                    }
+                }
+            }
+            const calls = observations.filter(row => {
+                if (!heldOutputs.has(row.outputId))
+                    return false;
+                if (forced.has(row.question.id))
+                    return true;
+                const band = plans.get(row.question.id)?.band;
+                const raw = rawGood(row, byKey.get(row.request.key));
+                const fit = fitted.find(fit => fit.questionId === row.question.id);
+                return raw !== undefined && band && fit && inEscalationBand(applyPlatt(raw, fit.params), band);
+            }).map(row => ({ outputId: row.outputId, projectionDigest: row.instance.projectionDigest, instanceKeys: [row.instance.key] }));
+            costCalls = calls;
+            for (const question of localQuestions) {
+                const updated = plan(question.id, calls), failure = failedPlans.get(question.id);
+                plans.set(question.id, failure ? { ...updated, status: failure.status === 'calibrated' ? updated.status : failure.status,
+                    failures: [...updated.failures, ...(failure.status === 'calibrated' ? [] : ['local cascade requires forced-teacher routing'])] } : updated);
+            }
+            if (grader.uncalibrated !== 'teacher' || before === forced.size &&
+                [...plans].every(([id, candidate]) => forced.has(id) || candidate.status === 'calibrated'))
+                break;
+        }
+    }
+    for (const question of allQuestions(grader)) {
+        const candidate = selected.get(question.id);
+        const { model, backend, identity } = candidate;
+        const id = question.id, sample = golden.rows.filter(row => row.question.id === id && !row.golden.disputed);
+        const indices = golden.rows.flatMap((row, index) => row.question.id === id && !row.golden.disputed ? [index] : []);
+        diagnostics[id] = { candidates: candidates.map(candidate => ({ id: candidate.model.id,
+                ece: candidate.fitted.find(fit => fit.questionId === id)?.metrics.ece ?? null,
+                brier: candidate.fitted.find(fit => fit.questionId === id)?.metrics.brier ?? null })), missingRate: sample.length ? indices.filter(i => results[i].status === 'missing').length / sample.length : 0,
+            boundedRate: sample.length ? indices.filter(i => results[i].bounded).length / sample.length : 0,
+            sourceMix: { human: 0, editor: 0, consensus: 0, 'single-teacher': 0 } };
+        for (const row of sample)
+            diagnostics[id].sourceMix[row.golden.source]++;
+        if (model.backend === 'teacher') {
+            // Teacher-only scoring uses the same pre-dispute held-out population.
+            const held = new Set(split.heldOut);
+            const teacherId = model.teacher;
+            const agreementRows = golden.rows.flatMap((row, index) => {
+                if (row.question.id !== id || row.golden.disputed || !held.has(valid[index].record))
+                    return [];
+                const result = results[index];
+                const answer = result.status === 'ok' ? row.request.optionLabels.find(option => result.probabilities?.[String(option)] === 1) : undefined;
+                return [{ source: row.golden.source, group: row.label.lineage, good: expectedOptions(row.question).includes(row.golden.answer),
+                        goldenAnswer: row.golden.answer, teacherAnswer: answer,
+                        goldenLabelers: Object.keys(row.golden.votes).filter(name => name !== teacherId) }];
+            });
+            const agreement = teacherAgreement(agreementRows, teacherId);
+            const heldCascade = valid.flatMap((item, index) => {
+                if (item.record.questionId !== id || !held.has(item.record))
+                    return [];
+                const result = results[index];
+                const answer = result.status === 'ok' ? item.row.request.optionLabels.find(option => result.probabilities?.[String(option)] === 1) : undefined;
+                const cascade = cascades.get(item.record);
+                return [{ ...cascade, votes: { ...cascade.votes, [teacherId]: answer ?? 'abstain' } }];
+            });
+            const escalation = planEscalation([], heldCascade, grader.escalate, { teacherId, forceTeacher: true, gate: gates.includes(id) });
+            diagnostics[id].sourceAgreement = Object.fromEntries(['human', 'editor', 'consensus', 'single-teacher'].map(source => {
+                const subset = agreementRows.filter(row => row.source === source);
+                const measured = teacherAgreement(subset, teacherId);
+                return [source, { instances: measured.instances, agreement: measured.agreement }];
+            }));
+            questions[id] = { model: model.id, status: agreement.status === 'calibrated' ? escalation.status : agreement.status,
+                failures: [...agreement.failures, ...escalation.failures], teacherAccuracy: escalation.teacherReport, metrics: {
+                    cascadeAccuracy: escalation.heldOutAccuracy, cascadeInterval: escalation.cascadeInterval, coverage: escalation.coverage,
+                    groups: agreement.groups, instances: agreement.instances, good: agreement.good, bad: agreement.bad,
+                    baseRate: agreement.instances ? agreement.good / agreement.instances : 0, agreement: agreement.agreement,
+                } };
+            continue;
+        }
+        const fit = fitted.find(fit => fit.questionId === id);
+        if (!fit) {
+            const metrics = calibrationMetrics([], 0, seed, gates.includes(id));
+            questions[id] = { model: model.id, status: 'insufficient-evidence', failures: evidenceFailures(metrics), metrics, teacherAccuracy: plans.get(id).teacherReport };
+            continue;
+        }
+        const escalation = plans.get(id);
+        questions[id] = { model: model.id, status: fit.status === 'calibrated' ? escalation.status : fit.status,
+            failures: [...fit.failures, ...escalation.failures], params: fit.params, teacherAccuracy: escalation.teacherReport,
+            ...(escalation.band ? { band: [...escalation.band] } : {}), metrics: { ...fit.metrics,
+                cascadeAccuracy: escalation.heldOutAccuracy, cascadeInterval: escalation.cascadeInterval, escalationRate: escalation.escalationRate, coverage: escalation.coverage } };
+        diagnostics[id].bySource = fit.bySource;
+        // A fixed sample observes positional sensitivity, without selecting or modifying the prompt.
+        if (model.backend === 'ollama-logprobs') {
+            const chosen = indices.slice(0, 16);
+            const reversed = await cachedScores(dirname(options.graderFile), backend, stableDigest({ model: model.id, identity }), chosen.map(index => ({ ...golden.rows[index].request, optionLabels: [...golden.rows[index].request.optionLabels].reverse() })), options.concurrency ?? 2, options.signal, receipts);
+            let compared = 0, flips = 0;
+            reversed.forEach((result, index) => {
+                const original = rawGood(golden.rows[chosen[index]], results[chosen[index]]);
+                const reverse = rawGood(golden.rows[chosen[index]], result);
+                if (original !== undefined && reverse !== undefined) {
+                    compared++;
+                    if ((original >= 0.5) !== (reverse >= 0.5))
+                        flips++;
+                }
+            });
+            diagnostics[id].permutation = { compared, missing: chosen.length - compared, flipRate: compared ? flips / compared : null };
+        }
+    }
+    for (const candidate of candidates) {
+        if (await backendIdentity(candidate.definition, candidate.backend, options.signal, receipts) !== candidate.identity) {
+            throw new Error(`Backend identity changed during calibration: ${candidate.model.id}`);
+        }
+    }
+    const modelIdentities = Object.fromEntries(candidates.map(candidate => [candidate.model.id, candidate.identity]));
+    const calibration = parseCalibration({ schema: 'spawnfile.training.grader-calibration.v1', questions,
+        identity: createCalibrationIdentity(grader, { backendModel: grader.models ? stableDigest(modelIdentities) : candidates[0].identity,
+            ...(grader.models ? { modelIdentities } : {}), golden: loaded.goldenDigest,
+            promptTemplateVersion: DECISION_PROMPT_VERSION, seed, exclusions: exclusions.identities }) });
+    const reportFile = options.reportFile ?? `${grader.calibration}.md`;
+    options.signal?.throwIfAborted();
+    await mkdir(dirname(grader.calibration), { recursive: true });
+    const temporary = `${grader.calibration}.${process.pid}.tmp`;
+    await writeFile(temporary, `${JSON.stringify(calibration, null, 2)}\n`, { mode: 0o600 });
+    await rename(temporary, grader.calibration);
+    await mkdir(dirname(reportFile), { recursive: true });
+    const decisionAccounting = candidates.some(candidate => candidate.backend.capabilities?.billing === 'api')
+        ? summarizeDecisionAccounting(receipts) : undefined;
+    const missingReasons = candidates.flatMap(candidate => candidate.results.filter(result => result.status === 'missing')
+        .map(result => ({ model: candidate.model.id, key: result.key, reason: result.reason ?? 'missing' })));
+    const cost = estimateEscalationCost(costCalls, labels, grader.escalate.to, grader.escalate.max_calls_per_trial);
+    await writeFile(reportFile, calibrationReport(calibration, diagnostics, { excluded: loaded.labels.labels.length - labels.labels.length,
+        disputed: golden.disputed, unavailable: golden.unavailable }) + `\nEscalation cost estimate (per represented escalated output; null = unknown):\n\n\`\`\`json\n${JSON.stringify({ escalation: cost, decisionAccounting, missingDecisions: missingReasons }, null, 2)}\n\`\`\`\n`, { mode: 0o600 });
+    return { calibrationFile: grader.calibration, reportFile, calibration, decisionAccounting,
+        code: Object.values(questions).every(question => question.status === 'calibrated') ? 0 : 1 };
+}
+//# sourceMappingURL=calibrate.js.map

package/dist/training/cli/grader/calibrate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibrate.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/calibrate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,6BAA6B,EAAE,2BAA2B,EAAE,YAAY,EAAE,UAAU,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,yBAAyB,EAC9J,cAAc,EAAE,cAAc,EAAE,mBAAmB,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,gBAAgB,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAChK,cAAc,EAAE,gBAAgB,EAAE,gBAAgB,EACf,MAAM,wBAAwB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAA2B,MAAM,cAAc,CAAC;AAC3G,OAAO,EAAE,sBAAsB,EAA2B,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAChH,OAAO,EAAE,iBAAiB,EAA4B,MAAM,aAAa,CAAC;AAO1E,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAyB,EAAE,eAAmC,EAAE;IACjG,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC5C,MAAM,QAAQ,GAA6B,EAAE,CAAC;IAC9C,OAAO,6BAA6B,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACvH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE,YAAgC,EAAE,QAA8B,EAAE,QAAkC;IAGtJ,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;IACnI,YAAY,GAAG,kBAAkB,CAAC,MAAM,EAAE,EAAE,GAAG,YAAY,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC;IAC7H,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvD,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC;IAC7F,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,GAAG,CAAC;IACjC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrD,iFAAiF;IACjF,2EAA2E;IAC3E,MAAM,cAAc,GAAG,KAAK,EAAE,KAA8B,EAAE,EAAE;QAC9D,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACjD,IAAI,CAAC;YACH,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,aAAa,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;YACxD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACtF,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAClH,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC3F,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAE,CAAC,CAAC;gBAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnH,CAAC,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC9C,wEAAwE;YACxE,6EAA6E;YAC7E,MAAM,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC9D,GAAG,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACxD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5F,CAAC,CAAC,CAAC;YACJ,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;gBAC1E,SAAS,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,GAAG,EAAE,gBAAgB,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE;gBACpH,MAAM,EAAE,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC9F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,CAAC,EAAE,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAAC,CAAC;IACjJ,CAAC,CAAC;IACF,MAAM,UAAU,GAAiD,EAAE,CAAC;IACpE,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,UAAU,CAAC,IAAI,CAAC,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;QAC3D,MAAM,EAAE,GAAG,mBAAmB,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC7G,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAE,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC,CAAC;IACJ,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAE,CAAC,KAAK,CAAC,KAAK,CAAE,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,EAAE,GAAG,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACtE,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAE,CAAC,OAAO,CAAC,KAAK,CAAE,CAAC,CAAC;IAChG,MAAM,KAAK,GAAG,oBAAoB,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IAClF,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACzI,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAE,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;IACvH,MAAM,SAAS,GAAiC,EAAE,CAAC;IACnD,MAAM,WAAW,GAAwC,EAAE,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC1E,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC5G,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QACzE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC/E,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,CAAC,WAAW,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtH,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,IAAI,SAAS,GAAyB,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,IAAI,GAAG,CAAC,EAAU,EAAE,KAAwD,EAAE,EAAE,CAAC,cAAc,CACnG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,EAC7E,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,EAAE,MAAM,CAAC,QAAQ,EAClG,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3J,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;QAC1B,KAAK,MAAM,QAAQ,IAAI,cAAc;YAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACtH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAoB,CAAC;QAChD,gFAAgF;QAChF,qFAAqF;QACrF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC;YAC3B,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBACtC,KAAK,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,KAAK,EAAE,CAAC;oBACpC,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,YAAY,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;wBAC5G,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACf,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;4BAAE,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;gBACtC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC;oBAAE,OAAO,KAAK,CAAC;gBACjD,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAAE,OAAO,IAAI,CAAC;gBAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC;gBAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAE,CAAC,CAAC;gBACtD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACnE,OAAO,GAAG,KAAK,SAAS,IAAI,IAAI,IAAI,GAAG,IAAI,gBAAgB,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACjG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,gBAAgB,EAAE,YAAY,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/H,SAAS,GAAG,KAAK,CAAC;YAClB,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;gBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACjF,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;oBACtH,QAAQ,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,+CAA+C,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAChJ,CAAC;YACD,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,IAAI,MAAM,KAAK,MAAM,CAAC,IAAI;gBAC7D,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,CAAC;gBAAE,MAAM;QACtG,CAAC;IACH,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAE,CAAC;QAC7C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;QAC/C,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3G,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnH,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE;gBACnF,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,IAAI;gBAC7E,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACpM,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAChG,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1E,KAAK,MAAM,GAAG,IAAI,MAAM;YAAE,WAAW,CAAC,EAAE,CAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1E,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAChC,sEAAsE;YACtE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC;YAChC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBACvD,IAAI,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC;oBAAE,OAAO,EAAE,CAAC;gBAChG,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC1I,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAO,CAAC;wBAC7H,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,MAAO,EAAE,aAAa,EAAE,MAAM;wBACvD,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC,CAAC;YACxF,CAAC,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,gBAAgB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;YAC7D,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBAChD,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;oBAAE,OAAO,EAAE,CAAC;gBACvE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC/I,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAE,CAAC;gBAC3C,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,IAAI,SAAS,EAAE,EAAE,CAAC,CAAC;YACzF,CAAC,CAAC,CAAC;YACH,MAAM,UAAU,GAAG,cAAc,CAAC,EAAE,EAAE,WAAW,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACjI,WAAW,CAAC,EAAE,CAAE,CAAC,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBACpH,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;gBAClE,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBACrD,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;YACpF,CAAC,CAAC,CAAC,CAAC;YACJ,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM;gBACjH,QAAQ,EAAE,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,eAAe,EAAE,UAAU,CAAC,aAAa,EAAE,OAAO,EAAE;oBAC/G,eAAe,EAAE,UAAU,CAAC,eAAe,EAAE,eAAe,EAAE,UAAU,CAAC,eAAe,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ;oBACvH,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG;oBAClG,QAAQ,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS;iBACzG,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,OAAO,GAAG,kBAAkB,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YACpE,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,uBAAuB,EAAE,QAAQ,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,aAAa,EAAE,CAAC;YAClK,SAAS;QACX,CAAC;QACD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QAClC,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM;YACrG,QAAQ,EAAE,CAAC,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,EAAE,UAAU,CAAC,aAAa;YAClH,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,GAAG,CAAC,OAAO;gBACnF,eAAe,EAAE,UAAU,CAAC,eAAe,EAAE,eAAe,EAAE,UAAU,CAAC,eAAe,EAAE,cAAc,EAAE,UAAU,CAAC,cAAc,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC3K,WAAW,CAAC,EAAE,CAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;QACzC,6FAA6F;QAC7F,IAAI,KAAK,CAAC,OAAO,KAAK,iBAAiB,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EACnH,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAChI,OAAO,CAAC,WAAW,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACtD,IAAI,QAAQ,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;YAC5B,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;gBACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAE,CAAE,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAE,CAAE,CAAC,CAAC;gBACjF,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAE,CAAE,EAAE,MAAM,CAAC,CAAC;gBAC9D,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAAC,QAAQ,EAAE,CAAC;oBAAC,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC;wBAAE,KAAK,EAAE,CAAC;gBAAC,CAAC;YAC3H,CAAC,CAAC,CAAC;YACH,WAAW,CAAC,EAAE,CAAE,CAAC,WAAW,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,GAAG,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/H,CAAC;IACH,CAAC;IACD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,MAAM,eAAe,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC;YACpH,MAAM,IAAI,KAAK,CAAC,gDAAgD,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IACD,MAAM,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAClH,MAAM,WAAW,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,0CAA0C,EAAE,SAAS;QAClG,QAAQ,EAAE,yBAAyB,CAAC,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC,QAAQ;YACjI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,YAAY;YAC1E,qBAAqB,EAAE,uBAAuB,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IAClG,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,MAAM,CAAC,WAAW,KAAK,CAAC;IACpE,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7D,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzF,MAAM,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IAC5C,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,kBAAkB,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,KAAK,KAAK,CAAC;QACxG,CAAC,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtD,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;SACnH,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACxG,MAAM,IAAI,GAAG,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAChH,MAAM,SAAS,CAAC,UAAU,EAAE,iBAAiB,CAAC,WAAW,EAAE,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM;QACpI,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,GAAG,iGAAiG,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,cAAc,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACrS,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,kBAAkB;QACvF,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACjG,CAAC"}

package/dist/training/cli/grader/cost.d.ts

@@ -0,0 +1,24 @@
+import { type Labels } from '../../graders/index.js';
+export interface EscalationCostCall {
+    outputId: string;
+    projectionDigest: string;
+    instanceKeys: string[];
+}
+export interface EscalationCostEstimate {
+    /** Outputs represented by requested escalation calls, not all dataset outputs. */
+    outputs: number;
+    requestedCalls: number;
+    projectedCalls: number;
+    overflowCalls: number;
+    overflowInstances: number;
+    averageCallsPerOutput: number;
+    matchingCalls: number;
+    meanInputTokens: number | null;
+    meanOutputTokens: number | null;
+    meanWallMs: number | null;
+    estimatedInputTokens: number | null;
+    estimatedOutputTokens: number | null;
+    estimatedWallMs: number | null;
+}
+/** Descriptive estimate only: missing receipts and unmatched call shapes never imply zero cost. */
+export declare function estimateEscalationCost(calls: readonly EscalationCostCall[], labels: Labels, teacherId: string, maxCallsPerTrial?: number): EscalationCostEstimate;

package/dist/training/cli/grader/cost.js

@@ -0,0 +1,54 @@
+import { voteProvenance } from '../../graders/labeling/index.js';
+import { stableDigest } from '../../graders/index.js';
+const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : undefined;
+const sumKnown = (values) => values.length && values.every(value => typeof value === 'number' && Number.isFinite(value) && value >= 0)
+    ? values.reduce((sum, value) => sum + value, 0) : null;
+/** Descriptive estimate only: missing receipts and unmatched call shapes never imply zero cost. */
+export function estimateEscalationCost(calls, labels, teacherId, maxCallsPerTrial = Number.MAX_SAFE_INTEGER) {
+    if (!Number.isSafeInteger(maxCallsPerTrial) || maxCallsPerTrial < 0)
+        throw new Error('Invalid maximum escalation calls');
+    const grouped = new Map();
+    for (const call of calls) {
+        if (!call.instanceKeys.length)
+            throw new Error('Escalation cost requires instance keys');
+        const key = JSON.stringify([call.outputId, call.projectionDigest]);
+        const existing = grouped.get(key);
+        grouped.set(key, { ...call, instanceKeys: [...new Set([...(existing?.instanceKeys ?? []), ...call.instanceKeys])].sort() });
+    }
+    const perOutput = new Map();
+    const admitted = [];
+    let overflowInstances = 0;
+    for (const call of grouped.values()) {
+        const count = perOutput.get(call.outputId) ?? 0;
+        perOutput.set(call.outputId, count + 1);
+        if (count < maxCallsPerTrial)
+            admitted.push(call);
+        else
+            overflowInstances += call.instanceKeys.length;
+    }
+    const measurements = admitted.flatMap(call => {
+        const label = labels.labels.find(label => label.id === call.outputId);
+        const entries = call.instanceKeys.map(key => voteProvenance(label, teacherId, key));
+        if (entries.some(entry => !entry || entry.projection_digest !== call.projectionDigest ||
+            !Array.isArray(entry.instance_keys) || stableDigest([...entry.instance_keys].sort()) !== stableDigest(call.instanceKeys)))
+            return [];
+        const attempts = entries[0].attempts;
+        if (!Array.isArray(attempts) || !attempts.length || entries.some(entry => !Array.isArray(entry.attempts) || stableDigest(entry.attempts) !== stableDigest(attempts)))
+            return [];
+        return [{ input: sumKnown(attempts.map(attempt => object(object(attempt)?.usage)?.inputTokens)),
+                output: sumKnown(attempts.map(attempt => object(object(attempt)?.usage)?.outputTokens)),
+                wall: sumKnown(attempts.map(attempt => object(attempt)?.wall_ms)) }];
+    });
+    const summarize = (field) => {
+        const total = sumKnown(measurements.map(measurement => measurement[field]));
+        return { mean: total === null ? null : total / measurements.length,
+            estimate: measurements.length === admitted.length ? total : null };
+    };
+    const input = summarize('input'), output = summarize('output'), wall = summarize('wall');
+    return { outputs: perOutput.size, requestedCalls: grouped.size, projectedCalls: admitted.length,
+        overflowCalls: grouped.size - admitted.length, overflowInstances,
+        averageCallsPerOutput: perOutput.size ? admitted.length / perOutput.size : 0,
+        matchingCalls: measurements.length, meanInputTokens: input.mean, meanOutputTokens: output.mean, meanWallMs: wall.mean,
+        estimatedInputTokens: input.estimate, estimatedOutputTokens: output.estimate, estimatedWallMs: wall.estimate };
+}
+//# sourceMappingURL=cost.js.map

package/dist/training/cli/grader/cost.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/cost.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EAAE,YAAY,EAAe,MAAM,wBAAwB,CAAC;AAqBnE,MAAM,MAAM,GAAG,CAAC,KAAc,EAA0B,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAmB,CAAC,CAAC,CAAC,SAAS,CAAC;AAClK,MAAM,QAAQ,GAAG,CAAC,MAAiB,EAAiB,EAAE,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IAC9J,CAAC,CAAE,MAAmB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAEvE,mGAAmG;AACnG,MAAM,UAAU,sBAAsB,CAAC,KAAoC,EAAE,MAAc,EAAE,SAAiB,EAC5G,gBAAgB,GAAG,MAAM,CAAC,gBAAgB;IAC1C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,IAAI,gBAAgB,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACzH,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACzF,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,YAAY,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,IAAI,EAAE,CAAC,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9H,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,MAAM,QAAQ,GAAyB,EAAE,CAAC;IAC1C,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChD,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,gBAAgB;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;YAC7C,iBAAiB,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IACrD,CAAC;IACD,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACpF,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,iBAAiB,KAAK,IAAI,CAAC,gBAAgB;YACnF,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,YAAY,CAAC,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QACvI,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAM,CAAC,QAAQ,CAAC,IAAI,YAAY,CAAC,KAAM,CAAC,QAAQ,CAAC,KAAK,YAAY,CAAC,QAAQ,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClL,OAAO,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC;gBAC7F,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,EAAE,YAAY,CAAC,CAAC;gBACvF,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,CAAC,KAAkC,EAAE,EAAE;QACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5E,OAAO,EAAE,IAAI,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,YAAY,CAAC,MAAM;YAChE,QAAQ,EAAE,YAAY,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACvE,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACzF,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,EAAE,cAAc,EAAE,OAAO,CAAC,IAAI,EAAE,cAAc,EAAE,QAAQ,CAAC,MAAM;QAC7F,aAAa,EAAE,OAAO,CAAC,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,iBAAiB;QAChE,qBAAqB,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5E,aAAa,EAAE,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI;QACrH,oBAAoB,EAAE,KAAK,CAAC,QAAQ,EAAE,qBAAqB,EAAE,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;AACnH,CAAC"}

package/dist/training/cli/grader/exclusions.d.ts

@@ -0,0 +1,12 @@
+import { type Label } from '../../graders/index.js';
+export interface DatasetExclusions {
+    identities: {
+        dataset: string;
+        digest: string;
+    }[];
+    ids: Set<string>;
+    examples: Set<string>;
+}
+/** Dataset groups are label lineages; example identities retain their dataset namespace. */
+export declare function readExclusions(files: readonly string[]): Promise<DatasetExclusions>;
+export declare function excludeLabels(labels: readonly Label[], exclusions: DatasetExclusions): Label[];

package/dist/training/cli/grader/exclusions.js

@@ -0,0 +1,49 @@
+import { canonicalWireId } from '../../contracts/index.js';
+import { readFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { parseDocument } from 'yaml';
+import { sha256 } from '../../graders/index.js';
+/** Dataset groups are label lineages; example identities retain their dataset namespace. */
+export async function readExclusions(files) {
+    const ids = new Set();
+    const examples = new Set();
+    const identities = [];
+    for (const file of [...new Set(files.map(file => resolve(file)))].sort()) {
+        const bytes = await readFile(file);
+        const document = parseDocument(bytes.toString(), { uniqueKeys: true });
+        if (document.errors.length || document.warnings.length)
+            throw new Error(`Invalid exclusion dataset: ${file}`);
+        const value = document.toJS({ maxAliasCount: 0 });
+        if (!value || typeof value !== 'object')
+            throw new Error(`Invalid exclusion dataset: ${file}`);
+        const data = value;
+        if (!['spawnfile.training.dataset.v1', 'spawnfile.dataset.v1'].includes(String(canonicalWireId(data.schema))))
+            throw new Error(`Unsupported exclusion dataset schema: ${file}`);
+        const records = data.records ?? data.examples;
+        if (!Array.isArray(records))
+            throw new Error(`Exclusion dataset requires records/examples: ${file}`);
+        for (const record of records) {
+            if (!record || typeof record !== 'object' || !('id' in record) || typeof record.id !== 'string' || !record.id.trim()) {
+                throw new Error(`Exclusion dataset requires example IDs: ${file}`);
+            }
+            ids.add(record.id);
+            if ('group' in record) {
+                if (typeof record.group !== 'string' || !record.group.trim())
+                    throw new Error(`Exclusion dataset requires nonempty groups: ${file}`);
+                ids.add(record.group);
+            }
+            examples.add(JSON.stringify([file, record.id]));
+            if (typeof data.id === 'string')
+                examples.add(JSON.stringify([data.id, record.id]));
+        }
+        identities.push({ dataset: file, digest: sha256(bytes) });
+    }
+    return { identities, ids, examples };
+}
+export function excludeLabels(labels, exclusions) {
+    const lineages = new Set(labels.filter(label => exclusions.ids.has(label.lineage) ||
+        (label.example && exclusions.examples.has(JSON.stringify([label.example.dataset, label.example.id]))))
+        .map(label => label.lineage));
+    return labels.filter(label => !lineages.has(label.lineage));
+}
+//# sourceMappingURL=exclusions.js.map

package/dist/training/cli/grader/exclusions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exclusions.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/exclusions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,MAAM,EAAc,MAAM,wBAAwB,CAAC;AAQ5D,4FAA4F;AAC5F,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAwB;IAC3D,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,UAAU,GAAoC,EAAE,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACvE,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,EAAE,CAAC,CAAC;QAC9G,MAAM,KAAK,GAAY,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,EAAE,CAAC,CAAC;QAC/F,MAAM,IAAI,GAAG,KAAgC,CAAC;QAC9C,IAAI,CAAC,CAAC,+BAA+B,EAAE,sBAAsB,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAC;QAChL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC;QAC9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,EAAE,CAAC,CAAC;QACrG,KAAK,MAAM,MAAM,IAAI,OAAoB,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,OAAO,MAAM,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;gBACrH,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnB,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;gBACtB,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,EAAE,CAAC,CAAC;gBACrI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAChD,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ;gBAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACtF,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAwB,EAAE,UAA6B;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;QAC/E,CAAC,KAAK,CAAC,OAAO,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;SACrG,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AAC9D,CAAC"}

package/dist/training/cli/grader/grade.d.ts

@@ -0,0 +1,26 @@
+import type { GraderDependencies } from './backend.js';
+export interface GradeOptions {
+    graderFile: string;
+    maxSpendUsd?: number;
+    outputFile: string;
+    contextFiles?: Readonly<Record<string, string>>;
+    signal?: AbortSignal;
+}
+/** Standalone grading has no trial snapshot; check gates therefore remain unknown. */
+export declare function runGrade(options: GradeOptions, dependencies?: GraderDependencies): Promise<import("../../graders/index.js").GradeResult>;
+/** Label builders use the same expansion and unit boundaries as grading, without inference. */
+export declare function runExpand(options: GradeOptions): Promise<{
+    instances: {
+        projectionDigest: string;
+        reason?: string | undefined;
+        key: string;
+        questionId: string;
+        level: "article" | "paragraph" | "sentence";
+        unit: {
+            paragraph?: number;
+            sentence?: number;
+            text?: string;
+        };
+        state: "missing" | "ready" | "not-applicable";
+    }[];
+}>;

package/dist/training/cli/grader/grade.js

@@ -0,0 +1,33 @@
+import { readFile } from 'node:fs/promises';
+import { expandInstances, gradeOutput, loadGrader } from '../../graders/index.js';
+import { buildGradingRuntime } from './runtime.js';
+/** Standalone grading has no trial snapshot; check gates therefore remain unknown. */
+export async function runGrade(options, dependencies = {}) {
+    options.signal?.throwIfAborted();
+    const output = JSON.parse(await readFile(options.outputFile, { encoding: 'utf8', signal: options.signal }));
+    const runtime = await buildGradingRuntime(options.graderFile, { ...dependencies, maxSpendUsd: options.maxSpendUsd ?? dependencies.maxSpendUsd }, options.signal);
+    const context = await readContexts(runtime.grader, options);
+    return gradeOutput({ ...runtime, output, context, signal: options.signal });
+}
+/** Label builders use the same expansion and unit boundaries as grading, without inference. */
+export async function runExpand(options) {
+    options.signal?.throwIfAborted();
+    const grader = await loadGrader(options.graderFile);
+    const output = JSON.parse(await readFile(options.outputFile, { encoding: 'utf8', signal: options.signal }));
+    const context = await readContexts(grader, options);
+    options.signal?.throwIfAborted();
+    return { instances: expandInstances(grader, output, context).map(({ key, questionId, level, unit, state, reasons, projectionDigest }) => ({
+            key, questionId, level, unit, state, ...(reasons.length ? { reason: reasons.join('; ') } : {}), projectionDigest,
+        })) };
+}
+async function readContexts(grader, options) {
+    const context = {};
+    for (const [name, file] of Object.entries(options.contextFiles ?? {})) {
+        if (!Object.hasOwn(grader.context, name))
+            throw new Error(`Unknown grader context ${name}`);
+        const text = await readFile(file, { encoding: 'utf8', signal: options.signal });
+        context[name] = file.endsWith('.json') ? JSON.parse(text) : text;
+    }
+    return context;
+}
+//# sourceMappingURL=grade.js.map

package/dist/training/cli/grader/grade.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/grade.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAe,MAAM,wBAAwB,CAAC;AAC/F,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAWnD,sFAAsF;AACtF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB,EAAE,eAAmC,EAAE;IACzF,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrH,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,GAAG,YAAY,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,YAAY,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACjK,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5D,OAAO,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9E,CAAC;AAED,+FAA+F;AAC/F,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAqB;IACnD,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpD,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,OAAO,EAAE,SAAS,EAAE,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC,CAAC;YACxI,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,gBAAgB;SACjH,CAAC,CAAC,EAAE,CAAC;AACR,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,OAAqB;IAC/D,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;QAC5F,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAChF,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/training/cli/grader/index.d.ts

@@ -0,0 +1,10 @@
+import type { CliArguments } from '../arguments.js';
+import type { CliIO } from '../types.js';
+import type { GraderDependencies } from './backend.js';
+export { runLabel, createGraderTeacher } from './label.js';
+export { runCalibrate } from './calibrate.js';
+export { runValidate } from './validate.js';
+export { runExpand, runGrade } from './grade.js';
+export { buildExecutableGradingRuntime, buildGradingRuntime } from './runtime.js';
+export type { GraderDependencies } from './backend.js';
+export declare function runGraderCommand(args: CliArguments, io: CliIO, dependencies?: GraderDependencies, signal?: AbortSignal): Promise<number>;

package/dist/training/cli/grader/index.js

@@ -0,0 +1,73 @@
+import { DecisionAccountingError } from '../../graders/index.js';
+import { resolve } from 'node:path';
+import { one, count, pairs } from '../arguments.js';
+import { runLabelImport } from './label-import.js';
+import { runLabel } from './label.js';
+import { runValidate } from './validate.js';
+import { runCalibrate } from './calibrate.js';
+import { runExpand, runGrade } from './grade.js';
+import { runAtomicCommand } from './atomic/index.js';
+export { runLabel, createGraderTeacher } from './label.js';
+export { runCalibrate } from './calibrate.js';
+export { runValidate } from './validate.js';
+export { runExpand, runGrade } from './grade.js';
+export { buildExecutableGradingRuntime, buildGradingRuntime } from './runtime.js';
+export async function runGraderCommand(args, io, dependencies = {}, signal) {
+    try {
+        return await executeGraderCommand(args, io, dependencies, signal);
+    }
+    catch (error) {
+        if (error instanceof DecisionAccountingError)
+            io.stdout(`${JSON.stringify({
+                schema: 'spawnfile.training.grader-failure.v1', status: 'failed', command: args.graderCommand,
+                decisionAccounting: error.decisionAccounting,
+            })}\n`);
+        throw error;
+    }
+}
+async function executeGraderCommand(args, io, dependencies, signal) {
+    if (['decompose', 'traps', 'reference'].includes(args.graderCommand ?? ''))
+        return runAtomicCommand(args, io, dependencies, signal);
+    const maxSpendUsd = args.flags['max-spend-usd'] ? Number(one(args, 'max-spend-usd')) : undefined;
+    const graderFile = resolve(io.cwd, one(args, 'grader'));
+    if (args.graderCommand === 'validate') {
+        io.stdout(`${JSON.stringify(await runValidate(graderFile, dependencies, signal))}\n`);
+        return 0;
+    }
+    if (args.graderCommand === 'grade' || args.graderCommand === 'expand') {
+        const options = { graderFile, maxSpendUsd, outputFile: resolve(io.cwd, one(args, 'output')),
+            contextFiles: Object.fromEntries(Object.entries(pairs(args.flags.context ?? [])).map(([name, file]) => [name, resolve(io.cwd, file)])),
+            signal };
+        if (args.graderCommand === 'expand') {
+            io.stdout(`${JSON.stringify(await runExpand(options))}\n`);
+            return 0;
+        }
+        const result = await runGrade(options, dependencies);
+        io.stdout(`${JSON.stringify(result)}\n`);
+        return result.status === 'pass' ? 0 : 1;
+    }
+    if (args.graderCommand === 'label' && args.flags.import) {
+        const result = await runLabelImport({ graderFile, importFile: resolve(io.cwd, one(args, 'import')),
+            relabel: Boolean(args.flags.relabel), signal });
+        io.stdout(`${JSON.stringify(result)}\n`);
+        return 0;
+    }
+    const concurrency = count(args, 'concurrency', 2, 64);
+    if (args.graderCommand === 'label') {
+        const result = await runLabel({ graderFile, inputsFile: resolve(io.cwd, one(args, 'inputs')),
+            teachers: one(args, 'teachers').split(',').map(value => value.trim()), relabel: Boolean(args.flags.relabel),
+            adjudication: args.flags.adjudication ? count(args, 'adjudication', 0, 1_000_000, 0) : undefined,
+            limit: args.flags.limit ? count(args, 'limit', 1, 1_000_000) : undefined, concurrency, signal }, dependencies);
+        io.stdout(`${JSON.stringify(result)}\n`);
+        for (const [teacher, kind] of Object.entries(result.stopped))
+            io.stderr(`Teacher ${teacher} stopped: ${kind}\n`);
+        return result.missing || Object.keys(result.stopped).length ? 1 : 0;
+    }
+    const result = await runCalibrate({ graderFile, maxSpendUsd,
+        excludeDatasets: (args.flags['exclude-dataset'] ?? []).map(file => resolve(io.cwd, file)),
+        seed: one(args, 'seed', '0'), reportFile: args.flags.report ? resolve(io.cwd, one(args, 'report')) : undefined,
+        concurrency, signal }, dependencies);
+    io.stdout(`${JSON.stringify({ calibrationFile: result.calibrationFile, reportFile: result.reportFile, code: result.code, decisionAccounting: result.decisionAccounting })}\n`);
+    return result.code;
+}
+//# sourceMappingURL=index.js.map