spawnfile 0.1.25 → 0.1.27

package/dist/training/cli/grader/atomic/reference.d.ts

@@ -0,0 +1,131 @@
+import type { GraderDependencies } from '../backend.js';
+export interface ReferenceOptions {
+    templatesFile: string;
+    decompositions: string;
+    traps?: string;
+    teachers: string[];
+    out: string;
+    allowRejections?: boolean;
+    concurrency?: number;
+    batchSize?: number;
+    timeoutMs?: number;
+    signal?: AbortSignal;
+    now?: Date;
+}
+export declare function runReference(options: ReferenceOptions, dependencies?: GraderDependencies): Promise<{
+    gate: "STOP" | "PASS";
+    failures: string[];
+    warnings: string[];
+    knownAnswer: {
+        total: number;
+        correct: number;
+        accuracy: number | null;
+    };
+    consistency: {
+        total: number;
+        correct: number;
+        accuracy: number | null;
+    };
+    coverage: {
+        missingChecks: number;
+        total: number;
+        covered: number;
+        uncovered: string[];
+        uncoveredRate: number;
+    };
+    completeness: {
+        byTemplate: {
+            [k: string]: {
+                total: number;
+                missing: number;
+                rate: number;
+            };
+        };
+        byOutput: {
+            [k: string]: {
+                total: number;
+                missing: number;
+                rate: number;
+            };
+        };
+        total: number;
+        missing: number;
+        rate: number;
+    };
+    byKind: {
+        [k: string]: {
+            total: number;
+            correct: number;
+            accuracy: number | null;
+        };
+    };
+    byTemplate: {
+        [k: string]: {
+            total: number;
+            correct: number;
+            accuracy: number | null;
+        };
+    };
+    diagnostic: {
+        total: number;
+        correct: number;
+        accuracy: number | null;
+    };
+    diagnosticByKind: {
+        [k: string]: {
+            total: number;
+            correct: number;
+            accuracy: number | null;
+        };
+    };
+    diagnosticByTemplate: {
+        [k: string]: {
+            total: number;
+            correct: number;
+            accuracy: number | null;
+        };
+    };
+    consistencyByKind: {
+        [k: string]: {
+            total: number;
+            correct: number;
+            accuracy: number | null;
+        };
+    };
+    consistencyByTemplate: {
+        [k: string]: {
+            total: number;
+            correct: number;
+            accuracy: number | null;
+        };
+    };
+    insufficientKinds: string[];
+    insufficientTemplates: string[];
+    failingKinds: string[];
+    failingTemplates: string[];
+    instability: {
+        items: number;
+        unstable: number;
+        rate: number;
+    };
+    variantAdjudication: {
+        items: number;
+        unstable: number;
+        rate: number;
+    };
+    optionPermutation: {
+        total: number;
+        correct: number;
+        accuracy: number | null;
+    };
+    teachers: {
+        [k: string]: {
+            items: number;
+            agreement: number;
+            softAgreement: number;
+        };
+    };
+    file: string;
+    reportFile: string;
+    outputs: number;
+}>;

package/dist/training/cli/grader/atomic/reference.js

@@ -0,0 +1,133 @@
+import { COUNCIL_WINDOW_OPTION_KEY, COUNCIL_WINDOW_CONFIG_KEY } from '../../../../shared/trainingLegacy/index.js';
+import { readdir, readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { stableDigest } from '../../../graders/index.js';
+import { instantiateChecks, trapDeclaration } from '../../../graders/decomposition/index.js';
+import { validateTrap } from '../../../graders/traps/index.js';
+import { runCouncil, referenceQuality } from '../../../graders/references/index.js';
+import { createGraderTeacher } from '../label-call.js';
+import { selectTeachers } from './decompose.js';
+import { templatesDigest, atomicWrite, readDecompositions, readOptional, readTemplates, workers } from './files.js';
+export async function runReference(options, dependencies = {}) {
+    options.signal?.throwIfAborted();
+    const templates = await readTemplates(options.templatesFile);
+    const teachers = selectTeachers(templates, options.teachers);
+    const inputs = await readDecompositions(options.decompositions, templates, options.allowRejections);
+    const trapBundles = [];
+    if (options.traps) {
+        const files = (await readdir(options.traps)).filter(file => file.endsWith('.traps.json')).sort();
+        for (const file of files) {
+            const bundle = JSON.parse(await readFile(join(options.traps, file), 'utf8'));
+            if (trapBundles.some(previous => previous.decomposition === bundle.decomposition))
+                throw new Error(`Duplicate trap decomposition: ${file}`);
+            const input = inputs.find(input => input.file === bundle.decomposition);
+            if (!input || input.digest !== bundle.decompositionDigest)
+                throw new Error(`Stale trap decomposition: ${file}`);
+            const units = [...input.doc.units, ...(input.doc.audit?.missed ?? [])];
+            const ids = new Set();
+            for (const trap of bundle.traps) {
+                const unit = units.find(unit => unit.id === trap.base_unit);
+                if (ids.has(trap.id) || !unit || !validateTrap(trap, unit, input.inputs, templates, input.doc).valid) {
+                    throw new Error(`Invalid trap checkpoint: ${file}`);
+                }
+                ids.add(trap.id);
+            }
+            trapBundles.push(bundle);
+        }
+    }
+    const providers = Object.fromEntries(teachers.map(teacher => [teacher.id, (dependencies.provider ?? createGraderTeacher)(teacher)]));
+    const outputs = {};
+    const checkpoints = `${options.out}.checkpoints`;
+    const run = async (key, outputDigest, instances) => {
+        options.signal?.throwIfAborted();
+        const filename = join(checkpoints, `${stableDigest(key)}.json`);
+        const previous = await readOptional(filename);
+        outputs[key] = await runCouncil({ outputDigest, instances, teachers, providers, previous,
+            batchSize: options.batchSize, now: options.now, [COUNCIL_WINDOW_OPTION_KEY]: templates[COUNCIL_WINDOW_CONFIG_KEY], timeoutMs: options.timeoutMs, signal: options.signal,
+            onCheckpoint: output => atomicWrite(filename, output) });
+    };
+    await workers(inputs, options.concurrency ?? 2, async (input) => {
+        await run(input.file, input.digest, instantiateChecks(templates, input.doc, input.inputs));
+        for (const bundle of trapBundles.filter(bundle => bundle.decomposition === input.file)) {
+            for (const trap of bundle.traps) {
+                const affected = trap.instances.filter(instance => Object.hasOwn(trap.intended_answers, instance.id));
+                if (!['format-paraphrase', 'option-permutation'].includes(trap.kind) && affected.some(instance => !instance.expected.includes(outputs[input.file].checks[instance.id]?.final ?? '')))
+                    continue;
+                await run(`${input.file}:${trap.id}`, stableDigest({ base: input.digest, trap }), affected);
+            }
+        }
+    }, options.signal);
+    const assessments = [];
+    for (const bundle of trapBundles)
+        for (const trap of bundle.traps) {
+            const base = outputs[bundle.decomposition], variant = outputs[`${bundle.decomposition}:${trap.id}`];
+            if (!variant)
+                continue;
+            for (const [id, intended] of Object.entries(trap.intended_answers)) {
+                const templateId = trap.instances.find(instance => instance.id === id).template_id;
+                const declaration = templates.templates.find(template => template.id === templateId).traps[trap.kind];
+                assessments.push({
+                    known_answer: trapDeclaration(declaration).known_answer,
+                    outputId: bundle.decomposition, kind: trap.kind, templateId,
+                    intended, actual: variant.checks[id]?.final ?? null, baseline: base.checks[id]?.final ?? null,
+                });
+            }
+        }
+    const sorted = Object.fromEntries(Object.entries(outputs).sort(([a], [b]) => a.localeCompare(b)));
+    const baseOutputs = inputs.map(input => outputs[input.file]);
+    const baseKeys = new Set(inputs.map(input => input.file));
+    const quality = referenceQuality(baseOutputs, assessments, { outputIds: inputs.map(input => input.file),
+        rejectedUnits: inputs.reduce((sum, input) => sum + (input.doc.rejected?.length ?? 0), 0),
+        variantOutputs: Object.entries(sorted).filter(([key]) => !baseKeys.has(key)).map(([, output]) => output) });
+    const artifact = { schema: 'spawnfile.training.references.v1', templatesDigest: templatesDigest(templates), outputs: sorted, quality };
+    await atomicWrite(options.out, { ...artifact, digest: stableDigest(artifact) });
+    const reportFile = `${options.out}.md`;
+    await atomicWrite(reportFile, referenceReport(artifact));
+    return { file: options.out, reportFile, outputs: Object.keys(outputs).length, ...quality };
+}
+function referenceReport(artifact) {
+    const { quality } = artifact;
+    const skipped = [...new Set(Object.values(artifact.outputs).flatMap(output => output.skipped.map(skip => `${skip.teacher}: ${skip.reason}`)))];
+    const retained = Object.values(artifact.outputs).flatMap(output => output.calls);
+    const calls = retained.filter(call => call.outcome !== 'cancelled');
+    return [`# Council references: ${quality.gate}`, '',
+        `Known-answer accuracy: ${quality.knownAnswer.accuracy === null ? 'unmeasured (no validated traps)' : `${(quality.knownAnswer.accuracy * 100).toFixed(2)}%`} (${quality.knownAnswer.correct}/${quality.knownAnswer.total}).`,
+        `Consistency: ${quality.consistency.accuracy === null ? 'unmeasured' : `${(quality.consistency.accuracy * 100).toFixed(2)}%`} (${quality.consistency.correct}/${quality.consistency.total}).`,
+        `Known-answer output coverage: ${quality.coverage.covered}/${quality.coverage.total}; uncovered: ${quality.coverage.uncovered.join(', ') || 'none'}.`,
+        `Rejected units counted as missing checks: ${quality.coverage.missingChecks}.`,
+        `Missing base checks: ${(quality.completeness.rate * 100).toFixed(2)}% (${quality.completeness.missing}/${quality.completeness.total}); required <= 0%.`,
+        ...Object.entries(quality.completeness.byTemplate).map(([id, score]) => `- Missing base checks for template ${id}: ${score.missing}/${score.total}.`),
+        ...Object.entries(quality.completeness.byOutput).map(([id, score]) => `- Missing base checks for output ${id}: ${score.missing}/${score.total}.`),
+        `Base adjudication instability: ${(quality.instability.rate * 100).toFixed(2)}% (${quality.instability.unstable}/${quality.instability.items}).`,
+        `Variant adjudication (diagnostic): ${(quality.variantAdjudication.rate * 100).toFixed(2)}% (${quality.variantAdjudication.unstable}/${quality.variantAdjudication.items}).`,
+        `Calls: ${calls.length}; recorded latency: ${calls.reduce((sum, call) => sum + call.wallMs, 0).toFixed(0)} ms. Usage and subscription accounting are recorded per call; unreported usage/cost remains unknown.`, '',
+        'Per-teacher accounting (all retained calls, including adjudication, failed/cancelled calls and earlier attempts; resume does not add reused calls):', '',
+        ...[...new Set(Object.values(artifact.outputs).flatMap(output => [...(output.active_teachers ?? []), ...output.skipped.map(skip => skip.teacher)]))].map(id => {
+            const own = retained.filter(call => call.teacher === id);
+            const tokens = (key) => {
+                const known = own.filter(call => call.usage?.[key] != null);
+                const sum = known.reduce((total, call) => total + call.usage[key], 0);
+                return known.length === own.length ? String(sum) : `${sum} reported; ${own.length - known.length} calls unknown`;
+            };
+            return `- ${id}: ${own.length} calls; input tokens: ${tokens('inputTokens')}; output tokens: ${tokens('outputTokens')}; summed call wall time: ${own.reduce((sum, call) => sum + call.wallMs, 0).toFixed(0)} ms.`;
+        }), '',
+        ...retained.filter(call => call.failure_class).map(call => `- Failed council call ${call.id} (${call.teacher}, ${call.phase}): ${call.failure_class}; ${call.check_ids.length} checks affected.`),
+        ...quality.warnings.map(warning => `- WARNING: ${warning}`),
+        ...quality.failures.map(failure => `- STOP: ${failure}`),
+        ...quality.failingKinds.map(kind => `- Failing trap kind: ${kind}`),
+        ...quality.failingTemplates.map(id => `- Failing template: ${id}`),
+        ...skipped.map(skip => `- Skipped ${skip}`), '',
+        ...quality.insufficientKinds.map(kind => `- Insufficient known-answer samples for kind: ${kind} (<10)`),
+        ...quality.insufficientTemplates.map(id => `- Insufficient known-answer samples for template: ${id} (<10)`),
+        'Known-answer accuracy by kind:', '', ...Object.entries(quality.byKind).map(([kind, score]) => `- ${kind}: ${score.correct}/${score.total}`), '',
+        'DIAGNOSTIC accuracy by kind (excluded from gates and coverage):', '',
+        ...Object.entries(quality.diagnosticByKind).map(([kind, score]) => `- ${kind}: ${score.correct}/${score.total} (${(score.accuracy * 100).toFixed(2)}%)`), '',
+        'DIAGNOSTIC accuracy by template:', '',
+        ...Object.entries(quality.diagnosticByTemplate).map(([id, score]) => `- ${id}: ${score.correct}/${score.total}`), '',
+        'Control CONSISTENCY by kind:', '', ...Object.entries(quality.consistencyByKind).map(([kind, score]) => `- ${kind}: ${score.correct}/${score.total}`), '',
+        'Option-permutation consistency: ' + (quality.optionPermutation.accuracy === null ? 'unmeasured' : `${quality.optionPermutation.correct}/${quality.optionPermutation.total}`) + '.', '',
+        'Teachers against references excluding themselves:', '',
+        ...Object.entries(quality.teachers).map(([id, score]) => `- ${id}: ${(score.agreement * 100).toFixed(2)}% agreement on ${score.items} checks; soft agreement ${score.softAgreement.toFixed(4)}.`), '',
+        'Agent agreement is not independent correctness. Known-answer traps measure constructed-control accuracy; external reviews and editor verdicts remain weak signals.', '',].join('\n');
+}
+//# sourceMappingURL=reference.js.map

package/dist/training/cli/grader/atomic/reference.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reference.js","sourceRoot":"","sources":["../../../../../src/training/cli/grader/atomic/reference.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,yBAAyB,EAAE,MAAM,4CAA4C,CAAC;AAClH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC;AAC7F,OAAO,EAAE,YAAY,EAAgB,MAAM,iCAAiC,CAAC;AAC7E,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAA2C,MAAM,sCAAsC,CAAC;AAE7H,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,kBAAkB,EAAE,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAOpH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAyB,EAAE,eAAmC,EAAE;IACjG,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IACpG,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjG,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAe,CAAC;YAC3F,IAAI,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,aAAa,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC;YAC5I,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,aAAa,CAAC,CAAC;YACxE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,mBAAmB;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,EAAE,CAAC,CAAC;YAChH,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YACvE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;YAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC5D,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;oBACrG,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,EAAE,CAAC,CAAC;gBACtD,CAAC;gBACD,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,CAAC;YACD,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,IAAI,mBAAmB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACrI,MAAM,OAAO,GAAkC,EAAE,CAAC;IAClD,MAAM,WAAW,GAAG,GAAG,OAAO,CAAC,GAAG,cAAc,CAAC;IACjD,MAAM,GAAG,GAAG,KAAK,EAAE,GAAW,EAAE,YAAoB,EAAE,SAAwD,EAAE,EAAE;QAChH,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAgB,QAAQ,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ;YACtF,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,yBAAyB,CAAC,EAAE,SAAS,CAAC,yBAAyB,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM;YACvK,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,EAAE,KAAK,EAAC,KAAK,EAAC,EAAE;QAC5D,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,iBAAiB,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3F,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACvF,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBACtG,IAAI,CAAC,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAC/F,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;oBAAE,SAAS;gBAChG,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,EAChF,QAAQ,CAAC,CAAC;YACZ,CAAC;QACH,CAAC;IACH,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACnB,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,KAAK,MAAM,MAAM,IAAI,WAAW;QAAE,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAClE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAE,EAAE,OAAO,GAAG,OAAO,CAAC,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,EAAE,EAAE,CAAE,CAAC;YACtG,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,KAAK,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACnE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAE,CAAC,WAAW,CAAC;gBACpF,MAAM,WAAW,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,UAAU,CAAE,CAAC,KAAM,CAAC,IAAI,CAAC,IAAI,CAAE,CAAC;gBACzG,WAAW,CAAC,IAAI,CAAC;oBACf,YAAY,EAAE,eAAe,CAAC,WAAW,CAAC,CAAC,YAAY;oBACvD,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU;oBAC3D,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,IAAI,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,IAAI,IAAI;iBAC9F,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,EAAE,WAAW,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;QACrG,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACxF,cAAc,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC9G,MAAM,QAAQ,GAAG,EAAE,MAAM,EAAE,kCAAkC,EAAE,eAAe,EAAE,eAAe,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACvI,MAAM,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAChF,MAAM,UAAU,GAAG,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC;IACvC,MAAM,WAAW,CAAC,UAAU,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzD,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,GAAG,OAAO,EAAE,CAAC;AAC7F,CAAC;AACD,SAAS,eAAe,CAAC,QAAkG;IACzH,MAAM,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAC7B,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/I,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACjF,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,WAAW,CAAC,CAAC;IACpE,OAAO,CAAC,yBAAyB,OAAO,CAAC,IAAI,EAAE,EAAE,EAAE;QACjD,0BAA0B,OAAO,CAAC,WAAW,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,IAAI;QAC5N,gBAAgB,OAAO,CAAC,WAAW,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,IAAI;QAC7L,iCAAiC,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,gBAAgB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,GAAG;QACrJ,6CAA6C,OAAO,CAAC,QAAQ,CAAC,aAAa,GAAG;QAC9E,wBAAwB,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,OAAO,IAAI,OAAO,CAAC,YAAY,CAAC,KAAK,oBAAoB;QACxJ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,sCAAsC,EAAE,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;QACrJ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,oCAAoC,EAAE,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;QACjJ,kCAAkC,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,WAAW,CAAC,QAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,IAAI;QAChJ,sCAAsC,CAAC,OAAO,CAAC,mBAAmB,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,mBAAmB,CAAC,QAAQ,IAAI,OAAO,CAAC,mBAAmB,CAAC,KAAK,IAAI;QAC5K,UAAU,KAAK,CAAC,MAAM,uBAAuB,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,sGAAsG,EAAE,EAAE;QACnN,qJAAqJ,EAAE,EAAE;QACzJ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAC5J,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;YACzD,MAAM,MAAM,GAAG,CAAC,GAAmC,EAAE,EAAE;gBACrD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC;gBAC5D,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,KAAM,CAAC,GAAG,CAAE,EAAE,CAAC,CAAC,CAAC;gBACxE,OAAO,KAAK,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,gBAAgB,CAAC;YACnH,CAAC,CAAC;YACF,OAAO,KAAK,EAAE,KAAK,GAAG,CAAC,MAAM,yBAAyB,MAAM,CAAC,aAAa,CAAC,oBAAoB,MAAM,CAAC,cAAc,CAAC,4BAA4B,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QACpN,CAAC,CAAC,EAAE,EAAE;QACN,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACxD,yBAAyB,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,mBAAmB,CAAC;QACxI,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,cAAc,OAAO,EAAE,CAAC;QAC3D,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,WAAW,OAAO,EAAE,CAAC;QACxD,GAAG,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,wBAAwB,IAAI,EAAE,CAAC;QACnE,GAAG,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,uBAAuB,EAAE,EAAE,CAAC;QAClE,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,IAAI,EAAE,CAAC,EAAE,EAAE;QAC/C,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,iDAAiD,IAAI,QAAQ,CAAC;QACvG,GAAG,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,qDAAqD,EAAE,QAAQ,CAAC;QAC3G,gCAAgC,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAChJ,iEAAiE,EAAE,EAAE;QACrE,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,QAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE;QAC7J,kCAAkC,EAAE,EAAE;QACtC,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACpH,8BAA8B,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACzJ,kCAAkC,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC,OAAO,IAAI,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC,GAAG,GAAG,EAAE,EAAE;QACvL,mDAAmD,EAAE,EAAE;QACvD,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,KAAK,CAAC,KAAK,2BAA2B,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE;QACrM,oKAAoK,EAAE,EAAE,EACzK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}

package/dist/training/cli/grader/atomic/runtime.d.ts

@@ -0,0 +1,3 @@
+import type { AtomicRuntime } from '../../../graders/atomic/index.js';
+import type { GraderDependencies } from '../backend.js';
+export declare function buildAtomicRuntime(file: string, dependencies?: GraderDependencies): Promise<AtomicRuntime>;

package/dist/training/cli/grader/atomic/runtime.js

@@ -0,0 +1,91 @@
+import { graderProvider } from '../../../contracts/index.js';
+import { checkTrainingFilename } from '../../../platform/index.js';
+import { readFile, mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { parseAtomicGraderYaml, validateAtomicTemplates } from '../../../graders/atomic/index.js';
+import { parseCheckTemplatesYaml } from '../../../graders/decomposition/index.js';
+import { sha256, stableDigest } from '../../../graders/index.js';
+import { assertModelRequestAdmission, budgetEvaluatorInference } from '../../../inference/index.js';
+import { createTypeSafeDecision } from '../../../inference/decision/index.js';
+import { createGraderTeacher } from '../label-call.js';
+import { runDecompose } from './decompose.js';
+import { atomicWrite, readOptional } from './files.js';
+const resources = new Map();
+function resource(key) {
+    if (!resources.has(key))
+        resources.set(key, {});
+    return resources.get(key);
+}
+export async function buildAtomicRuntime(file, dependencies = {}) {
+    checkTrainingFilename(file);
+    const source = await readFile(file, 'utf8'), grader = parseAtomicGraderYaml(source);
+    const templateFile = resolve(dirname(file), grader.templates);
+    checkTrainingFilename(templateFile);
+    const templateSource = await readFile(templateFile, 'utf8');
+    const templates = parseCheckTemplatesYaml(templateSource);
+    validateAtomicTemplates(grader, templates);
+    assertModelRequestAdmission(graderProvider(grader.extractor.provider).inferenceProviderId, 'atomic extraction', { purpose: 'extraction', model: grader.extractor.model }, dependencies.inferenceProviders);
+    assertModelRequestAdmission('typesafe', 'atomic decisions', { purpose: 'decisions', model: grader.decision.model }, dependencies.inferenceProviders);
+    const maxSpendUsd = Math.min(grader.decision.max_spend_usd, dependencies.maxSpendUsd ?? Infinity);
+    // Construct before extraction: a missing key must never consume a subscription call.
+    const backend = createTypeSafeDecision({ ...(dependencies.inferenceProviders ? { inferenceProviders: dependencies.inferenceProviders } : {}), model: grader.decision.model, maxSpendUsd,
+        pricePerMtokUsd: grader.decision.price_per_mtok_usd, maxAttempts: 1, concurrency: 1,
+        conservativeSpend: true, fetch: dependencies.atomicFetch });
+    // Snapshot all authoring bytes into memory; subjects cannot change this runtime's policy.
+    const { deadline_ms: _deadline, ...teacher } = grader.extractor;
+    const frozen = { ...templates, teachers: [teacher] };
+    return { kind: 'atomic', grader, templates, backend, definitionDigest: sha256(source), templatesDigest: sha256(templateSource),
+        resource: resource('typesafe'), resources: [resource('typesafe'), resource(`extractor:${grader.extractor.model}`)],
+        async extract(inputs, signal) {
+            const directory = await mkdtemp(join(tmpdir(), 'training-atomic-trial-'));
+            const out = join(directory, 'decomposition');
+            const receipts = [];
+            const judgeCalls = [];
+            let error;
+            try {
+                await atomicWrite(join(directory, 'templates.json'), frozen);
+                await atomicWrite(join(directory, 'output.json'), JSON.stringify(inputs.output));
+                const context = Object.fromEntries(Object.entries(inputs.context).map(([name]) => [name, { from: `${sha256(name)}.json` }]));
+                for (const [name, value] of Object.entries(inputs.context))
+                    await atomicWrite(join(directory, `${sha256(name)}.json`), JSON.stringify(value));
+                await atomicWrite(join(directory, 'inputs.json'), { schema: 'spawnfile.training.grader-inputs.v1', items: [{
+                            id: 'trial', lineage: 'trial', output: { from: 'output.json' }, context,
+                        }] });
+                const deadline = AbortSignal.timeout(grader.extractor.deadline_ms);
+                await runDecompose({ templatesFile: join(directory, 'templates.json'), inputsFile: join(directory, 'inputs.json'),
+                    obligationKinds: grader.aggregation.obligation_kinds,
+                    out, extractors: [grader.extractor.id], signal: signal ? AbortSignal.any([signal, deadline]) : deadline }, { provider: teacher => budgetEvaluatorInference((dependencies.provider ?? (definition => createGraderTeacher(definition, grader.extractor.deadline_ms)))(teacher), graderProvider(teacher.provider).inferenceProviderId, 'extraction', teacher.model, dependencies.inferenceProviders) });
+            }
+            catch {
+                error = 'Atomic extraction or omission audit failed';
+            }
+            try {
+                const checkpoint = await readOptional(join(out, `${stableDigest('trial')}.checkpoint.json`));
+                if (checkpoint) {
+                    receipts.push(JSON.parse(JSON.stringify(checkpoint)));
+                    for (const receipt of checkpoint.receipts) {
+                        if (receipt.attempts?.length) {
+                            for (const attempt of receipt.attempts)
+                                judgeCalls.push({ id: attempt.id, judge: receipt.teacher,
+                                    providerId: attempt.providerId, requestedModel: attempt.requestedModel, responseModel: attempt.responseModel,
+                                    responseRoute: attempt.responseRoute, outcome: attempt.outcome === 'failed' ? 'generation-error'
+                                        : receipt.outcome === 'invalid-response' ? 'invalid-response' : 'completed',
+                                    ...(attempt.failureKind ? { failureKind: attempt.failureKind } : {}), usage: attempt.usage, billing: 'subscription' });
+                        }
+                        else
+                            judgeCalls.push({ id: randomUUID(), judge: receipt.teacher,
+                                providerId: receipt.provider, requestedModel: receipt.model, responseModel: receipt.reported_model,
+                                responseRoute: receipt.route, outcome: receipt.outcome === 'completed' ? 'completed' : receipt.outcome === 'invalid-response' ? 'invalid-response' : 'generation-error',
+                                usage: receipt.usage, wallMs: receipt.wall_ms, billing: 'subscription' });
+                    }
+                }
+                return { decomposition: checkpoint?.decomposition, receipts, judgeCalls, ...(error ? { error } : {}) };
+            }
+            finally {
+                await rm(directory, { recursive: true, force: true });
+            }
+        } };
+}
+//# sourceMappingURL=runtime.js.map

package/dist/training/cli/grader/atomic/runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../../../../src/training/cli/grader/atomic/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAClG,OAAO,EAAE,uBAAuB,EAAE,MAAM,yCAAyC,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,2BAA2B,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAE9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,YAAY,EAA0B,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEvD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;AAC5C,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAChD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY,EAAE,eAAmC,EAAE;IAC1F,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IACpF,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAC9D,qBAAqB,CAAC,YAAY,CAAC,CAAC;IACpC,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,uBAAuB,CAAC,cAAc,CAAC,CAAC;IAC1D,uBAAuB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3C,2BAA2B,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,mBAAmB,EAAE,mBAAmB,EAC5G,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,YAAY,CAAC,kBAAkB,CAAC,CAAC;IAC7F,2BAA2B,CAAC,UAAU,EAAE,kBAAkB,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,YAAY,CAAC,kBAAkB,CAAC,CAAC;IACrJ,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,YAAY,CAAC,WAAW,IAAI,QAAQ,CAAC,CAAC;IAClG,qFAAqF;IACrF,MAAM,OAAO,GAAG,sBAAsB,CAAC,EAAE,GAAG,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,YAAY,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,WAAW;QACrL,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC;QACnF,iBAAiB,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC;IAC9D,0FAA0F;IAC1F,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC;IAChE,MAAM,MAAM,GAAG,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,cAAc,CAAC;QAC5H,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,aAAa,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;QAClH,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM;YAC1B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC,CAAC;YAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAiC,EAAE,CAAC;YAClD,MAAM,UAAU,GAAgB,EAAE,CAAC;YACnC,IAAI,KAAyB,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC,CAAC;gBAC7D,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBACjF,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC7H,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;oBAAE,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC9I,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,qCAAqC,EAAE,KAAK,EAAE,CAAC;4BACzG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,OAAO;yBACxE,CAAC,EAAE,CAAC,CAAC;gBACN,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBACnE,MAAM,YAAY,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC;oBAC/G,eAAe,EAAE,MAAM,CAAC,WAAW,CAAC,gBAAgB;oBACpD,GAAG,EAAE,UAAU,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAC3G,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,wBAAwB,CAC7C,CAAC,YAAY,CAAC,QAAQ,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,mBAAmB,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EACjH,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,mBAAmB,EAAE,YAAY,EAAE,OAAO,CAAC,KAAK,EAAE,YAAY,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;YAC3H,CAAC;YAAC,MAAM,CAAC;gBAAC,KAAK,GAAG,4CAA4C,CAAC;YAAC,CAAC;YACjE,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,YAAY,CACiE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;gBAC7J,IAAI,UAAU,EAAE,CAAC;oBACf,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAA4B,CAAC,CAAC;oBACjF,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;wBAC1C,IAAI,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;4BAC7B,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ;gCAAE,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO;oCAC9F,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa;oCAC5G,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,kBAAkB;wCAC9F,CAAC,CAAC,OAAO,CAAC,OAAO,KAAK,kBAAkB,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,WAAW;oCAC7E,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;wBAC3H,CAAC;;4BAAM,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO;gCAC/D,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,CAAC,cAAc;gCAClG,aAAa,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,KAAK,kBAAkB,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,kBAAkB;gCACvK,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;oBAC9E,CAAC;gBACH,CAAC;gBACD,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACzG,CAAC;oBAAS,CAAC;gBAAC,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAAC,CAAC;QACtE,CAAC,EAAE,CAAC;AACR,CAAC"}

package/dist/training/cli/grader/atomic/traps.d.ts

@@ -0,0 +1,15 @@
+export interface TrapsOptions {
+    templatesFile: string;
+    decompositions: string;
+    out: string;
+    seed?: string;
+    references?: string;
+    allowRejections?: boolean;
+    signal?: AbortSignal;
+}
+export declare function runTraps(options: TrapsOptions): Promise<{
+    generated: number;
+    discarded: number;
+    missingChecks: number;
+    directory: string;
+}>;

package/dist/training/cli/grader/atomic/traps.js

@@ -0,0 +1,38 @@
+import { TRAP_CHECKPOINT_DOMAIN } from '../../../contracts/index.js';
+import { join } from 'node:path';
+import { generateTraps } from '../../../graders/traps/index.js';
+import { stableDigest } from '../../../graders/index.js';
+import { templatesDigest, atomicWrite, readDecompositions, readOptional, readTemplates } from './files.js';
+export async function runTraps(options) {
+    options.signal?.throwIfAborted();
+    const templates = await readTemplates(options.templatesFile);
+    const inputs = await readDecompositions(options.decompositions, templates, options.allowRejections);
+    const reference = options.references ? await readOptional(options.references) : undefined;
+    if (options.references) {
+        if (!reference)
+            throw new Error('Missing base references');
+        const { digest, ...content } = reference;
+        if (digest !== stableDigest(content) || reference.templatesDigest !== templatesDigest(templates))
+            throw new Error('Stale or corrupt base references');
+    }
+    let generated = 0, discarded = 0, missingChecks = 0;
+    for (const input of inputs) {
+        options.signal?.throwIfAborted();
+        const base = reference?.outputs[input.file];
+        if (reference && (!base?.complete || base.outputDigest !== input.digest))
+            throw new Error('Stale or incomplete base references');
+        const baseFinals = base ? Object.fromEntries(Object.entries(base.checks).map(([id, check]) => [id, check.final])) : {};
+        missingChecks += input.doc.rejected?.length ?? 0;
+        const identity = stableDigest({ baseFinals, input: input.digest, seed: options.seed ?? '0', protocol: TRAP_CHECKPOINT_DOMAIN });
+        const file = join(options.out, `${stableDigest(input.file)}.traps.json`);
+        const previous = await readOptional(file);
+        if (previous && previous.identity !== identity)
+            throw new Error(`Stale trap checkpoint: ${input.file}`);
+        const result = generateTraps(templates, input.doc, input.inputs, { seed: options.seed ?? '0', baseFinals });
+        await atomicWrite(file, { ...result, identity, decomposition: input.file, decompositionDigest: input.digest });
+        generated += result.traps.length;
+        discarded += result.discarded;
+    }
+    return { generated, discarded, missingChecks, directory: options.out };
+}
+//# sourceMappingURL=traps.js.map

package/dist/training/cli/grader/atomic/traps.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"traps.js","sourceRoot":"","sources":["../../../../../src/training/cli/grader/atomic/traps.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,kBAAkB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG3G,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IAClD,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IACpG,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,YAAY,CAAsF,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/K,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC3D,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,GAAG,SAAS,CAAC;QACzC,IAAI,MAAM,KAAK,YAAY,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,eAAe,KAAK,eAAe,CAAC,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACxJ,CAAC;IACD,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,CAAC,EAAE,aAAa,GAAG,CAAC,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,EAAE,QAAQ,IAAI,IAAI,CAAC,YAAY,KAAK,KAAK,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACjI,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvH,aAAa,IAAI,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAChI,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAuB,IAAI,CAAC,CAAC;QAChE,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACxG,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5G,MAAM,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,IAAI,EAAE,mBAAmB,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/G,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QACjC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IAChC,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;AACzE,CAAC"}

package/dist/training/cli/grader/backend.d.ts

@@ -0,0 +1,19 @@
+import type { InferenceProviderRegistry } from '../../contracts/index.js';
+import type { DecisionBackend, DecisionRequestReceipt, DecisionSpendBudget } from '../../contracts/grading.js';
+import type { InferenceProvider } from '../../contracts/execution.js';
+import type { Grader } from '../../graders/index.js';
+export interface GraderDependencies {
+    inferenceProviders?: InferenceProviderRegistry;
+    /** Explicit transport injection for atomic graders; tests never call live JEV. */
+    atomicFetch?: typeof fetch;
+    decisionBudget?: DecisionSpendBudget;
+    maxSpendUsd?: number;
+    backend?: (grader: Grader) => DecisionBackend;
+    provider?: (teacher: Grader['teachers'][number]) => InferenceProvider;
+}
+export declare function graderBackend(grader: Grader, dependencies?: GraderDependencies): DecisionBackend;
+/** Include routing and teacher policy as well as the backend's reported identity. */
+export declare function backendIdentity(grader: Grader, backend: DecisionBackend, signal?: AbortSignal, receipts?: DecisionRequestReceipt[]): Promise<string>;
+export declare function graderBackends(grader: Grader, dependencies?: GraderDependencies, ids?: ReadonlySet<string>): Record<string, DecisionBackend>;
+/** One command budget spans every authored candidate, including concurrent calls. */
+export declare function withDecisionBudget(grader: Grader, dependencies: GraderDependencies): GraderDependencies;

package/dist/training/cli/grader/backend.js

@@ -0,0 +1,39 @@
+import { decisionBackendIdentity, decisionModels, graderForModel } from '../../graders/index.js';
+import { createOllamaDecision, createJevDecision, createTeacherDecision, createTypeSafeDecision, createDecisionSpendBudget } from '../../inference/decision/index.js';
+import { createGraderTeacher } from './label.js';
+export function graderBackend(grader, dependencies = {}) {
+    if (dependencies.backend)
+        return dependencies.backend(grader);
+    const model = grader.model;
+    if (model.backend === 'ollama-logprobs')
+        return createOllamaDecision({ endpoint: model.endpoint, model: model.name, contextTokens: model.context_tokens });
+    if (model.backend === 'typesafe')
+        return createTypeSafeDecision({ endpoint: model.endpoint, model: model.name,
+            ...(dependencies.inferenceProviders ? { inferenceProviders: dependencies.inferenceProviders } : {}),
+            concurrency: model.concurrency, pricePerMtokUsd: model.price_per_mtok_usd, budget: dependencies.decisionBudget, maxSpendUsd: dependencies.maxSpendUsd ?? grader.max_spend_usd });
+    if (model.backend !== 'teacher')
+        return createJevDecision({ endpoint: model.endpoint });
+    const teacher = grader.teachers.find(item => item.id === model.teacher);
+    return createTeacherDecision({ provider: (dependencies.provider ?? createGraderTeacher)(teacher), model: teacher.model });
+}
+/** Include routing and teacher policy as well as the backend's reported identity. */
+export async function backendIdentity(grader, backend, signal, receipts) {
+    return decisionBackendIdentity(grader, backend, signal, receipts);
+}
+export function graderBackends(grader, dependencies = {}, ids) {
+    dependencies = withDecisionBudget(grader, dependencies);
+    return Object.fromEntries(decisionModels(grader).filter(model => !ids || ids.has(model.id)).map(model => {
+        try {
+            return [model.id, graderBackend(graderForModel(grader, model), dependencies)];
+        }
+        catch (cause) {
+            const reason = cause instanceof Error && cause.message === 'TypeSafe requires TYPESAFE_API_KEY' ? cause.message : 'failed to start';
+            throw new Error(`Decision model ${model.id}: ${reason}`, { cause });
+        }
+    }));
+}
+/** One command budget spans every authored candidate, including concurrent calls. */
+export function withDecisionBudget(grader, dependencies) {
+    return { ...dependencies, decisionBudget: dependencies.decisionBudget ?? createDecisionSpendBudget(dependencies.maxSpendUsd ?? grader.max_spend_usd) };
+}
+//# sourceMappingURL=backend.js.map

package/dist/training/cli/grader/backend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/backend.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACjG,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AACtK,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAYjD,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,eAAmC,EAAE;IACjF,IAAI,YAAY,CAAC,OAAO;QAAE,OAAO,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAM,CAAC;IAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,iBAAiB;QAAE,OAAO,oBAAoB,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC3J,IAAI,KAAK,CAAC,OAAO,KAAK,UAAU;QAAE,OAAO,sBAAsB,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI;YAC3G,GAAG,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,YAAY,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnG,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,eAAe,EAAE,KAAK,CAAC,kBAAkB,EAAE,MAAM,EAAE,YAAY,CAAC,cAAc,EAAE,WAAW,EAAE,YAAY,CAAC,WAAW,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnL,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,iBAAiB,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,KAAK,CAAC,OAAO,CAAE,CAAC;IACzE,OAAO,qBAAqB,CAAC,EAAE,QAAQ,EAAE,CAAC,YAAY,CAAC,QAAQ,IAAI,mBAAmB,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;AAC5H,CAAC;AAED,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc,EAAE,OAAwB,EAAE,MAAoB,EAAE,QAAmC;IACvI,OAAO,uBAAuB,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,eAAmC,EAAE,EAAE,GAAyB;IAC7G,YAAY,GAAG,kBAAkB,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACxD,OAAO,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACtG,IAAI,CAAC;YAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,aAAa,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;QAAC,CAAC;QACtF,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,oCAAoC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC;YACpI,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,CAAC,EAAE,KAAK,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,kBAAkB,CAAC,MAAc,EAAE,YAAgC;IACjF,OAAO,EAAE,GAAG,YAAY,EAAE,cAAc,EAAE,YAAY,CAAC,cAAc,IAAI,yBAAyB,CAAC,YAAY,CAAC,WAAW,IAAI,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;AACzJ,CAAC"}

package/dist/training/cli/grader/cache.d.ts

@@ -0,0 +1,2 @@
+import type { DecisionBackend, DecisionRequestReceipt, DecisionRequest, DecisionResult } from '../../contracts/grading.js';
+export declare function cachedScores(directory: string, backend: DecisionBackend, identity: string, requests: readonly DecisionRequest[], concurrency: number, signal?: AbortSignal, receipts?: DecisionRequestReceipt[]): Promise<DecisionResult[]>;

package/dist/training/cli/grader/cache.js

@@ -0,0 +1,75 @@
+import { mkdir, readFile, appendFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { stableDigest } from '../../graders/index.js';
+import { GRADER_CACHE_PROMPT_DOMAIN, GRADER_CACHE_LEGACY_DIRECTORY } from '../../contracts/index.js';
+export async function cachedScores(directory, backend, identity, requests, concurrency, signal, receipts) {
+    if (!Number.isSafeInteger(concurrency) || concurrency < 1)
+        throw new Error('concurrency must be a positive integer');
+    signal?.throwIfAborted();
+    const root = join(directory, '.spawnfile-training', 'grader-cache');
+    await mkdir(root, { recursive: true });
+    const file = join(root, `${identity}.jsonl`);
+    const cache = new Map();
+    // Load historical entries first; a current entry wins per digest, even when
+    // the current file exists but contains only some of the requested scores.
+    for (const candidate of [join(directory, GRADER_CACHE_LEGACY_DIRECTORY, 'grader-cache', `${identity}.jsonl`), file]) {
+        let source = '';
+        try {
+            source = await readFile(candidate, 'utf8');
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+        }
+        for (const line of source.split('\n').filter(Boolean)) {
+            const row = JSON.parse(line);
+            if (typeof row.digest !== 'string' || !row.result || !['ok', 'missing'].includes(row.result.status))
+                throw new Error('Invalid grader cache entry');
+            // Missing scores are retryable, including historical stored entries.
+            if (row.result.status === 'ok')
+                cache.set(row.digest, row.result);
+        }
+    }
+    const results = new Array(requests.length);
+    let next = 0;
+    const controller = new AbortController();
+    const workSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
+    // Abort at the first worker failure, but drain all scores and writes before returning.
+    await Promise.allSettled(Array.from({ length: Math.min(concurrency, requests.length) }, async () => {
+        try {
+            while (next < requests.length) {
+                workSignal.throwIfAborted();
+                // Batch-capable transports own projection grouping and request limits.
+                const indices = backend.capabilities?.batch
+                    ? Array.from({ length: requests.length - next }, () => next++) : [next++];
+                const entries = indices.map(index => {
+                    const request = requests[index];
+                    const { key: _key, ...content } = request;
+                    const digest = stableDigest({ content, identity, prompt: GRADER_CACHE_PROMPT_DOMAIN });
+                    return { index, request, digest, cached: cache.get(digest) };
+                });
+                const pending = entries.filter(entry => !entry.cached);
+                const scored = pending.length ? await backend.score(pending.map(entry => entry.request), workSignal, receipts) : [];
+                workSignal.throwIfAborted();
+                for (const { index, request, digest, cached } of entries) {
+                    const result = cached ?? scored.find(item => item.key === request.key);
+                    if (!result)
+                        throw new Error(`Backend omitted result ${request.key}`);
+                    // Missing/spend-cap answers must be resumable, never a permanent cache hit.
+                    if (result.status === 'ok' && !cache.has(digest)) {
+                        cache.set(digest, result);
+                        await appendFile(file, `${JSON.stringify({ digest, result })}\n`, { mode: 0o600 });
+                    }
+                    results[index] = { ...result, key: request.key };
+                }
+            }
+        }
+        catch (error) {
+            controller.abort(error);
+            throw error;
+        }
+    }));
+    workSignal.throwIfAborted();
+    return results;
+}
+//# sourceMappingURL=cache.js.map

package/dist/training/cli/grader/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD,OAAO,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAC;AAErG,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,SAAiB,EAAE,OAAwB,EAAE,QAAgB,EAC9F,QAAoC,EAAE,WAAmB,EAAE,MAAoB,EAAE,QAAmC;IACpH,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IACrH,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,cAAc,CAAC,CAAC;IACpE,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,QAAQ,QAAQ,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,4EAA4E;IAC5E,0EAA0E;IAC1E,KAAK,MAAM,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,6BAA6B,EAAE,cAAc,EAAE,GAAG,QAAQ,QAAQ,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;QACpH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC;YAAC,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAAC,CAAC;QACnD,OAAO,KAAK,EAAE,CAAC;YAAC,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;QAAC,CAAC;QACtF,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACtD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA+C,CAAC;YAC3E,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YACnJ,qEAAqE;YACrE,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAqB,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7D,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;IAC7F,uFAAuF;IACvF,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACjG,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC9B,UAAU,CAAC,cAAc,EAAE,CAAC;gBAC5B,uEAAuE;gBACvE,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,EAAE,KAAK;oBACzC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;oBAClC,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAE,CAAC;oBACjC,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC;oBAC1C,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;oBACvF,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/D,CAAC,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpH,UAAU,CAAC,cAAc,EAAE,CAAC;gBAC5B,KAAK,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;oBACzD,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;oBACvE,IAAI,CAAC,MAAM;wBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;oBACtE,4EAA4E;oBAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;wBACjD,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;wBAC1B,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;oBACrF,CAAC;oBACD,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;gBACnD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC,CAAC;IACJ,UAAU,CAAC,cAAc,EAAE,CAAC;IAC5B,OAAO,OAAO,CAAC;AACjB,CAAC"}