spawnfile 0.1.25 → 0.1.28

package/dist/training/graders/references/types.d.ts

@@ -0,0 +1,92 @@
+import { COUNCIL_WINDOW_OPTION_KEY } from '../../../shared/trainingLegacy/index.js';
+import type { ProviderAvailabilityWindow } from '../../contracts/index.js';
+import type { InferenceProvider, Usage, InferenceAttempt, InferenceFailureKind, InferenceDiagnostic } from '../../contracts/index.js';
+import type { CheckInstance } from '../decomposition/index.js';
+export interface CouncilTeacher {
+    id: string;
+    provider: string;
+    model: string;
+    effort?: string;
+}
+export type AvailabilityWindow = ProviderAvailabilityWindow;
+export type TeacherAnswer = ({
+    answer: string;
+    status?: never;
+} | {
+    answer: null;
+    status: 'missing' | 'abstain' | 'uncertain';
+}) & {
+    p?: number;
+    span?: string;
+};
+export interface ReferenceCall {
+    id: string;
+    teacher: string;
+    provider: string;
+    requestedModel: string;
+    reportedModel: string | null;
+    route: string | null;
+    prompt_digest: string;
+    phase: 'teacher' | 'adjudication';
+    run?: number;
+    check_ids: string[];
+    at: string;
+    wallMs: number;
+    usage: Usage | null;
+    attempts?: readonly InferenceAttempt[];
+    outcome: 'received' | 'completed' | 'invalid-response' | 'generation-error' | 'cancelled';
+    failure_class?: InferenceFailureKind;
+    diagnostic?: InferenceDiagnostic;
+    response_text?: string;
+    answers?: Record<string, TeacherAnswer> | null;
+}
+export interface CheckReference {
+    ready?: boolean;
+    digest: string;
+    template_id: string;
+    teachers: Record<string, TeacherAnswer>;
+    distribution: Record<string, number>;
+    unknown_mass: number;
+    adjudication: TeacherAnswer[];
+    final: string | null;
+    status: 'agreed' | 'adjudicated' | 'uncertain';
+}
+export interface CouncilOutput {
+    outputDigest: string;
+    identity: string;
+    digest: string;
+    complete: boolean;
+    shuffle_secret?: string;
+    checks: Record<string, CheckReference>;
+    calls: ReferenceCall[];
+    active_teachers?: string[];
+    skipped: {
+        teacher: string;
+        reason: string;
+    }[];
+}
+export interface CouncilOptions {
+    outputDigest: string;
+    instances: CheckInstance[];
+    teachers: CouncilTeacher[];
+    providers: Record<string, InferenceProvider>;
+    adjudicator?: string;
+    [COUNCIL_WINDOW_OPTION_KEY]?: AvailabilityWindow;
+    now?: Date;
+    seed?: string;
+    timeoutMs?: number;
+    batchSize?: number;
+    signal?: AbortSignal;
+    previous?: CouncilOutput;
+    onCheckpoint?: (output: CouncilOutput) => void | Promise<void>;
+}
+export interface TrapAssessment {
+    /** Authored status; core always coerces uncertifiable kinds to diagnostic. */
+    known_answer?: boolean;
+    outputId?: string;
+    kind: string;
+    templateId: string;
+    intended: string;
+    actual: string | null;
+    baseline?: string | null;
+}

package/dist/training/graders/references/types.js

@@ -0,0 +1,2 @@
+import { COUNCIL_WINDOW_OPTION_KEY } from '../../../shared/trainingLegacy/index.js';
+//# sourceMappingURL=types.js.map

package/dist/training/graders/references/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/training/graders/references/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,yCAAyC,CAAC"}

package/dist/training/graders/rows.d.ts

@@ -0,0 +1,14 @@
+import type { Grader } from './format/index.js';
+type RowStructure = NonNullable<Grader['context'][string]['rows']>;
+interface Row {
+    id: string;
+    text: string;
+}
+export interface CitedResolution {
+    value?: Row[];
+    missing?: string;
+    notApplicable?: boolean;
+}
+export { moveLeadingCitations } from './format/index.js';
+/** Keep whole heading lines and exact row bytes; preamble never enters a row. */
+export declare function citedRows(source: unknown, unit: string, structure: RowStructure): CitedResolution;

package/dist/training/graders/rows.js

@@ -0,0 +1,39 @@
+export { moveLeadingCitations } from './format/index.js';
+/** Keep whole heading lines and exact row bytes; preamble never enters a row. */
+export function citedRows(source, unit, structure) {
+    if (typeof source !== 'string')
+        return { missing: 'row context must be a string' };
+    if (source.length > 2_000_000)
+        return { missing: 'row context exceeds 2000000 characters' };
+    const heading = new RegExp(structure.heading, 'm');
+    const rows = [];
+    const ids = new Set();
+    for (const line of source.split(/(?<=\n|[\u2028\u2029])|(?<=\r)(?!\n)/u)) {
+        const match = heading.exec(line);
+        if (match) {
+            const id = match[1];
+            if (!id)
+                return { missing: 'row heading has no row id' };
+            if (ids.has(id))
+                return { missing: `duplicate row id ${id}` };
+            ids.add(id);
+            rows.push({ id, text: line });
+        }
+        else if (rows.length)
+            rows.at(-1).text += line;
+    }
+    const cited = new Set();
+    for (const match of unit.matchAll(new RegExp(structure.cite, 'gm'))) {
+        if (!match[1])
+            return { missing: 'citation has no row id' };
+        cited.add(match[1]);
+    }
+    if (!cited.size)
+        return { value: [], notApplicable: true };
+    const unknown = [...cited].filter(id => !ids.has(id));
+    if (unknown.length)
+        return { missing: `unknown cited row id: ${unknown.join(', ')}` };
+    // Context order is stable even when citations repeat or appear in another order.
+    return { value: rows.filter(row => cited.has(row.id)) };
+}
+//# sourceMappingURL=rows.js.map

package/dist/training/graders/rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rows.js","sourceRoot":"","sources":["../../../src/training/graders/rows.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEzD,iFAAiF;AACjF,MAAM,UAAU,SAAS,CAAC,MAAe,EAAE,IAAY,EAAE,SAAuB;IAC9E,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,8BAA8B,EAAE,CAAC;IACnF,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS;QAAE,OAAO,EAAE,OAAO,EAAE,wCAAwC,EAAE,CAAC;IAC5F,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,CAAC,EAAE;gBAAE,OAAO,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC;YACzD,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,EAAE,EAAE,CAAC;YAC9D,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACZ,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAChC,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,IAAI,IAAI,CAAC;IACpD,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;QACpE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC;QAC5D,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,IAAI;QAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IAC3D,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACtD,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,EAAE,OAAO,EAAE,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;IACtF,iFAAiF;IACjF,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AAC1D,CAAC"}

package/dist/training/graders/runtime/accounting.d.ts

@@ -0,0 +1,10 @@
+import type { DecisionAccounting, DecisionBackend, DecisionRequestReceipt } from '../../contracts/grading.js';
+/** Current invocation only: cached scores are not new billable requests. */
+export declare function summarizeDecisionAccounting(receipts: DecisionRequestReceipt[]): DecisionAccounting;
+/** Failure data excludes diagnostics, request bodies and provider response extensions. */
+export declare class DecisionAccountingError extends Error {
+    readonly decisionAccounting: DecisionAccounting;
+    constructor(cause: unknown, decisionAccounting: DecisionAccounting);
+}
+/** Register each backend before calling it; await drained provider work before rejection. */
+export declare function withDecisionFailureAccounting<T>(backends: Set<DecisionBackend>, receipts: DecisionRequestReceipt[], operation: () => Promise<T>): Promise<T>;

package/dist/training/graders/runtime/accounting.js

@@ -0,0 +1,43 @@
+import { randomUUID } from 'node:crypto';
+/** Current invocation only: cached scores are not new billable requests. */
+export function summarizeDecisionAccounting(receipts) {
+    // Compatibility for injected backends: this invocation owns these dispatched
+    // receipts. Assign once before the grade can be cloned, persisted or reused.
+    for (const receipt of receipts)
+        receipt.requestId ??= randomUUID();
+    const sum = (field) => receipts.some(receipt => receipt[field] === null)
+        ? null : receipts.reduce((total, receipt) => total + receipt[field], 0);
+    const latencies = receipts.map(receipt => receipt.latencyMs).sort((a, b) => a - b);
+    const percentile = (p) => latencies.length ? latencies[Math.ceil(p * latencies.length) - 1] : null;
+    return { billing: 'api', requests: receipts.length, inputTokens: sum('inputTokens'), outputTokens: sum('outputTokens'),
+        usd: sum('usd'), latencyMs: { p50: percentile(0.5), p95: percentile(0.95) }, receipts };
+}
+/** Failure data excludes diagnostics, request bodies and provider response extensions. */
+export class DecisionAccountingError extends Error {
+    decisionAccounting;
+    constructor(cause, decisionAccounting) {
+        super(cause instanceof Error ? cause.message : 'Decision operation failed', { cause });
+        this.decisionAccounting = decisionAccounting;
+        this.name = 'DecisionAccountingError';
+    }
+}
+/** Register each backend before calling it; await drained provider work before rejection. */
+export async function withDecisionFailureAccounting(backends, receipts, operation) {
+    try {
+        return await operation();
+    }
+    catch (cause) {
+        const paid = [...backends].filter(backend => backend.capabilities?.billing === 'api');
+        if (!paid.length)
+            throw cause;
+        const allowed = receipts.map(receipt => ({
+            ...(receipt.requestId === undefined ? {} : { requestId: receipt.requestId }),
+            billing: receipt.billing, model: receipt.model, endpoint: receipt.endpoint,
+            inputTokens: receipt.inputTokens, outputTokens: receipt.outputTokens, usd: receipt.usd,
+            estimatedInputTokens: receipt.estimatedInputTokens, estimatedUsd: receipt.estimatedUsd,
+            latencyMs: receipt.latencyMs, status: receipt.status,
+        }));
+        throw new DecisionAccountingError(cause, summarizeDecisionAccounting(allowed));
+    }
+}
+//# sourceMappingURL=accounting.js.map

package/dist/training/graders/runtime/accounting.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"accounting.js","sourceRoot":"","sources":["../../../../src/training/graders/runtime/accounting.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,4EAA4E;AAC5E,MAAM,UAAU,2BAA2B,CAAC,QAAkC;IAC5E,6EAA6E;IAC7E,6EAA6E;IAC7E,KAAK,MAAM,OAAO,IAAI,QAAQ;QAAE,OAAO,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;IACnE,MAAM,GAAG,GAAG,CAAC,KAA6C,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC;QAC9G,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAE,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,UAAU,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5G,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,aAAa,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,cAAc,CAAC;QACpH,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC;AAC5F,CAAC;AAED,0FAA0F;AAC1F,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IACvC,kBAAkB,CAAqB;IAChD,YAAY,KAAc,EAAE,kBAAsC;QAChE,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACvF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAED,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAI,QAA8B,EAAE,QAAkC,EAAE,SAA2B;IACpJ,IAAI,CAAC;QAAC,OAAO,MAAM,SAAS,EAAE,CAAC;IAAC,CAAC;IACjC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,KAAK,KAAK,CAAC,CAAC;QACtF,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,KAAK,CAAC;QAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACvC,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;YAC5E,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1E,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG;YACtF,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY;YACtF,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM;SACrD,CAAC,CAAC,CAAC;QACJ,MAAM,IAAI,uBAAuB,CAAC,KAAK,EAAE,2BAA2B,CAAC,OAAO,CAAC,CAAC,CAAC;IACjF,CAAC;AACH,CAAC"}

package/dist/training/graders/runtime/grade.d.ts

@@ -0,0 +1,3 @@
+import type { GradeOptions, GradeResult } from './types.js';
+/** No golden examples enter requests. Calibration is checked before any scoring. */
+export declare function gradeOutput(options: GradeOptions): Promise<GradeResult>;

package/dist/training/graders/runtime/grade.js

@@ -0,0 +1,161 @@
+import { GRADING_POLICY_VERSION, CALIBRATION_DIGEST_SCHEMA } from '../../contracts/index.js';
+import { allQuestions, decisionModels, expectedOptions, optionsFor, stableDigest } from '../format/index.js';
+import { planCallAdmission } from '../admission.js';
+import { expandInstances } from '../instances.js';
+import { aggregateGrader } from '../aggregation.js';
+import { checkCalibration, createCalibrationIdentity } from '../identity.js';
+import { applyPlatt, inEscalationBand } from '../calibration/index.js';
+import { teacherPolicyDigest } from '../labeling/index.js';
+import { summarizeDecisionAccounting, withDecisionFailureAccounting } from './accounting.js';
+import { decisionModelIdentities } from './identity.js';
+import { escalateGroup } from './teacher.js';
+function goodProbability(question, result) {
+    if (result.status !== 'ok' || !result.probabilities)
+        return undefined;
+    if (Object.keys(result.probabilities).length !== optionsFor(question).length)
+        throw new Error(`Invalid decision probabilities: ${question.id}`);
+    const values = optionsFor(question).map(option => result.probabilities[String(option)]);
+    if (values.some(value => !Number.isFinite(value) || value < 0 || value > 1) ||
+        Math.abs(values.reduce((sum, value) => sum + value, 0) - 1) > 1e-5) {
+        throw new Error(`Invalid decision probabilities: ${question.id}`);
+    }
+    return expectedOptions(question).reduce((sum, option) => sum + result.probabilities[String(option)], 0);
+}
+/** No golden examples enter requests. Calibration is checked before any scoring. */
+export async function gradeOutput(options) {
+    const backends = options.backends ?? (options.grader.model ? { default: options.backend } : {});
+    const receipts = [];
+    return withDecisionFailureAccounting(new Set(Object.values(backends)), receipts, () => gradeOutputInternal(options, receipts));
+}
+async function gradeOutputInternal(options, receipts) {
+    const { grader, calibration, signal } = options;
+    const identityMode = grader.calibration === 'identity' && calibration.identity.mode === 'identity';
+    signal?.throwIfAborted();
+    if (grader.gates.some(gate => 'check' in gate && options.gateOutcomes?.[gate.id] === 'fail')) {
+        return { ...aggregateGrader(grader, expandInstances(grader, options.output, options.context), {}, options.gateOutcomes),
+            judgeCalls: [], decisions: [], ...resultIdentity(options) };
+    }
+    const models = decisionModels(grader);
+    const backends = options.backends ?? (grader.model ? { default: options.backend } : {});
+    const selectedId = (question) => calibration.questions[question]?.model ?? 'default';
+    for (const question of allQuestions(grader)) {
+        if (!backends[selectedId(question.id)])
+            throw new Error(`Decision model ${selectedId(question.id)}: missing backend`);
+    }
+    const reported = await decisionModelIdentities(grader, backends, signal, receipts);
+    const modelIdentities = { ...calibration.identity.modelIdentities, ...reported };
+    const backendModel = grader.models ? stableDigest(modelIdentities) : reported.default;
+    const identity = createCalibrationIdentity(grader, { ...calibration.identity, backendModel,
+        ...(grader.models ? { modelIdentities } : {}),
+        segmenter: undefined, golden: options.goldenDigest, promptTemplateVersion: options.promptTemplateVersion });
+    const checked = checkCalibration(grader, calibration, identity);
+    if (!checked.ok)
+        throw new Error(checked.refusals.join('; '));
+    const questions = new Map(allQuestions(grader).map(question => [question.id, question]));
+    const selectedModel = (question) => models.find(model => model.id === selectedId(question));
+    for (const [id] of questions) {
+        if (calibration.questions[id]?.status === 'calibrated' && selectedModel(id).backend !== 'teacher' &&
+            (!calibration.questions[id].params || !calibration.questions[id].band)) {
+            throw new Error(`question ${id}: calibration lacks Platt parameters or escalation band`);
+        }
+    }
+    const instances = expandInstances(grader, options.output, options.context);
+    const outcomes = {};
+    const groups = new Map();
+    const escalate = (instance) => {
+        let group = groups.get(instance.projectionDigest);
+        if (!group) {
+            group = { projection: instance.projection, projectionDigest: instance.projectionDigest, instances: [] };
+            groups.set(instance.projectionDigest, group);
+        }
+        group.instances.push({ instance, question: questions.get(instance.questionId) });
+        outcomes[instance.key] = { status: 'missing', escalated: true };
+    };
+    const requestsByModel = new Map();
+    for (const instance of instances) {
+        if (instance.state !== 'ready')
+            continue;
+        if (!identityMode && selectedModel(instance.questionId).backend !== 'teacher' && checked.notCalibrated.includes(instance.questionId)) {
+            escalate(instance);
+            continue;
+        }
+        const question = questions.get(instance.questionId);
+        const id = selectedId(instance.questionId);
+        const requests = requestsByModel.get(id) ?? [];
+        requestsByModel.set(id, requests);
+        requests.push({ key: instance.key, type: question.type, optionLabels: optionsFor(question),
+            expectedOptions: expectedOptions(question), instructions: question.ask ?? question.id, projection: instance.projection });
+    }
+    const judgeCalls = [];
+    const decisions = [];
+    const resultMap = new Map();
+    for (const [model, requests] of requestsByModel) {
+        const results = await backends[model].score(requests, signal, receipts);
+        for (const result of results) {
+            if (resultMap.has(result.key) || !requests.some(request => request.key === result.key))
+                throw new Error(`Unexpected decision key: ${result.key}`);
+            resultMap.set(result.key, result);
+            judgeCalls.push(...result.judgeCalls ?? []);
+        }
+        for (const request of requests)
+            decisions.push({ instance: request.key, model,
+                status: resultMap.get(request.key)?.status === 'ok' && resultMap.get(request.key)?.probabilities ? 'ok' : 'missing',
+                ...(resultMap.get(request.key)?.reason ? { reason: resultMap.get(request.key).reason } : {}) });
+    }
+    for (const instance of instances) {
+        const result = resultMap.get(instance.key);
+        if (!result)
+            continue;
+        const raw = goodProbability(questions.get(instance.questionId), result);
+        if (raw === undefined)
+            continue;
+        const fitted = calibration.questions[instance.questionId];
+        const teacherModel = selectedModel(instance.questionId).backend === 'teacher';
+        const pGood = identityMode || teacherModel ? raw : applyPlatt(raw, fitted.params);
+        if (!identityMode && !teacherModel && inEscalationBand(pGood, fitted.band))
+            escalate(instance);
+        else
+            outcomes[instance.key] = { pGood };
+    }
+    const calls = planCallAdmission(instances.filter(instance => outcomes[instance.key]?.escalated)
+        .map(instance => ({ outputId: 'trial', projectionDigest: instance.projectionDigest })), grader.escalate.max_calls_per_trial);
+    for (const call of calls) {
+        if (!options.teacher || !call.admitted)
+            continue;
+        const group = groups.get(call.projectionDigest);
+        group.instances.sort((a, b) => instances.indexOf(a.instance) - instances.indexOf(b.instance));
+        const definition = grader.teachers.find(teacher => teacher.id === grader.escalate.to);
+        if (options.teacher.model !== definition.model || options.teacher.provider.id !== `${definition.provider}-subscription`) {
+            throw new Error('Escalation teacher route does not match declared policy');
+        }
+        const result = await escalateGroup(group, options.teacher, definition.id, signal);
+        judgeCalls.push(result.call);
+        for (const { instance, question } of group.instances) {
+            const vote = result.votes?.[instance.key];
+            if (vote !== undefined && vote !== 'abstain')
+                outcomes[instance.key] = {
+                    pGood: Number(expectedOptions(question).includes(vote)), escalated: true
+                };
+        }
+    }
+    signal?.throwIfAborted();
+    if (stableDigest(await decisionModelIdentities(grader, backends, signal, receipts)) !== stableDigest(reported))
+        throw new Error('Backend identity changed during grading');
+    const apiBackends = [...new Set(Object.values(backends))].filter(backend => backend.capabilities?.billing === 'api');
+    return { ...aggregateGrader(grader, instances, outcomes, options.gateOutcomes), judgeCalls, decisions,
+        ...(apiBackends.length ? { decisionAccounting: summarizeDecisionAccounting(receipts) } : {}),
+        ...resultIdentity(options, identity) };
+}
+function resultIdentity(options, identity = options.calibration.identity) {
+    const { grader, calibration } = options;
+    const selected = new Set(Object.values(calibration.questions).map(question => question.model ?? 'default'));
+    const billing = new Set(decisionModels(grader).filter(model => selected.has(model.id)).map(model => model.backend === 'teacher' ? 'subscription' : model.backend === 'typesafe' ? 'api' : 'local'));
+    return {
+        backendBilling: billing.size > 1 ? 'mixed' : billing.has('subscription') ? 'subscription' : billing.has('api') ? 'api' : 'local',
+        identity: { calibration: identity, calibrationDigest: stableDigest({ ...calibration, schema: CALIBRATION_DIGEST_SCHEMA }), escalation: stableDigest(grader.escalate),
+            teacherPolicy: stableDigest({ escalation: grader.calibration === 'identity' ? null : teacherPolicyDigest(grader.teachers.find(t => t.id === grader.escalate.to)),
+                uncalibrated: grader.uncalibrated }),
+            scoringPolicy: stableDigest({ version: GRADING_POLICY_VERSION, questions: grader.questions, gates: grader.gates, score: grader.score }) }
+    };
+}
+//# sourceMappingURL=grade.js.map

package/dist/training/graders/runtime/grade.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../../../src/training/graders/runtime/grade.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAE7F,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAyB,MAAM,oBAAoB,CAAC;AACpI,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAwB,MAAM,mBAAmB,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAC7E,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,mBAAmB,EAAmB,MAAM,sBAAsB,CAAC;AAC5E,OAAO,EAAE,2BAA2B,EAAE,6BAA6B,EAAE,MAAM,iBAAiB,CAAC;AAC7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAG7C,SAAS,eAAe,CAAC,QAA0B,EAAE,MAAsB;IACzE,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa;QAAE,OAAO,SAAS,CAAC;IACtE,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IAChJ,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,aAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACzF,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAM,GAAG,CAAC,IAAI,KAAM,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC;QAC9E,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,aAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAE,EAAE,CAAC,CAAC,CAAC;AACpH,CAAC;AAED,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAqB;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAChG,MAAM,QAAQ,GAA6B,EAAE,CAAC;IAC9C,OAAO,6BAA6B,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjI,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,OAAqB,EAAE,QAAkC;IAC1F,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,KAAK,UAAU,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU,CAAC;IACnG,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,IAAI,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,MAAM,CAAC,EAAE,CAAC;QAC7F,OAAO,EAAE,GAAG,eAAe,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,YAAY,CAAC;YACrH,UAAU,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;IAChE,CAAC;IACD,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxF,MAAM,UAAU,GAAG,CAAC,QAAgB,EAAE,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,KAAK,IAAI,SAAS,CAAC;IAC7F,KAAK,MAAM,QAAQ,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAAC;IACxH,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,uBAAuB,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IACnF,MAAM,eAAe,GAAG,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,eAAe,EAAE,GAAG,QAAQ,EAAE,CAAC;IACjF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAQ,CAAC;IACvF,MAAM,QAAQ,GAAG,yBAAyB,CAAC,MAAM,EAAE,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,YAAY;QACxF,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC;IAC9G,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;IAChE,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,UAAU,CAAC,QAAQ,CAAC,CAAE,CAAC;IACrG,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,MAAM,KAAK,YAAY,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC,OAAO,KAAK,SAAS;YAC/F,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAE,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3E,MAAM,IAAI,KAAK,CAAC,YAAY,EAAE,yDAAyD,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAoC,EAAE,CAAC;IACrD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,MAAM,QAAQ,GAAG,CAAC,QAAkC,EAAE,EAAE;QACtD,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;YACxG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAE,EAAE,CAAC,CAAC;QAClF,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAClE,CAAC,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,GAAG,EAA6B,CAAC;IAC7D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO;YAAE,SAAS;QACzC,IAAI,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACvK,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAE,CAAC;QACrD,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/C,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QAClC,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,YAAY,EAAE,UAAU,CAAC,QAAQ,CAAC;YACxF,eAAe,EAAE,eAAe,CAAC,QAAQ,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IAC9H,CAAC;IACD,MAAM,UAAU,GAA8B,EAAE,CAAC;IACjD,MAAM,SAAS,GAA0C,EAAE,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAC;IACpD,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,eAAe,EAAE,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;YAClJ,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAClC,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,QAAQ;YAAE,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK;gBAC3E,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACnH,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACrG,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,GAAG,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAE,EAAE,MAAM,CAAC,CAAC;QACzE,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAE,CAAC;QAC3D,MAAM,YAAY,GAAG,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC;QAC9E,MAAM,KAAK,GAAG,YAAY,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAO,CAAC,CAAC;QACnF,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,IAAK,CAAC;YAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;;YAC3F,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;IAC1C,CAAC;IACD,MAAM,KAAK,GAAG,iBAAiB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,SAAS,CAAC;SAC5F,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC/H,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,SAAS;QACjD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAE,CAAC;QACjD,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9F,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAE,CAAC;QACvF,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,KAAK,UAAU,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,QAAQ,eAAe,EAAE,CAAC;YACxH,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAClF,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;gBAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG;oBACrE,KAAK,EAAE,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI;iBAAE,CAAC;QAC/E,CAAC;IACH,CAAC;IACD,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,IAAI,YAAY,CAAC,MAAM,uBAAuB,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,KAAK,YAAY,CAAC,QAAQ,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC3K,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,KAAK,KAAK,CAAC,CAAC;IACrH,OAAO,EAAE,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,SAAS;QACnG,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,2BAA2B,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG,cAAc,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,cAAc,CAAC,OAAqB,EAAE,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ;IACpF,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC;IAC5G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACpM,OAAO;QACL,cAAc,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO;QAChI,QAAQ,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,iBAAiB,EAAE,YAAY,CAAC,EAAE,GAAG,WAAW,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC;YAClK,aAAa,EAAE,YAAY,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAE,CAAC;gBAC/J,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;YACtC,aAAa,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,sBAAsB,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE;KAAE,CAAC;AAClJ,CAAC"}

package/dist/training/graders/runtime/identity.d.ts

@@ -0,0 +1,6 @@
+import type { DecisionBackend, DecisionRequestReceipt } from '../../contracts/grading.js';
+import { type Grader } from '../format/index.js';
+/** Shared by calibration and grading; pins routing in addition to reported model. */
+export declare function decisionBackendIdentity(grader: Grader, backend: DecisionBackend, signal?: AbortSignal, receipts?: DecisionRequestReceipt[]): Promise<string>;
+/** Probe every referenced route; omitted routes retain their pinned calibration identity. */
+export declare function decisionModelIdentities(grader: Grader, backends: Record<string, DecisionBackend>, signal?: AbortSignal, receipts?: DecisionRequestReceipt[]): Promise<Record<string, string>>;

package/dist/training/graders/runtime/identity.js

@@ -0,0 +1,24 @@
+import { decisionModels, graderForModel, stableDigest } from '../format/index.js';
+/** Shared by calibration and grading; pins routing in addition to reported model. */
+export async function decisionBackendIdentity(grader, backend, signal, receipts) {
+    const model = grader.model;
+    return stableDigest({ configured: model, reported: await backend.identity(signal, receipts),
+        ...(model.backend === 'teacher' ? { teacher: grader.teachers.find(t => t.id === model.teacher) } : {}) });
+}
+/** Probe every referenced route; omitted routes retain their pinned calibration identity. */
+export async function decisionModelIdentities(grader, backends, signal, receipts) {
+    const identities = {};
+    for (const model of decisionModels(grader)) {
+        const backend = backends[model.id];
+        if (!backend)
+            continue;
+        try {
+            identities[model.id] = await decisionBackendIdentity(graderForModel(grader, model), backend, signal, receipts);
+        }
+        catch (cause) {
+            throw new Error(`Decision model ${model.id}: identity failed`, { cause });
+        }
+    }
+    return identities;
+}
+//# sourceMappingURL=identity.js.map

package/dist/training/graders/runtime/identity.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"identity.js","sourceRoot":"","sources":["../../../../src/training/graders/runtime/identity.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,EAAe,MAAM,oBAAoB,CAAC;AAE/F,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,MAAc,EAAE,OAAwB,EAAE,MAAoB,EAAE,QAAmC;IAC/I,MAAM,KAAK,GAAG,MAAM,CAAC,KAAM,CAAC;IAC5B,OAAO,YAAY,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;QACzF,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAC9G,CAAC;AAED,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,MAAc,EAAE,QAAyC,EACrG,MAAoB,EAAE,QAAmC;IACzD,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,CAAC;YAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,MAAM,uBAAuB,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAAC,CAAC;QACvH,OAAO,KAAK,EAAE,CAAC;YAAC,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,CAAC,EAAE,mBAAmB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAAC,CAAC;IAC9F,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/training/graders/runtime/index.d.ts

@@ -0,0 +1,5 @@
+export { gradeOutput } from './grade.js';
+export { decisionModelIdentities, decisionBackendIdentity } from './identity.js';
+export type { GradeOptions, GradeResult, GradingRuntime, GraderTeacher } from './types.js';
+export { decisionAccountingSchema, createGradeResultSchema } from './schema.js';
+export { DecisionAccountingError, withDecisionFailureAccounting, summarizeDecisionAccounting } from './accounting.js';

package/dist/training/graders/runtime/index.js

@@ -0,0 +1,5 @@
+export { gradeOutput } from './grade.js';
+export { decisionModelIdentities, decisionBackendIdentity } from './identity.js';
+export { decisionAccountingSchema, createGradeResultSchema } from './schema.js';
+export { DecisionAccountingError, withDecisionFailureAccounting, summarizeDecisionAccounting } from './accounting.js';
+//# sourceMappingURL=index.js.map

package/dist/training/graders/runtime/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/training/graders/runtime/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAEjF,OAAO,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAChF,OAAO,EAAE,uBAAuB,EAAE,6BAA6B,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/training/graders/runtime/schema.d.ts

@@ -0,0 +1,29 @@
+import { z } from 'zod';
+import type { JudgeCall } from '../../contracts/evaluation.js';
+import type { GradeResult } from './types.js';
+export declare const decisionAccountingSchema: z.ZodObject<{
+    billing: z.ZodLiteral<"api">;
+    requests: z.ZodNumber;
+    inputTokens: z.ZodNullable<z.ZodNumber>;
+    outputTokens: z.ZodNullable<z.ZodNumber>;
+    usd: z.ZodNullable<z.ZodNumber>;
+    latencyMs: z.ZodObject<{
+        p50: z.ZodNullable<z.ZodNumber>;
+        p95: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    receipts: z.ZodArray<z.ZodObject<{
+        requestId: z.ZodOptional<z.ZodString>;
+        billing: z.ZodLiteral<"api">;
+        model: z.ZodString;
+        endpoint: z.ZodString;
+        inputTokens: z.ZodNullable<z.ZodNumber>;
+        outputTokens: z.ZodNullable<z.ZodNumber>;
+        usd: z.ZodNullable<z.ZodNumber>;
+        estimatedInputTokens: z.ZodNumber;
+        estimatedUsd: z.ZodNumber;
+        latencyMs: z.ZodNumber;
+        status: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+/** Persisted grading results use the host's existing, explicit judge-call schema. */
+export declare function createGradeResultSchema(judgeCall: z.ZodType<JudgeCall>): z.ZodType<GradeResult>;

package/dist/training/graders/runtime/schema.js

@@ -0,0 +1,27 @@
+import { z } from 'zod';
+import { calibrationIdentitySchema } from '../identity.js';
+const count = z.number().int().nonnegative();
+export const decisionAccountingSchema = z.strictObject({ billing: z.literal('api'), requests: count,
+    inputTokens: count.nullable(), outputTokens: count.nullable(), usd: z.number().nonnegative().nullable(),
+    latencyMs: z.strictObject({ p50: z.number().nonnegative().nullable(), p95: z.number().nonnegative().nullable() }),
+    receipts: z.array(z.strictObject({ requestId: z.string().min(1).optional(), billing: z.literal('api'), model: z.string(), endpoint: z.string(),
+        inputTokens: count.nullable(), outputTokens: count.nullable(), usd: z.number().nonnegative().nullable(),
+        estimatedInputTokens: count, estimatedUsd: z.number().nonnegative(), latencyMs: z.number().nonnegative(), status: count.nullable() })),
+});
+/** Persisted grading results use the host's existing, explicit judge-call schema. */
+export function createGradeResultSchema(judgeCall) {
+    const probability = z.number().finite().min(0).max(1);
+    const digest = z.string().regex(/^[a-f0-9]{64}$/);
+    return z.strictObject({
+        status: z.enum(['pass', 'fail', 'unknown']), score: probability, coverage: probability,
+        gates: z.array(z.strictObject({ id: z.string().min(1), status: z.enum(['pass', 'fail', 'unknown', 'error']) })),
+        questions: z.array(z.strictObject({ id: z.string().min(1), score: probability,
+            instances: count, not_applicable: count, escalated: count, missing: count })),
+        identity: z.strictObject({ calibration: calibrationIdentitySchema, calibrationDigest: digest,
+            escalation: digest, teacherPolicy: digest, scoringPolicy: digest }),
+        backendBilling: z.enum(['local', 'subscription', 'api', 'mixed']), judgeCalls: z.array(judgeCall),
+        decisionAccounting: decisionAccountingSchema.optional(),
+        decisions: z.array(z.strictObject({ instance: z.string().min(1), model: z.string().min(1), status: z.enum(['ok', 'missing']), reason: z.string().max(200).optional() })).optional(),
+    });
+}
+//# sourceMappingURL=schema.js.map

package/dist/training/graders/runtime/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../../src/training/graders/runtime/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAG3D,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;AAC7C,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,KAAK;IACjG,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IACvG,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjH,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QAC5I,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;QACvG,oBAAoB,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;CACzI,CAAC,CAAC;AAEH,qFAAqF;AACrF,MAAM,UAAU,uBAAuB,CAAC,SAA+B;IACrE,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC,YAAY,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW;QACtF,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/G,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW;YAC3E,SAAS,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/E,QAAQ,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,WAAW,EAAE,yBAAyB,EAAE,iBAAiB,EAAE,MAAM;YAC1F,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;QACrE,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;QACjG,kBAAkB,EAAE,wBAAwB,CAAC,QAAQ,EAAE;QACvD,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;KACpL,CAAC,CAAC;AACL,CAAC"}

package/dist/training/graders/runtime/teacher.d.ts

@@ -0,0 +1,8 @@
+import type { JudgeCall } from '../../contracts/evaluation.js';
+import { type LabelGroup } from '../labeling/index.js';
+import type { GraderTeacher } from './types.js';
+/** Grading makes exactly one attempt per projection; failure spends its call slot. */
+export declare function escalateGroup(group: LabelGroup, teacher: GraderTeacher, name: string, signal?: AbortSignal): Promise<{
+    votes: Record<string, import("../index.js").Option> | null;
+    call: JudgeCall;
+}>;

package/dist/training/graders/runtime/teacher.js

@@ -0,0 +1,43 @@
+import { randomUUID } from 'node:crypto';
+import { buildLabelPrompt, decodeLabelVotes } from '../labeling/index.js';
+/** Grading makes exactly one attempt per projection; failure spends its call slot. */
+export async function escalateGroup(group, teacher, name, signal) {
+    if (teacher.provider.capabilities.billing !== 'subscription' || !['disabled', 'contained'].includes(teacher.provider.capabilities.tools)) {
+        throw new Error('Escalation teacher requires a tool-isolated subscription provider');
+    }
+    signal?.throwIfAborted();
+    const request = buildLabelPrompt(group);
+    const start = performance.now();
+    const call = { id: randomUUID(), judge: name, providerId: teacher.provider.id,
+        requestedModel: teacher.model, responseModel: null, responseRoute: null,
+        outcome: 'generation-error', usage: null };
+    let votes = null;
+    try {
+        const response = await teacher.provider.generate({ model: teacher.model, system: request.system,
+            prompt: request.prompt, responseSchema: request.schema, signal });
+        call.responseModel = response.modelBasis === 'requested-assumption' ? null : response.model;
+        call.responseRoute = response.route;
+        call.usage = response.usage;
+        votes = decodeLabelVotes(response.text, group);
+        call.outcome = votes ? 'completed' : 'invalid-response';
+    }
+    catch (error) {
+        signal?.throwIfAborted();
+        // Neutral provider failures may carry measured evidence; never parse error prose.
+        if (error !== null && typeof error === 'object') {
+            const failure = error;
+            if (error instanceof Error && failure.kind === 'protocol' && failure.detail === 'tool-call') {
+                call.outcome = 'refused';
+                call.failureKind = 'protocol';
+                call.failureDetail = 'tool-call';
+            }
+            call.usage = failure.usage ?? null;
+            call.responseModel = failure.responseModel ?? null;
+            call.responseRoute = failure.responseRoute ?? null;
+        }
+    }
+    call.wallMs = performance.now() - start;
+    call.billing = 'subscription';
+    return { votes, call };
+}
+//# sourceMappingURL=teacher.js.map

package/dist/training/graders/runtime/teacher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"teacher.js","sourceRoot":"","sources":["../../../../src/training/graders/runtime/teacher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAmB,MAAM,sBAAsB,CAAC;AAG3F,sFAAsF;AACtF,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAiB,EAAE,OAAsB,EAAE,IAAY,EAAE,MAAoB;IAC/G,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,KAAK,cAAc,IAAI,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACzI,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IACD,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,IAAI,GAAc,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE;QACtF,cAAc,EAAE,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI;QACvE,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAC7C,IAAI,KAAK,GAAwC,IAAI,CAAC;IACtD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM;YAC7F,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACpE,IAAI,CAAC,aAAa,GAAG,QAAQ,CAAC,UAAU,KAAK,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;QAC5F,IAAI,CAAC,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC5B,KAAK,GAAG,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,kFAAkF;QAClF,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAChD,MAAM,OAAO,GAAG,KAAqH,CAAC;YACtI,IAAI,KAAK,YAAY,KAAK,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAC5F,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;gBAAC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;gBAAC,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC;YAC5F,CAAC;YACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;YACnC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;YACnD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;QACrD,CAAC;IACH,CAAC;IACD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IACxC,IAAI,CAAC,OAAO,GAAG,cAAc,CAAC;IAC9B,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC"}