spawnfile 0.1.24 → 0.1.27

package/dist/training/graders/format/labels.js

@@ -0,0 +1,109 @@
+import { LABELS_DIGEST_SCHEMA } from '../../contracts/index.js';
+import { checkTrainingFilename } from '../../platform/index.js';
+import { compatibleWireLiteral } from '../../contracts/index.js';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod';
+import { identifier, provider, parseSchema, parseYaml, sha256, stableDigest, rejectUnsafeKeys, sourceDirectory } from './common.js';
+import { optionsFor, allQuestions } from './questions.js';
+const referenceSchema = z.strictObject({ from: z.string().min(1), sha256: z.string().regex(/^[a-f0-9]{64}$/) });
+const labelerSchema = z.union([
+    z.strictObject({ kind: z.literal('model'), provider, model: z.string().min(1) }),
+    z.strictObject({ kind: z.literal('signal'), name: z.string().min(1), source: z.string().min(1) }),
+    z.strictObject({ kind: z.literal('human'), name: z.string().min(1) }),
+    z.strictObject({ kind: z.literal('editor'), name: z.string().min(1), source: z.string().min(1).optional() }),
+]);
+export const labelsSchema = z.strictObject({
+    schema: compatibleWireLiteral('spawnfile.training.labels.v1'), grader: identifier,
+    labelers: z.record(identifier, labelerSchema),
+    labels: z.array(z.strictObject({
+        id: identifier, lineage: identifier,
+        example: z.strictObject({ dataset: identifier, id: identifier }).optional(),
+        output: z.json(), context: z.record(identifier, z.json()),
+        asked: z.array(identifier).optional(),
+        askedByInstance: z.record(identifier, z.array(identifier)).optional(),
+        votes: z.record(identifier, z.record(identifier, z.union([z.string(), z.number().int()]))),
+        provenance: z.record(identifier, z.record(z.string(), z.json())),
+    })),
+});
+function validateReference(value, location) {
+    if (value !== null && typeof value === 'object' && !Array.isArray(value) && 'from' in value) {
+        try {
+            parseSchema(referenceSchema, value);
+        }
+        catch (error) {
+            throw new Error(`${location}: ${String(error)}`, { cause: error });
+        }
+    }
+}
+export function parseLabels(value, grader) {
+    const labels = parseSchema(labelsSchema, value);
+    if (labels.grader !== grader.id)
+        throw new Error('$.grader: does not match grader.id');
+    const questions = new Map(allQuestions(grader).map(question => [question.id, question]));
+    const ids = new Set();
+    labels.labels.forEach((label, i) => {
+        const location = `$.labels.${i}`;
+        if (ids.has(label.id))
+            throw new Error(`${location}.id: duplicate label`);
+        ids.add(label.id);
+        validateReference(label.output, `${location}.output`);
+        for (const [name, value] of Object.entries(label.context)) {
+            if (!Object.hasOwn(grader.context, name))
+                throw new Error(`${location}.context.${name}: undeclared context`);
+            validateReference(value, `${location}.context.${name}`);
+        }
+        for (const name of [...label.asked ?? [], ...Object.values(label.askedByInstance ?? {}).flat(), ...Object.keys(label.provenance)]) {
+            if (!Object.hasOwn(labels.labelers, name))
+                throw new Error(`${location}.asked/provenance.${name}: unknown labeler`);
+        }
+        for (const [key, votes] of Object.entries({ ...Object.fromEntries(Object.keys(label.askedByInstance ?? {}).map(key => [key, {}])), ...label.votes })) {
+            const match = /^(.*?)(?:@p(0|[1-9]\d*)(?:\.s(0|[1-9]\d*))?)?$/s.exec(key);
+            const question = questions.get(match[1]);
+            if (!question)
+                throw new Error(`${location}.votes.${key}: unknown question`);
+            const hasParagraph = match[2] !== undefined;
+            const hasSentence = match[3] !== undefined;
+            if (question.level === 'article' ? hasParagraph : question.level === 'paragraph' ? !hasParagraph || hasSentence : !hasSentence) {
+                throw new Error(`${location}.votes.${key}: instance suffix does not match level`);
+            }
+            const options = optionsFor(question);
+            for (const [labeler, vote] of Object.entries(votes)) {
+                if (!Object.hasOwn(labels.labelers, labeler))
+                    throw new Error(`${location}.votes.${key}.${labeler}: unknown labeler`);
+                if (vote !== 'abstain' && !options.includes(vote))
+                    throw new Error(`${location}.votes.${key}.${labeler}: invalid option`);
+            }
+        }
+    });
+    return labels;
+}
+export function parseLabelsYaml(source, grader) { return parseLabels(parseYaml(source), grader); }
+/** JSON references decode JSON; other references retain exact UTF-8 text. */
+async function resolveValue(value, directory, label) {
+    if (value === null || typeof value !== 'object' || Array.isArray(value) || !('from' in value))
+        return value;
+    const reference = parseSchema(referenceSchema, value);
+    const bytes = await readFile(path.resolve(directory, reference.from));
+    if (sha256(bytes) !== reference.sha256)
+        throw new Error(`label ${label}: sha256 mismatch for ${reference.from}`);
+    const resolved = reference.from.endsWith('.json') ? JSON.parse(bytes.toString('utf8')) : bytes.toString('utf8');
+    rejectUnsafeKeys(resolved, `label ${label}`);
+    return resolved;
+}
+export async function loadLabels(source, grader) {
+    if (typeof source === 'string')
+        checkTrainingFilename(source);
+    const labels = typeof source === 'string' ? parseLabelsYaml(await readFile(source, 'utf8'), grader)
+        : parseLabels({ ...source, grader: source.grader ?? grader.id }, grader);
+    const directory = typeof source === 'string' ? path.dirname(source) : sourceDirectory(grader);
+    // Identity includes original content addresses, votes, asked set and provenance.
+    const goldenDigest = stableDigest({ ...labels, schema: LABELS_DIGEST_SCHEMA });
+    const resolved = await Promise.all(labels.labels.map(async (label) => ({
+        ...label,
+        output: await resolveValue(label.output, directory, label.id),
+        context: Object.fromEntries(await Promise.all(Object.entries(label.context).map(async ([name, value]) => [name, await resolveValue(value, directory, label.id)]))),
+    })));
+    return { labels: { ...labels, labels: resolved }, goldenDigest };
+}
+//# sourceMappingURL=labels.js.map

package/dist/training/graders/format/labels.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"labels.js","sourceRoot":"","sources":["../../../../src/training/graders/format/labels.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEpI,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE1D,MAAM,eAAe,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;AAChH,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC;IAC5B,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACjG,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;CAC7G,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;IACzC,MAAM,EAAE,qBAAqB,CAAC,8BAA8B,CAAC,EAAE,MAAM,EAAE,UAAU;IACjF,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC;IAC7C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;QAC7B,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU;QACnC,OAAO,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE;QAC3E,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACzD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAE;QACrC,eAAe,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE;QACrE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1F,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;KACjE,CAAC,CAAC;CACJ,CAAC,CAAC;AAKH,SAAS,iBAAiB,CAAC,KAAc,EAAE,QAAgB;IACzD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QAC5F,IAAI,CAAC;YAAC,WAAW,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QAAC,CAAC;QAC5C,OAAO,KAAK,EAAE,CAAC;YAAC,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAc,EAAE,MAAc;IACxD,MAAM,MAAM,GAAG,WAAW,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACvF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACjC,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,CAAC;QACjC,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,sBAAsB,CAAC,CAAC;QAC1E,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClB,iBAAiB,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,QAAQ,SAAS,CAAC,CAAC;QACtD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,YAAY,IAAI,sBAAsB,CAAC,CAAC;YAC7G,iBAAiB,CAAC,KAAK,EAAE,GAAG,QAAQ,YAAY,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YAClI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,qBAAqB,IAAI,mBAAmB,CAAC,CAAC;QACtH,CAAC;QACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;YACrJ,MAAM,KAAK,GAAG,iDAAiD,CAAC,IAAI,CAAC,GAAG,CAAE,CAAC;YAC3E,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,UAAU,GAAG,oBAAoB,CAAC,CAAC;YAC7E,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC;YAC5C,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC;YAC3C,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,YAAY,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC/H,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,UAAU,GAAG,wCAAwC,CAAC,CAAC;YACpF,CAAC;YACD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACrC,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,UAAU,GAAG,IAAI,OAAO,mBAAmB,CAAC,CAAC;gBACtH,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,UAAU,GAAG,IAAI,OAAO,kBAAkB,CAAC,CAAC;YAC5H,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,MAAc,IAAY,OAAO,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAE1H,6EAA6E;AAC7E,KAAK,UAAU,YAAY,CAAC,KAAsB,EAAE,SAAiB,EAAE,KAAa;IAClF,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5G,MAAM,SAAS,GAAG,WAAW,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,yBAAyB,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IACjH,MAAM,QAAQ,GAAoB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAoB,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpJ,gBAAgB,CAAC,QAAQ,EAAE,SAAS,KAAK,EAAE,CAAC,CAAC;IAC7C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,MAAwB,EAAE,MAAc;IACvE,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QACjG,CAAC,CAAC,WAAW,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IAC9F,iFAAiF;IACjF,MAAM,YAAY,GAAG,YAAY,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC;QACnE,GAAG,KAAK;QACR,MAAM,EAAE,MAAM,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;QAC7D,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CACtG,CAAC,IAAI,EAAE,MAAM,YAAY,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;KAC5D,CAAC,CAAC,CAAC,CAAC;IACL,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,CAAC;AACnE,CAAC"}

package/dist/training/graders/format/questions.d.ts

@@ -0,0 +1,67 @@
+import { z } from 'zod';
+import type { Grader } from './grader.js';
+export declare const questionSchema: z.ZodObject<{
+    weight: z.ZodDefault<z.ZodNumber>;
+    id: z.ZodString;
+    level: z.ZodDefault<z.ZodEnum<{
+        article: "article";
+        paragraph: "paragraph";
+        sentence: "sentence";
+    }>>;
+    ask: z.ZodOptional<z.ZodString>;
+    type: z.ZodEnum<{
+        score: "score";
+        "yes-no": "yes-no";
+        choice: "choice";
+    }>;
+    expected: z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodArray<z.ZodString>, z.ZodArray<z.ZodNumber>]>;
+    options: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    scale: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+    reads: z.ZodArray<z.ZodString>;
+    when: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
+        field: z.ZodString;
+        equals: z.ZodJSONSchema;
+    }, z.core.$strict>, z.ZodObject<{
+        field: z.ZodString;
+        in: z.ZodArray<z.ZodJSONSchema>;
+    }, z.core.$strict>]>]>>;
+    not_applicable: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const gateQuestionSchema: z.ZodObject<{
+    id: z.ZodString;
+    level: z.ZodDefault<z.ZodEnum<{
+        article: "article";
+        paragraph: "paragraph";
+        sentence: "sentence";
+    }>>;
+    ask: z.ZodOptional<z.ZodString>;
+    type: z.ZodEnum<{
+        score: "score";
+        "yes-no": "yes-no";
+        choice: "choice";
+    }>;
+    expected: z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodArray<z.ZodString>, z.ZodArray<z.ZodNumber>]>;
+    options: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    scale: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+    reads: z.ZodArray<z.ZodString>;
+    when: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
+        field: z.ZodString;
+        equals: z.ZodJSONSchema;
+    }, z.core.$strict>, z.ZodObject<{
+        field: z.ZodString;
+        in: z.ZodArray<z.ZodJSONSchema>;
+    }, z.core.$strict>]>]>>;
+    not_applicable: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strict>;
+export type Question = z.infer<typeof questionSchema>;
+export type GateQuestion = z.infer<typeof gateQuestionSchema>;
+export type DecisionQuestion = Question | GateQuestion;
+export type Option = string | number;
+export declare function optionsFor(question: DecisionQuestion): Option[];
+export declare function expectedOptions(question: DecisionQuestion): Option[];
+export declare function validRead(read: string, question: DecisionQuestion, context: Record<string, unknown>): boolean;
+export declare function validateQuestion(question: DecisionQuestion, context: Record<string, unknown>, path: string): void;
+export declare function applicabilityQuestion(question: DecisionQuestion): GateQuestion;
+export declare function allQuestions(grader: Grader): DecisionQuestion[];
+/** Cited-row extraction is executable question semantics, including for derived questions. */
+export declare function questionDefinitionDigest(grader: Grader, question: DecisionQuestion): string;

package/dist/training/graders/format/questions.js

@@ -0,0 +1,104 @@
+import { z } from 'zod';
+import { identifier, probability, stableDigest } from './common.js';
+import { sentenceCitationPolicy } from './rows.js';
+const predicate = z.union([
+    z.strictObject({ field: z.string().min(1), equals: z.json() }),
+    z.strictObject({ field: z.string().min(1), in: z.array(z.json()).min(1) }),
+]);
+const fields = {
+    id: identifier,
+    level: z.enum(['article', 'paragraph', 'sentence']).default('article'),
+    ask: z.string().min(1).optional(),
+    type: z.enum(['yes-no', 'choice', 'score']),
+    expected: z.union([z.string(), z.number(), z.array(z.string()), z.array(z.number())]),
+    options: z.array(identifier).min(2).max(16).optional(),
+    scale: z.tuple([z.number().int(), z.number().int()]).optional(),
+    reads: z.array(z.string().min(1)),
+    when: z.union([z.string().min(1), predicate]).optional(),
+    not_applicable: probability.default(1),
+};
+export const questionSchema = z.strictObject({ ...fields, weight: z.number().nonnegative().default(1) });
+export const gateQuestionSchema = z.strictObject(fields);
+export function optionsFor(question) {
+    if (question.type === 'yes-no')
+        return ['yes', 'no'];
+    if (question.type === 'choice')
+        return question.options;
+    const [lo, hi] = question.scale;
+    return Array.from({ length: hi - lo + 1 }, (_, i) => lo + i);
+}
+export function expectedOptions(question) {
+    const expected = question.expected;
+    if (question.type !== 'score' || typeof expected !== 'string') {
+        return Array.isArray(expected) ? expected : [expected];
+    }
+    const match = /^(>=|<=)\s*(-?\d+)$/.exec(expected);
+    if (!match)
+        return [];
+    const n = Number(match[2]);
+    return optionsFor(question).filter(value => match[1] === '>=' ? Number(value) >= n : Number(value) <= n);
+}
+export function validRead(read, question, context) {
+    if (/^output\.[^.]+(?:\.[^.]+)*$/.test(read))
+        return true;
+    if (read.startsWith('context.')) {
+        if (read.endsWith('@cited')) {
+            const declaration = context[read.slice(8, -6)];
+            return question.level !== 'article' && declaration?.rows !== undefined;
+        }
+        return Object.hasOwn(context, read.slice(8));
+    }
+    if (read === 'sentence')
+        return question.level === 'sentence';
+    return ['paragraph', 'previous-paragraph', 'next-paragraph'].includes(read) && question.level !== 'article';
+}
+export function validateQuestion(question, context, path) {
+    const fail = (field, message) => { throw new Error(`${path}.${field}: ${message}`); };
+    if (question.type === 'choice' && (!question.options || new Set(question.options).size !== question.options.length || question.options.includes('abstain'))) {
+        fail('options', 'requires 2–16 unique options; abstain is reserved');
+    }
+    if (question.type === 'score' && (!question.scale || question.scale[1] - question.scale[0] < 1 ||
+        question.scale[1] - question.scale[0] > 15))
+        fail('scale', 'requires 2–16 integer points');
+    if (question.type !== 'choice' && question.options)
+        fail('options', 'only valid for choice');
+    if (question.type !== 'score' && question.scale)
+        fail('scale', 'only valid for score');
+    if (question.type === 'yes-no' && (typeof question.expected !== 'string' || !['yes', 'no'].includes(question.expected)))
+        fail('expected', 'requires yes or no');
+    if (question.type === 'choice' && !Array.isArray(question.expected))
+        fail('expected', 'requires an option subset');
+    const expected = expectedOptions(question);
+    const options = optionsFor(question);
+    if (!expected.length || expected.length >= options.length || new Set(expected).size !== expected.length ||
+        expected.some(value => !options.includes(value)))
+        fail('expected', 'requires a nonempty proper subset of options');
+    question.reads.forEach((read, index) => {
+        if (!validRead(read, question, context))
+            fail(`reads.${index}`, 'unknown or unavailable read');
+    });
+    if (question.when && typeof question.when !== 'string' && !validRead(question.when.field, question, context)) {
+        fail('when.field', 'unknown or unavailable field');
+    }
+}
+export function applicabilityQuestion(question) {
+    return {
+        id: `${question.id}.applies`, level: question.level, ask: question.when,
+        type: 'yes-no', expected: 'yes', reads: [...question.reads], not_applicable: 1,
+    };
+}
+export function allQuestions(grader) {
+    const questions = [...grader.questions, ...grader.gates.filter((gate) => !('check' in gate))];
+    return questions.flatMap(question => typeof question.when === 'string' ? [question, applicabilityQuestion(question)] : [question]);
+}
+/** Cited-row extraction is executable question semantics, including for derived questions. */
+export function questionDefinitionDigest(grader, question) {
+    const reads = [...question.reads, ...(typeof question.when === 'object' ? [question.when.field] : [])];
+    const rows = Object.fromEntries(reads.filter(read => read.startsWith('context.') && read.endsWith('@cited'))
+        .map(read => [read, grader.context[read.slice(8, -6)].rows]));
+    const definition = Object.keys(rows).length ? { question, rows, extraction: 'cited-rows.v2' } : question;
+    const policy = sentenceCitationPolicy(grader);
+    return stableDigest(question.level === 'sentence' && policy.patterns.length
+        ? { definition, sentenceCitations: policy } : definition);
+}
+//# sourceMappingURL=questions.js.map

package/dist/training/graders/format/questions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"questions.js","sourceRoot":"","sources":["../../../../src/training/graders/format/questions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAEnD,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9D,CAAC,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;CAC3E,CAAC,CAAC;AACH,MAAM,MAAM,GAAG;IACb,EAAE,EAAE,UAAU;IACd,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IACtE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACjC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC3C,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrF,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/D,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,cAAc,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;CACvC,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACzG,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;AAMzD,MAAM,UAAU,UAAU,CAAC,QAA0B;IACnD,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACrD,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC,OAAQ,CAAC;IACzD,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAM,CAAC;IACjC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAA0B;IACxD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACnC,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9D,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,KAAK,GAAG,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IACtB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,QAA0B,EAAE,OAAgC;IAClG,IAAI,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1D,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAA0C,CAAC;YACxF,OAAO,QAAQ,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,EAAE,IAAI,KAAK,SAAS,CAAC;QACzE,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,IAAI,KAAK,UAAU;QAAE,OAAO,QAAQ,CAAC,KAAK,KAAK,UAAU,CAAC;IAC9D,OAAO,CAAC,WAAW,EAAE,oBAAoB,EAAE,gBAAgB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC;AAC9G,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAA0B,EAAE,OAAgC,EAAE,IAAY;IACzG,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,OAAe,EAAS,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7G,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC5J,IAAI,CAAC,SAAS,EAAE,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5F,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAAE,IAAI,CAAC,OAAO,EAAE,8BAA8B,CAAC,CAAC;IAC7F,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,QAAQ,CAAC,OAAO;QAAE,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;IAC7F,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,QAAQ,CAAC,KAAK;QAAE,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IACvF,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAAE,IAAI,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC;IAChK,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,IAAI,CAAC,UAAU,EAAE,2BAA2B,CAAC,CAAC;IACnH,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM;QACrG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAAE,IAAI,CAAC,UAAU,EAAE,8CAA8C,CAAC,CAAC;IACrH,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACrC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC;YAAE,IAAI,CAAC,SAAS,KAAK,EAAE,EAAE,6BAA6B,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;QAC7G,IAAI,CAAC,YAAY,EAAE,8BAA8B,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAA0B;IAC9D,OAAO;QACL,EAAE,EAAE,GAAG,QAAQ,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAc;QACjF,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;KAC/E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,MAAM,SAAS,GAAuB,CAAC,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAA6C,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7J,OAAO,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AACrI,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,wBAAwB,CAAC,MAAc,EAAE,QAA0B;IACjF,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvG,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SACzG,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IACzG,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC9C,OAAO,YAAY,CAAC,QAAQ,CAAC,KAAK,KAAK,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM;QACzE,CAAC,CAAC,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AAC9D,CAAC"}

package/dist/training/graders/format/row-regex.d.ts

@@ -0,0 +1,2 @@
+/** Enforce the prescribed syntax restrictions; these alone do not bound engine runtime. */
+export declare function rowRegexSafetyIssue(source: string): string | undefined;

package/dist/training/graders/format/row-regex.js

@@ -0,0 +1,56 @@
+/** Enforce the prescribed syntax restrictions; these alone do not bound engine runtime. */
+export function rowRegexSafetyIssue(source) {
+    const groups = [{ quantified: false }];
+    let atomQuantified = false;
+    for (let index = 0; index < source.length; index++) {
+        const token = source[index];
+        if (token === '\\') {
+            const escaped = source[++index];
+            if (/[1-9]/u.test(escaped) || (escaped === 'k' && source[index + 1] === '<')) {
+                return 'row regex subset forbids backreferences';
+            }
+            atomQuantified = false;
+        }
+        else if (token === '[') {
+            while (++index < source.length && source[index] !== ']') {
+                if (source[index] === '\\')
+                    index++;
+            }
+            atomQuantified = false;
+        }
+        else if (token === '(') {
+            if (source[index + 1] === '?') {
+                if (source[index + 2] === ':')
+                    index += 2;
+                else if (source[index + 2] === '<' && !['=', '!'].includes(source[index + 3])) {
+                    index = source.indexOf('>', index + 3);
+                }
+                else
+                    return 'row regex subset forbids lookaround';
+            }
+            groups.push({ quantified: false });
+            atomQuantified = false;
+        }
+        else if (token === ')') {
+            atomQuantified = groups.pop().quantified;
+            groups.at(-1).quantified ||= atomQuantified;
+        }
+        else {
+            const counted = token === '{' ? /^\{\d+(?:,\d*)?\}/u.exec(source.slice(index)) : null;
+            if (token === '*' || token === '+' || token === '?' || counted) {
+                if (atomQuantified)
+                    return 'row regex subset requires star height <= 1 (no nested quantifiers)';
+                groups.at(-1).quantified = true;
+                atomQuantified = true;
+                if (counted)
+                    index += counted[0].length - 1;
+                if (source[index + 1] === '?')
+                    index++; // Lazy suffix is not another repetition.
+            }
+            else
+                atomQuantified = false;
+        }
+    }
+    return undefined;
+}
+//# sourceMappingURL=row-regex.js.map

package/dist/training/graders/format/row-regex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"row-regex.js","sourceRoot":"","sources":["../../../../src/training/graders/format/row-regex.ts"],"names":[],"mappings":"AAEA,2FAA2F;AAC3F,MAAM,UAAU,mBAAmB,CAAC,MAAc;IAChD,MAAM,MAAM,GAAY,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;IAChD,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;QAC7B,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,KAAK,CAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBAC7E,OAAO,yCAAyC,CAAC;YACnD,CAAC;YACD,cAAc,GAAG,KAAK,CAAC;QACzB,CAAC;aAAM,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;YACzB,OAAO,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;gBACxD,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI;oBAAE,KAAK,EAAE,CAAC;YACtC,CAAC;YACD,cAAc,GAAG,KAAK,CAAC;QACzB,CAAC;aAAM,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBAC9B,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG;oBAAE,KAAK,IAAI,CAAC,CAAC;qBACrC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;oBAC/E,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBACzC,CAAC;;oBAAM,OAAO,qCAAqC,CAAC;YACtD,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;YACnC,cAAc,GAAG,KAAK,CAAC;QACzB,CAAC;aAAM,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;YACzB,cAAc,GAAG,MAAM,CAAC,GAAG,EAAG,CAAC,UAAU,CAAC;YAC1C,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,UAAU,KAAK,cAAc,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACtF,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC/D,IAAI,cAAc;oBAAE,OAAO,oEAAoE,CAAC;gBAChG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,UAAU,GAAG,IAAI,CAAC;gBACjC,cAAc,GAAG,IAAI,CAAC;gBACtB,IAAI,OAAO;oBAAE,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC5C,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG;oBAAE,KAAK,EAAE,CAAC,CAAC,yCAAyC;YACnF,CAAC;;gBAAM,cAAc,GAAG,KAAK,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/training/graders/format/rows.d.ts

@@ -0,0 +1,15 @@
+import { z } from 'zod';
+import type { Grader } from './grader.js';
+export declare const rowsSchema: z.ZodObject<{
+    heading: z.ZodString;
+    cite: z.ZodString;
+    leading_citations: z.ZodDefault<z.ZodEnum<{
+        keep: "keep";
+        previous: "previous";
+    }>>;
+}, z.core.$strict>;
+/** Shared by unit construction and identity, including sentence-only reads. */
+export declare function sentenceCitationPolicy(grader: Grader): {
+    version: string;
+    patterns: string[];
+};

package/dist/training/graders/format/rows.js

@@ -0,0 +1,31 @@
+import { z } from 'zod';
+import { rowRegexSafetyIssue } from './row-regex.js';
+/** The empty alternative exposes the engine's capture count without parsing regex syntax. */
+const rowRegex = z.string().max(200).superRefine((source, context) => {
+    if (source.length > 200)
+        return;
+    try {
+        new RegExp(source, 'm');
+        const issue = rowRegexSafetyIssue(source);
+        if (issue) {
+            context.addIssue({ code: 'custom', message: issue });
+            return;
+        }
+        if (new RegExp(`(?:${source})|`, 'm').exec('').length !== 2) {
+            context.addIssue({ code: 'custom', message: 'requires exactly one capture group for the row id' });
+        }
+    }
+    catch {
+        context.addIssue({ code: 'custom', message: 'invalid row regex' });
+    }
+});
+export const rowsSchema = z.strictObject({
+    heading: rowRegex, cite: rowRegex,
+    leading_citations: z.enum(['keep', 'previous']).default('keep'),
+});
+/** Shared by unit construction and identity, including sentence-only reads. */
+export function sentenceCitationPolicy(grader) {
+    const patterns = Object.values(grader.context).flatMap(({ rows }) => rows?.leading_citations === 'previous' ? [rows.cite] : []);
+    return { version: 'leading-citations.previous.v1', patterns };
+}
+//# sourceMappingURL=rows.js.map

package/dist/training/graders/format/rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rows.js","sourceRoot":"","sources":["../../../../src/training/graders/format/rows.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,6FAA6F;AAC7F,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;IACnE,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO;IAChC,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QACD,IAAI,IAAI,MAAM,CAAC,MAAM,MAAM,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,mDAAmD,EAAE,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,YAAY,CAAC;IACvC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ;IACjC,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;CAChE,CAAC,CAAC;AAEH,+EAA+E;AAC/E,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAClE,IAAI,EAAE,iBAAiB,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7D,OAAO,EAAE,OAAO,EAAE,+BAA+B,EAAE,QAAQ,EAAE,CAAC;AAChE,CAAC"}

package/dist/training/graders/format/sentences.d.ts

@@ -0,0 +1,2 @@
+/** One paragraph at a time; never create empty units from citation-only tails. */
+export declare function moveLeadingCitations(sentences: string[], patterns: string[]): string[];

package/dist/training/graders/format/sentences.js

@@ -0,0 +1,30 @@
+/** One paragraph at a time; never create empty units from citation-only tails. */
+export function moveLeadingCitations(sentences, patterns) {
+    const citations = patterns.map(pattern => new RegExp(pattern, 'my'));
+    const result = [];
+    for (const sentence of sentences) {
+        let end = 0;
+        if (result.length) {
+            let start = 0;
+            while (start < sentence.length) {
+                let length = 0;
+                for (const citation of citations) {
+                    citation.lastIndex = start;
+                    length = Math.max(length, citation.exec(sentence)?.[0].length ?? 0);
+                }
+                // Zero-width matches are not tokens and must not prevent termination.
+                if (!length)
+                    break;
+                end = start + length;
+                start = end + /^\s*/u.exec(sentence.slice(end))[0].length;
+            }
+        }
+        if (end)
+            result[result.length - 1] += ` ${sentence.slice(0, end)}`;
+        const remaining = sentence.slice(end).trim();
+        if (remaining)
+            result.push(remaining);
+    }
+    return result;
+}
+//# sourceMappingURL=sentences.js.map

package/dist/training/graders/format/sentences.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentences.js","sourceRoot":"","sources":["../../../../src/training/graders/format/sentences.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,MAAM,UAAU,oBAAoB,CAAC,SAAmB,EAAE,QAAkB;IAC1E,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACrE,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,OAAO,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC;oBAC3B,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;gBACtE,CAAC;gBACD,sEAAsE;gBACtE,IAAI,CAAC,MAAM;oBAAE,MAAM;gBACnB,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;gBACrB,KAAK,GAAG,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC7D,CAAC;QACH,CAAC;QACD,IAAI,GAAG;YAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACnE,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,SAAS;YAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/training/graders/identity-calibration.d.ts

@@ -0,0 +1,4 @@
+import { type Grader } from './format/index.js';
+import { type CalibrationFile } from './identity.js';
+/** Explicit raw probabilities, with zero labeled observations; never a fitted calibration. */
+export declare function createIdentityCalibration(grader: Grader, backendModel: string, promptTemplateVersion: string): CalibrationFile;

package/dist/training/graders/identity-calibration.js

@@ -0,0 +1,15 @@
+import { allQuestions, decisionModels, stableDigest } from './format/index.js';
+import { createCalibrationIdentity, parseCalibration } from './identity.js';
+/** Explicit raw probabilities, with zero labeled observations; never a fitted calibration. */
+export function createIdentityCalibration(grader, backendModel, promptTemplateVersion) {
+    if (grader.calibration !== 'identity')
+        throw new Error('Identity calibration must be explicitly authored');
+    return parseCalibration({ schema: 'spawnfile.training.grader-calibration.v1',
+        identity: createCalibrationIdentity(grader, { backendModel, promptTemplateVersion,
+            golden: stableDigest({ mode: 'identity', labels: null }), seed: '', exclusions: [] }),
+        questions: Object.fromEntries(allQuestions(grader).map(question => [question.id, {
+                model: decisionModels(grader)[0].id, status: 'uncalibrated', failures: ['explicit identity calibration; no labeled evidence'],
+                metrics: { groups: 0, instances: 0, good: 0, bad: 0, baseRate: 0 },
+            }])) });
+}
+//# sourceMappingURL=identity-calibration.js.map

package/dist/training/graders/identity-calibration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"identity-calibration.js","sourceRoot":"","sources":["../../../src/training/graders/identity-calibration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,YAAY,EAAe,MAAM,mBAAmB,CAAC;AAC5F,OAAO,EAAE,yBAAyB,EAAE,gBAAgB,EAAwB,MAAM,eAAe,CAAC;AAElG,8FAA8F;AAC9F,MAAM,UAAU,yBAAyB,CAAC,MAAc,EAAE,YAAoB,EAAE,qBAA6B;IAC3G,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IAC3G,OAAO,gBAAgB,CAAC,EAAE,MAAM,EAAE,0CAA0C;QAC1E,QAAQ,EAAE,yBAAyB,CAAC,MAAM,EAAE,EAAE,YAAY,EAAE,qBAAqB;YAC/E,MAAM,EAAE,YAAY,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QACvF,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAC/E,KAAK,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAE,CAAC,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,oDAAoD,CAAC;gBAC9H,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;aACnE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACZ,CAAC"}

package/dist/training/graders/identity.d.ts

@@ -0,0 +1,119 @@
+import { z } from 'zod';
+import { type Grader, type Labels } from './format/index.js';
+export declare const calibrationIdentitySchema: z.ZodObject<{
+    mode: z.ZodOptional<z.ZodLiteral<"identity">>;
+    questions: z.ZodRecord<z.ZodString, z.ZodString>;
+    expansion: z.ZodArray<z.ZodTuple<[z.ZodString, z.ZodString], null>>;
+    promptTemplateVersion: z.ZodString;
+    segmenter: z.ZodString;
+    optionMapping: z.ZodRecord<z.ZodString, z.ZodString>;
+    backendModel: z.ZodString;
+    models: z.ZodOptional<z.ZodString>;
+    modelIdentities: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    golden: z.ZodString;
+    seed: z.ZodString;
+    escalation: z.ZodString;
+    escalationTeacher: z.ZodString;
+    uncalibrated: z.ZodEnum<{
+        teacher: "teacher";
+        refuse: "refuse";
+    }>;
+    exclusions: z.ZodArray<z.ZodObject<{
+        dataset: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type CalibrationIdentity = z.infer<typeof calibrationIdentitySchema>;
+export declare const calibrationFileSchema: z.ZodObject<{
+    schema: z.ZodPreprocess<z.ZodLiteral<"spawnfile.training.grader-calibration.v1">>;
+    identity: z.ZodObject<{
+        mode: z.ZodOptional<z.ZodLiteral<"identity">>;
+        questions: z.ZodRecord<z.ZodString, z.ZodString>;
+        expansion: z.ZodArray<z.ZodTuple<[z.ZodString, z.ZodString], null>>;
+        promptTemplateVersion: z.ZodString;
+        segmenter: z.ZodString;
+        optionMapping: z.ZodRecord<z.ZodString, z.ZodString>;
+        backendModel: z.ZodString;
+        models: z.ZodOptional<z.ZodString>;
+        modelIdentities: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        golden: z.ZodString;
+        seed: z.ZodString;
+        escalation: z.ZodString;
+        escalationTeacher: z.ZodString;
+        uncalibrated: z.ZodEnum<{
+            teacher: "teacher";
+            refuse: "refuse";
+        }>;
+        exclusions: z.ZodArray<z.ZodObject<{
+            dataset: z.ZodString;
+            digest: z.ZodString;
+        }, z.core.$strict>>;
+    }, z.core.$strict>;
+    questions: z.ZodRecord<z.ZodString, z.ZodObject<{
+        model: z.ZodOptional<z.ZodString>;
+        status: z.ZodEnum<{
+            uncalibrated: "uncalibrated";
+            calibrated: "calibrated";
+            "insufficient-evidence": "insufficient-evidence";
+            "escalation-infeasible": "escalation-infeasible";
+            unverified: "unverified";
+        }>;
+        failures: z.ZodArray<z.ZodString>;
+        params: z.ZodOptional<z.ZodObject<{
+            a: z.ZodNumber;
+            b: z.ZodNumber;
+        }, z.core.$strict>>;
+        band: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+        teacherAccuracy: z.ZodOptional<z.ZodObject<{
+            groups: z.ZodNumber;
+            instances: z.ZodNumber;
+            good: z.ZodNumber;
+            bad: z.ZodNumber;
+            agreement: z.ZodNumber;
+            source: z.ZodEnum<{
+                human: "human";
+                editor: "editor";
+                independent: "independent";
+            }>;
+            failures: z.ZodArray<z.ZodString>;
+            warnings: z.ZodArray<z.ZodString>;
+        }, z.core.$strict>>;
+        metrics: z.ZodObject<{
+            groups: z.ZodNumber;
+            instances: z.ZodNumber;
+            good: z.ZodNumber;
+            bad: z.ZodNumber;
+            baseRate: z.ZodNumber;
+            accuracy: z.ZodOptional<z.ZodNumber>;
+            brier: z.ZodOptional<z.ZodNumber>;
+            baseRateBrier: z.ZodOptional<z.ZodNumber>;
+            ece: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+            eceInterval: z.ZodOptional<z.ZodNullable<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>>;
+            brierInterval: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+            falsePassRate: z.ZodOptional<z.ZodNumber>;
+            falsePassInterval: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+            cascadeInterval: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+            agreement: z.ZodOptional<z.ZodNumber>;
+            cascadeAccuracy: z.ZodOptional<z.ZodNumber>;
+            escalationRate: z.ZodOptional<z.ZodNumber>;
+            coverage: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strict>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type CalibrationFile = z.infer<typeof calibrationFileSchema>;
+export declare function parseCalibration(value: unknown): CalibrationFile;
+export declare function parseCalibrationJson(source: string): CalibrationFile;
+export { segmenterIdentity } from './instances.js';
+export declare function createCalibrationIdentity(grader: Grader, supplied: Omit<CalibrationIdentity, 'questions' | 'expansion' | 'optionMapping' | 'segmenter' | 'escalation' | 'escalationTeacher' | 'uncalibrated' | 'models'> & {
+    segmenter?: string;
+}): CalibrationIdentity;
+export declare function goldenIdentity(labels: Labels): string;
+export interface CalibrationCheck {
+    ok: boolean;
+    refusals: string[];
+    notCalibrated: string[];
+}
+/** Identity mismatches always refuse, including when teacher fallback is selected. */
+export declare function checkCalibration(grader: Grader, calibration: CalibrationFile, identity: CalibrationIdentity, options?: {
+    quality?: boolean;
+}): CalibrationCheck;