spawnfile 0.1.25 → 0.1.28

package/dist/training/graders/format/grader.d.ts

@@ -0,0 +1,236 @@
+import { z } from 'zod';
+declare const modelSchema: z.ZodUnion<readonly [z.ZodObject<{
+    backend: z.ZodLiteral<"jev-http">;
+    endpoint: z.ZodURL;
+    name: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    endpoint: z.ZodURL;
+    name: z.ZodString;
+    backend: z.ZodLiteral<"ollama-logprobs">;
+    context_tokens: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strict>, z.ZodObject<{
+    backend: z.ZodLiteral<"teacher">;
+    teacher: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    backend: z.ZodLiteral<"typesafe">;
+    name: z.ZodString;
+    endpoint: z.ZodDefault<z.ZodURL>;
+    price_per_mtok_usd: z.ZodOptional<z.ZodNumber>;
+    concurrency: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strict>]>;
+declare const namedModelSchema: z.ZodUnion<readonly [z.ZodObject<{
+    backend: z.ZodLiteral<"jev-http">;
+    endpoint: z.ZodURL;
+    name: z.ZodString;
+    id: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    endpoint: z.ZodURL;
+    name: z.ZodString;
+    backend: z.ZodLiteral<"ollama-logprobs">;
+    context_tokens: z.ZodDefault<z.ZodNumber>;
+    id: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    backend: z.ZodLiteral<"teacher">;
+    teacher: z.ZodString;
+    id: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    backend: z.ZodLiteral<"typesafe">;
+    name: z.ZodString;
+    endpoint: z.ZodDefault<z.ZodURL>;
+    price_per_mtok_usd: z.ZodOptional<z.ZodNumber>;
+    concurrency: z.ZodDefault<z.ZodNumber>;
+    id: z.ZodString;
+}, z.core.$strict>]>;
+export type DecisionModel = z.infer<typeof modelSchema>;
+export type NamedDecisionModel = z.infer<typeof namedModelSchema>;
+export declare const graderSchema: z.ZodObject<{
+    schema: z.ZodPreprocess<z.ZodLiteral<"spawnfile.training.grader.v1">>;
+    id: z.ZodString;
+    golden: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+        schema: z.ZodPreprocess<z.ZodLiteral<"spawnfile.training.labels.v1">>;
+        labelers: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
+            kind: z.ZodLiteral<"model">;
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"signal">;
+            name: z.ZodString;
+            source: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"human">;
+            name: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"editor">;
+            name: z.ZodString;
+            source: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>]>>;
+        labels: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            lineage: z.ZodString;
+            example: z.ZodOptional<z.ZodObject<{
+                dataset: z.ZodString;
+                id: z.ZodString;
+            }, z.core.$strict>>;
+            output: z.ZodJSONSchema;
+            context: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+            asked: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            askedByInstance: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>>;
+            votes: z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>>;
+            provenance: z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>>;
+        }, z.core.$strict>>;
+        grader: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>]>;
+    calibration: z.ZodString;
+    context: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
+        description: z.ZodString;
+        rows: z.ZodOptional<z.ZodObject<{
+            heading: z.ZodString;
+            cite: z.ZodString;
+            leading_citations: z.ZodDefault<z.ZodEnum<{
+                keep: "keep";
+                previous: "previous";
+            }>>;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>>;
+    teachers: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        provider: z.ZodString;
+        model: z.ZodString;
+        effort: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>;
+    model: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
+        backend: z.ZodLiteral<"jev-http">;
+        endpoint: z.ZodURL;
+        name: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        endpoint: z.ZodURL;
+        name: z.ZodString;
+        backend: z.ZodLiteral<"ollama-logprobs">;
+        context_tokens: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strict>, z.ZodObject<{
+        backend: z.ZodLiteral<"teacher">;
+        teacher: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        backend: z.ZodLiteral<"typesafe">;
+        name: z.ZodString;
+        endpoint: z.ZodDefault<z.ZodURL>;
+        price_per_mtok_usd: z.ZodOptional<z.ZodNumber>;
+        concurrency: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strict>]>>;
+    max_spend_usd: z.ZodOptional<z.ZodNumber>;
+    models: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+        backend: z.ZodLiteral<"jev-http">;
+        endpoint: z.ZodURL;
+        name: z.ZodString;
+        id: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        endpoint: z.ZodURL;
+        name: z.ZodString;
+        backend: z.ZodLiteral<"ollama-logprobs">;
+        context_tokens: z.ZodDefault<z.ZodNumber>;
+        id: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        backend: z.ZodLiteral<"teacher">;
+        teacher: z.ZodString;
+        id: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        backend: z.ZodLiteral<"typesafe">;
+        name: z.ZodString;
+        endpoint: z.ZodDefault<z.ZodURL>;
+        price_per_mtok_usd: z.ZodOptional<z.ZodNumber>;
+        concurrency: z.ZodDefault<z.ZodNumber>;
+        id: z.ZodString;
+    }, z.core.$strict>]>>>;
+    escalate: z.ZodUnion<readonly [z.ZodObject<{
+        to: z.ZodString;
+        between: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
+        max_calls_per_trial: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strict>, z.ZodObject<{
+        to: z.ZodString;
+        target_accuracy: z.ZodNumber;
+        max_calls_per_trial: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strict>]>;
+    uncalibrated: z.ZodDefault<z.ZodEnum<{
+        teacher: "teacher";
+        refuse: "refuse";
+    }>>;
+    gates: z.ZodDefault<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+        id: z.ZodString;
+        check: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, z.core.$strict>, z.ZodObject<{
+        id: z.ZodString;
+        level: z.ZodDefault<z.ZodEnum<{
+            article: "article";
+            paragraph: "paragraph";
+            sentence: "sentence";
+        }>>;
+        ask: z.ZodOptional<z.ZodString>;
+        type: z.ZodEnum<{
+            score: "score";
+            "yes-no": "yes-no";
+            choice: "choice";
+        }>;
+        expected: z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodArray<z.ZodString>, z.ZodArray<z.ZodNumber>]>;
+        options: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        scale: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+        reads: z.ZodArray<z.ZodString>;
+        when: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
+            field: z.ZodString;
+            equals: z.ZodJSONSchema;
+        }, z.core.$strict>, z.ZodObject<{
+            field: z.ZodString;
+            in: z.ZodArray<z.ZodJSONSchema>;
+        }, z.core.$strict>]>]>>;
+        not_applicable: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strict>]>>>;
+    questions: z.ZodArray<z.ZodObject<{
+        weight: z.ZodDefault<z.ZodNumber>;
+        id: z.ZodString;
+        level: z.ZodDefault<z.ZodEnum<{
+            article: "article";
+            paragraph: "paragraph";
+            sentence: "sentence";
+        }>>;
+        ask: z.ZodOptional<z.ZodString>;
+        type: z.ZodEnum<{
+            score: "score";
+            "yes-no": "yes-no";
+            choice: "choice";
+        }>;
+        expected: z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodArray<z.ZodString>, z.ZodArray<z.ZodNumber>]>;
+        options: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        scale: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+        reads: z.ZodArray<z.ZodString>;
+        when: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
+            field: z.ZodString;
+            equals: z.ZodJSONSchema;
+        }, z.core.$strict>, z.ZodObject<{
+            field: z.ZodString;
+            in: z.ZodArray<z.ZodJSONSchema>;
+        }, z.core.$strict>]>]>>;
+        not_applicable: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strict>>;
+    score: z.ZodObject<{
+        method: z.ZodEnum<{
+            weighted: "weighted";
+            "fitted-weights": "fitted-weights";
+        }>;
+        pass_at: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export type Grader = z.infer<typeof graderSchema>;
+export type Gate = Grader['gates'][number];
+/** Check parameters remain opaque; inspect only logical children for gate binding. */
+export declare function checkBinding(check: Record<string, unknown>, location: string): void;
+export { allQuestions } from './questions.js';
+/** Parsing preserves recognized future methods; executable validation refuses them. */
+export declare function parseGrader(value: unknown): Grader;
+export declare function parseGraderYaml(source: string): Grader;
+export declare function validateGrader(value: unknown): Grader;
+export declare function loadGrader(filename: string): Promise<Grader>;
+/** Legacy model syntax has exactly the same routing semantics as id default. */
+export declare function decisionModels(grader: Grader): NamedDecisionModel[];
+/** A single-model view for transports and independent teacher truth. */
+export declare function graderForModel(grader: Grader, model: NamedDecisionModel): Grader & {
+    model: DecisionModel;
+};

package/dist/training/graders/format/grader.js

@@ -0,0 +1,161 @@
+import { checkTrainingFilename } from '../../platform/index.js';
+import { compatibleWireLiteral } from '../../contracts/index.js';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod';
+import { labelsSchema, parseLabels } from './labels.js';
+import { identifier, probability, provider, parseSchema, parseYaml, rememberSourceDirectory } from './common.js';
+import { questionSchema, gateQuestionSchema, validateQuestion, allQuestions } from './questions.js';
+import { rowsSchema } from './rows.js';
+import { requiresArgumentBinding } from './binding.js';
+const between = z.tuple([probability, probability]).refine(([lo, hi]) => lo < hi, 'requires lo < hi');
+const escalation = z.union([
+    z.strictObject({ to: identifier, between, max_calls_per_trial: z.number().int().positive().optional() }),
+    z.strictObject({ to: identifier, target_accuracy: z.number().gt(0.5).max(1), max_calls_per_trial: z.number().int().positive().optional() }),
+]);
+const localModel = z.strictObject({ backend: z.literal('jev-http'), endpoint: z.url(), name: z.string().min(1) });
+const ollamaModel = localModel.extend({ backend: z.literal('ollama-logprobs'),
+    context_tokens: z.number().int().min(2048).max(131072).default(16384) });
+const teacherModel = z.strictObject({ backend: z.literal('teacher'), teacher: identifier });
+const typeSafeModel = z.strictObject({ backend: z.literal('typesafe'), name: z.string().regex(/^jev-\d+\.\d+\.\d+$/),
+    endpoint: z.url({ protocol: /^https$/ }).default('https://api.typesafe.ai/v1/systemone'),
+    price_per_mtok_usd: z.number().finite().positive().optional(), concurrency: z.number().int().min(1).max(32).default(8) });
+const modelSchema = z.union([localModel, ollamaModel, teacherModel, typeSafeModel]);
+const namedModelSchema = z.union([localModel.extend({ id: identifier }), ollamaModel.extend({ id: identifier }), teacherModel.extend({ id: identifier }), typeSafeModel.extend({ id: identifier })]);
+export const graderSchema = z.strictObject({
+    schema: compatibleWireLiteral('spawnfile.training.grader.v1'), id: identifier,
+    golden: z.union([z.string().min(1), labelsSchema.extend({ grader: identifier.optional() })]), calibration: z.string().min(1),
+    context: z.record(identifier, z.strictObject({ description: z.string(), rows: rowsSchema.optional() })).default({}),
+    teachers: z.array(z.strictObject({ id: identifier, provider, model: z.string().min(1), effort: z.string().min(1).optional() })),
+    model: modelSchema.optional(),
+    max_spend_usd: z.number().finite().nonnegative().optional(),
+    models: z.array(namedModelSchema).min(1).max(4).optional(),
+    escalate: escalation,
+    uncalibrated: z.enum(['refuse', 'teacher']).default('refuse'),
+    gates: z.array(z.union([z.strictObject({ id: identifier, check: z.record(z.string(), z.unknown()) }), gateQuestionSchema])).default([]),
+    questions: z.array(questionSchema).min(1),
+    score: z.strictObject({ method: z.enum(['weighted', 'fitted-weights']), pass_at: probability.default(0) }),
+}).refine(value => (value.model === undefined) !== (value.models === undefined), {
+    path: ['models'], message: 'exactly one of model or models is required',
+});
+/** Check parameters remain opaque; inspect only logical children for gate binding. */
+export function checkBinding(check, location) {
+    if (check.eval === 'tool-called' || check.eval === 'tool-count')
+        throw new Error(`${location}.eval: unbound tool gate`);
+    if (check.eval === 'tool-sequence' && check.ending_at?.event !== 'accepted-call') {
+        throw new Error(`${location}.ending_at.event: requires accepted-call`);
+    }
+    if (check.eval === 'tool-sequence') {
+        const steps = Array.isArray(check.steps) ? check.steps : [];
+        const cel = steps.at(-1)?.where?.cel;
+        const bound = new Set(steps.slice(0, -1).flatMap(step => Object.keys(step?.bind ?? {})));
+        if (!requiresArgumentBinding(cel, bound)) {
+            throw new Error(`${location}.steps: final where.cel must require argument equality to an earlier step binding`);
+        }
+    }
+    for (const op of ['all', 'any', 'not']) {
+        const children = check[op];
+        const items = Array.isArray(children) ? children : [children];
+        items.forEach((child, i) => {
+            if (child && typeof child === 'object')
+                checkBinding(child, `${location}.${op}.${i}`);
+        });
+    }
+}
+export { allQuestions } from './questions.js';
+function validateStructure(grader) {
+    const teachers = new Set();
+    grader.teachers.forEach((teacher, i) => {
+        if (teachers.has(teacher.id))
+            throw new Error(`$.teachers.${i}.id: duplicate teacher`);
+        teachers.add(teacher.id);
+    });
+    if (grader.calibration !== 'identity' && !teachers.size)
+        throw new Error('$.teachers: requires at least one teacher');
+    if (grader.calibration !== 'identity' && !teachers.has(grader.escalate.to))
+        throw new Error('$.escalate.to: unknown teacher');
+    const models = decisionModels(grader);
+    if (grader.calibration === 'identity' && (grader.models !== undefined || models.length !== 1 || models[0].backend === 'teacher' ||
+        grader.teachers.length || grader.escalate.to !== 'none' || grader.uncalibrated !== 'refuse')) {
+        throw new Error('$.calibration: identity requires one decision model, no teachers or escalation, and uncalibrated: refuse');
+    }
+    if (grader.calibration === 'identity' && models[0].backend === 'typesafe' && grader.max_spend_usd === undefined) {
+        throw new Error('$.max_spend_usd: identity TypeSafe grading requires an explicit spend cap');
+    }
+    if (grader.calibration === 'identity' && (typeof grader.golden === 'string' || grader.golden.labels.length ||
+        Object.keys(grader.golden.labelers).length || !('between' in grader.escalate) ||
+        grader.escalate.between[0] !== 0 || grader.escalate.between[1] !== 1 || grader.escalate.max_calls_per_trial !== undefined)) {
+        throw new Error('$.calibration: identity cannot declare golden labels or escalation');
+    }
+    const modelIds = new Set();
+    for (const model of models) {
+        if (modelIds.has(model.id))
+            throw new Error(`$.models.${model.id}: duplicate model id`);
+        modelIds.add(model.id);
+        if (model.backend === 'teacher' && !teachers.has(model.teacher))
+            throw new Error(`$.model.teacher (${model.id}): unknown teacher`);
+    }
+    const ids = new Set();
+    for (const [collection, entries] of [['questions', grader.questions], ['gates', grader.gates]]) {
+        entries.forEach((entry, i) => {
+            const location = `$.${collection}.${i}`;
+            if (ids.has(entry.id) || entry.id.includes('@'))
+                throw new Error(`${location}.id: duplicate or reserved instance ID`);
+            ids.add(entry.id);
+            if ('check' in entry)
+                checkBinding(entry.check, `${location}.check`);
+            else {
+                validateQuestion(entry, grader.context, location);
+                if (collection === 'gates' && entry.level !== 'article')
+                    throw new Error(`${location}.level: gates require article`);
+            }
+        });
+    }
+    for (const question of allQuestions(grader)) {
+        if (typeof question.when === 'string' && ids.has(`${question.id}.applies`))
+            throw new Error(`$.questions.${question.id}.id: derived ID collision`);
+    }
+    const total = grader.questions.reduce((sum, question) => sum + question.weight, 0);
+    if (!Number.isFinite(total) || total <= 0)
+        throw new Error('$.questions.weight: requires finite positive total');
+}
+/** Parsing preserves recognized future methods; executable validation refuses them. */
+export function parseGrader(value) {
+    // Only explicit identity mode supplies the unused legacy authoring fields.
+    if (value && typeof value === 'object' && 'calibration' in value && value.calibration === 'identity') {
+        value = { golden: { schema: 'spawnfile.training.labels.v1', labelers: {}, labels: [] },
+            teachers: [], escalate: { to: 'none', between: [0, 1] }, ...value };
+    }
+    const grader = parseSchema(graderSchema, value);
+    validateStructure(grader);
+    if (typeof grader.golden !== 'string')
+        parseLabels({ ...grader.golden, grader: grader.golden.grader ?? grader.id }, grader);
+    return grader;
+}
+export function parseGraderYaml(source) { return parseGrader(parseYaml(source)); }
+export function validateGrader(value) {
+    const grader = parseGrader(value);
+    if (grader.score.method === 'fitted-weights')
+        throw new Error('$.score.method: fitted-weights not implemented in v1');
+    return grader;
+}
+export async function loadGrader(filename) {
+    checkTrainingFilename(filename);
+    const grader = validateGrader(parseGraderYaml(await readFile(filename, 'utf8')));
+    const directory = path.dirname(path.resolve(filename));
+    const loaded = { ...grader, golden: typeof grader.golden === 'string' ? path.resolve(directory, grader.golden) : grader.golden,
+        calibration: grader.calibration === 'identity' ? 'identity' : path.resolve(directory, grader.calibration) };
+    rememberSourceDirectory(loaded, directory);
+    return loaded;
+}
+/** Legacy model syntax has exactly the same routing semantics as id default. */
+export function decisionModels(grader) {
+    return grader.models ?? [{ ...grader.model, id: 'default' }];
+}
+/** A single-model view for transports and independent teacher truth. */
+export function graderForModel(grader, model) {
+    const { models: _models, ...rest } = grader;
+    const { id: _id, ...definition } = model;
+    return { ...rest, model: definition };
+}
+//# sourceMappingURL=grader.js.map

package/dist/training/graders/format/grader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grader.js","sourceRoot":"","sources":["../../../../src/training/graders/format/grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACjH,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACpG,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAEvD,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,kBAAkB,CAAC,CAAC;AACtG,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC;IACzB,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;IACxG,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CAC5I,CAAC,CAAC;AACH,MAAM,UAAU,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAClH,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAC3E,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAC3E,MAAM,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;AAC5F,MAAM,aAAa,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC;IAClH,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,sCAAsC,CAAC;IACxF,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC5H,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC;AACpF,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;AAGrM,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;IACzC,MAAM,EAAE,qBAAqB,CAAC,8BAA8B,CAAC,EAAE,EAAE,EAAE,UAAU;IAC7E,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5H,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACnH,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC/H,KAAK,EAAE,WAAW,CAAC,QAAQ,EAAE;IAC7B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC3D,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC1D,QAAQ,EAAE,UAAU;IACpB,YAAY,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC7D,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACvI,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzC,KAAK,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;CAC3G,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,EAAE;IAC/E,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,4CAA4C;CACxE,CAAC,CAAC;AAIH,sFAAsF;AACtF,MAAM,UAAU,YAAY,CAAC,KAA8B,EAAE,QAAgB;IAC3E,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,0BAA0B,CAAC,CAAC;IACxH,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,IAAK,KAAK,CAAC,SAA6C,EAAE,KAAK,KAAK,eAAe,EAAE,CAAC;QACtH,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,0CAA0C,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAwE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/H,MAAM,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,mFAAmF,CAAC,CAAC;QAClH,CAAC;IACH,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC9D,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACzB,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,YAAY,CAAC,KAAgC,EAAE,GAAG,QAAQ,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QACnH,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,SAAS,iBAAiB,CAAC,MAAc;IACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;QACrC,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,wBAAwB,CAAC,CAAC;QACvF,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU,IAAI,CAAC,QAAQ,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IACtH,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC9H,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,KAAK,SAAS;QAC9H,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,MAAM,IAAI,MAAM,CAAC,YAAY,KAAK,QAAQ,CAAC,EAAE,CAAC;QAC/F,MAAM,IAAI,KAAK,CAAC,0GAA0G,CAAC,CAAC;IAC9H,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,KAAK,UAAU,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACjH,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU,IAAI,CAAC,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM;QACxG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC;QAC7E,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,mBAAmB,KAAK,SAAS,CAAC,EAAE,CAAC;QAC7H,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACxF,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACrI,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAU,EAAE,CAAC;QACxG,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAC3B,MAAM,QAAQ,GAAG,KAAK,UAAU,IAAI,CAAC,EAAE,CAAC;YACxC,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,wCAAwC,CAAC,CAAC;YACtH,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,IAAI,OAAO,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,QAAQ,QAAQ,CAAC,CAAC;iBAChE,CAAC;gBACJ,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAClD,IAAI,UAAU,KAAK,OAAO,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;oBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,+BAA+B,CAAC,CAAC;YACvH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,EAAE,UAAU,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACrJ,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACnH,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,2EAA2E;IAC3E,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,aAAa,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;QACrG,KAAK,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,8BAA8B,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YACpF,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC;IACxE,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAChD,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC1B,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,WAAW,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;IAC5H,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,MAAM,UAAU,eAAe,CAAC,MAAc,IAAY,OAAO,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAClG,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,gBAAgB;QAAE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IACtH,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,cAAc,CAAC,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACjF,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;QAC5H,WAAW,EAAE,MAAM,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;IAC9G,uBAAuB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,OAAO,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAM,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;AAChE,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,KAAyB;IACtE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IAC5C,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,GAAG,KAAK,CAAC;IACzC,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;AACxC,CAAC"}

package/dist/training/graders/format/index.d.ts

@@ -0,0 +1,11 @@
+export { checkBinding, decisionModels, graderForModel, graderSchema, parseGrader, parseGraderYaml, validateGrader, loadGrader, allQuestions } from './grader.js';
+export type { Grader, DecisionModel, NamedDecisionModel, Gate } from './grader.js';
+export { optionsFor, expectedOptions, questionDefinitionDigest, applicabilityQuestion } from './questions.js';
+export type { Question, GateQuestion, DecisionQuestion, Option } from './questions.js';
+export { labelsSchema, parseLabels, parseLabelsYaml, loadLabels } from './labels.js';
+export type { Labels, Label, Labeler } from './labels.js';
+export { stableDigest, sha256, parseYaml, parseSchema, identifier, probability, provider } from './common.js';
+export { parseLabelImport } from './label-import.js';
+export type { LabelImport } from './label-import.js';
+export { rowsSchema, sentenceCitationPolicy } from './rows.js';
+export { moveLeadingCitations } from './sentences.js';

package/dist/training/graders/format/index.js

@@ -0,0 +1,8 @@
+export { checkBinding, decisionModels, graderForModel, graderSchema, parseGrader, parseGraderYaml, validateGrader, loadGrader, allQuestions } from './grader.js';
+export { optionsFor, expectedOptions, questionDefinitionDigest, applicabilityQuestion } from './questions.js';
+export { labelsSchema, parseLabels, parseLabelsYaml, loadLabels } from './labels.js';
+export { stableDigest, sha256, parseYaml, parseSchema, identifier, probability, provider } from './common.js';
+export { parseLabelImport } from './label-import.js';
+export { rowsSchema, sentenceCitationPolicy } from './rows.js';
+export { moveLeadingCitations } from './sentences.js';
+//# sourceMappingURL=index.js.map

package/dist/training/graders/format/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/training/graders/format/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,cAAc,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEjK,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAE9G,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAErF,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC9G,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAE/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/training/graders/format/label-import.d.ts

@@ -0,0 +1,42 @@
+import { z } from 'zod';
+import type { Grader } from './grader.js';
+declare const importSchema: z.ZodObject<{
+    schema: z.ZodPreprocess<z.ZodLiteral<"spawnfile.training.label-import.v1">>;
+    grader: z.ZodString;
+    labelers: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
+        kind: z.ZodLiteral<"model">;
+        provider: z.ZodString;
+        model: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"signal">;
+        name: z.ZodString;
+        source: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"human">;
+        name: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"editor">;
+        name: z.ZodString;
+        source: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>]>>;
+    items: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        lineage: z.ZodString;
+        example: z.ZodOptional<z.ZodObject<{
+            dataset: z.ZodString;
+            id: z.ZodString;
+        }, z.core.$strict>>;
+        output: z.ZodObject<{
+            from: z.ZodString;
+        }, z.core.$strict>;
+        context: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
+            from: z.ZodString;
+        }, z.core.$strict>>>;
+        labeler: z.ZodString;
+        votes: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>;
+        note: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type LabelImport = z.infer<typeof importSchema>;
+export declare function parseLabelImport(source: string, grader: Grader): LabelImport;
+export {};

package/dist/training/graders/format/label-import.js

@@ -0,0 +1,44 @@
+import { compatibleWireLiteral } from '../../contracts/index.js';
+import { z } from 'zod';
+import { identifier, parseSchema, parseYaml } from './common.js';
+import { labelsSchema } from './labels.js';
+const reference = z.strictObject({ from: z.string().min(1) });
+const importSchema = z.strictObject({
+    schema: compatibleWireLiteral('spawnfile.training.label-import.v1'), grader: identifier,
+    labelers: labelsSchema.shape.labelers,
+    items: z.array(z.strictObject({
+        id: identifier, lineage: identifier,
+        example: z.strictObject({ dataset: identifier, id: identifier }).optional(),
+        output: reference, context: z.record(identifier, reference).default({}),
+        labeler: identifier, votes: z.record(identifier, z.union([z.string(), z.number().int()])),
+        note: z.string().optional(),
+    })),
+});
+export function parseLabelImport(source, grader) {
+    const manifest = parseSchema(importSchema, parseYaml(source));
+    if (manifest.grader !== grader.id)
+        throw new Error('Import grader does not match grader.id');
+    for (const [id, labeler] of Object.entries(manifest.labelers)) {
+        if (grader.teachers.some(teacher => teacher.id === id))
+            throw new Error(`Import labeler ${id} collides with configured teacher`);
+        if (labeler.kind !== 'human' && labeler.kind !== 'editor' && labeler.kind !== 'signal')
+            throw new Error(`Import labeler ${id}: only human/editor/signal kinds are allowed`);
+    }
+    const seen = new Set();
+    for (const item of manifest.items) {
+        if (!Object.hasOwn(manifest.labelers, item.labeler))
+            throw new Error(`Label ${item.id}: undeclared labeler ${item.labeler}`);
+        for (const key of Object.keys(item.votes)) {
+            const coordinate = JSON.stringify([item.id, item.labeler, key]);
+            if (seen.has(coordinate))
+                throw new Error(`Label ${item.id}: duplicate import vote ${key}/${item.labeler}`);
+            seen.add(coordinate);
+        }
+        for (const name of Object.keys(item.context)) {
+            if (!Object.hasOwn(grader.context, name))
+                throw new Error(`Label ${item.id}: undeclared context ${name}`);
+        }
+    }
+    return manifest;
+}
+//# sourceMappingURL=label-import.js.map

package/dist/training/graders/format/label-import.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"label-import.js","sourceRoot":"","sources":["../../../../src/training/graders/format/label-import.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,MAAM,SAAS,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC9D,MAAM,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;IAClC,MAAM,EAAE,qBAAqB,CAAC,oCAAoC,CAAC,EAAE,MAAM,EAAE,UAAU;IACvF,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ;IACrC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;QAC5B,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU;QACnC,OAAO,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;QACvE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACzF,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC5B,CAAC,CAAC;CACJ,CAAC,CAAC;AAGH,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,MAAc;IAC7D,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IAC9D,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7F,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,mCAAmC,CAAC,CAAC;QACjI,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,8CAA8C,CAAC,CAAC;IAC9K,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,EAAE,wBAAwB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7H,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,EAAE,2BAA2B,GAAG,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5G,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvB,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,EAAE,wBAAwB,IAAI,EAAE,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/training/graders/format/labels.d.ts

@@ -0,0 +1,45 @@
+import { z } from 'zod';
+import type { Grader } from './grader.js';
+export declare const labelsSchema: z.ZodObject<{
+    schema: z.ZodPreprocess<z.ZodLiteral<"spawnfile.training.labels.v1">>;
+    grader: z.ZodString;
+    labelers: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
+        kind: z.ZodLiteral<"model">;
+        provider: z.ZodString;
+        model: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"signal">;
+        name: z.ZodString;
+        source: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"human">;
+        name: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"editor">;
+        name: z.ZodString;
+        source: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>]>>;
+    labels: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        lineage: z.ZodString;
+        example: z.ZodOptional<z.ZodObject<{
+            dataset: z.ZodString;
+            id: z.ZodString;
+        }, z.core.$strict>>;
+        output: z.ZodJSONSchema;
+        context: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+        asked: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        askedByInstance: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>>;
+        votes: z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>>;
+        provenance: z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type Labels = z.infer<typeof labelsSchema>;
+export type Label = Labels['labels'][number];
+export type Labeler = Labels['labelers'][string];
+export declare function parseLabels(value: unknown, grader: Grader): Labels;
+export declare function parseLabelsYaml(source: string, grader: Grader): Labels;
+export declare function loadLabels(source: Grader['golden'], grader: Grader): Promise<{
+    labels: Labels;
+    goldenDigest: string;
+}>;