spawnfile 0.1.24 → 0.1.27

package/dist/training/cli/grader/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,6BAA6B,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAGlF,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAkB,EAAE,EAAS,EAAE,eAAmC,EAAE,EAAE,MAAoB;IAC/H,IAAI,CAAC;QAAC,OAAO,MAAM,oBAAoB,CAAC,IAAI,EAAE,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAAC,CAAC;IAC1E,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,KAAK,YAAY,uBAAuB;YAAE,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;gBACxE,MAAM,EAAE,sCAAsC,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa;gBAC7F,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;aAC7C,CAAC,IAAI,CAAC,CAAC;QACR,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,IAAkB,EAAE,EAAS,EAAE,YAAgC,EAAE,MAAoB;IACvH,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC;QAAE,OAAO,gBAAgB,CAAC,IAAI,EAAE,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IACpI,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACjG,MAAM,UAAU,GAAG,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IACxD,IAAI,IAAI,CAAC,aAAa,KAAK,UAAU,EAAE,CAAC;QACtC,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,WAAW,CAAC,UAAU,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;QACtF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,IAAI,CAAC,aAAa,KAAK,OAAO,IAAI,IAAI,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtE,MAAM,OAAO,GAAG,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACzF,YAAY,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;YACtI,MAAM,EAAE,CAAC;QACX,IAAI,IAAI,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;YACpC,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3D,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACrD,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,IAAI,CAAC,aAAa,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAChG,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1F,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;YAC3G,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;YAChG,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC;QACjH,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzC,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;YAAE,EAAE,CAAC,MAAM,CAAC,WAAW,OAAO,aAAa,IAAI,IAAI,CAAC,CAAC;QACjH,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW;QACzD,eAAe,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACzF,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC9G,WAAW,EAAE,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC;IACvC,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,eAAe,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,EAAE,CAAC,IAAI,CAAC,CAAC;IAC/K,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}

package/dist/training/cli/grader/label-call.d.ts

@@ -0,0 +1,13 @@
+import type { InferenceProvider, InferenceFailureKind } from '../../contracts/execution.js';
+import { type Grader, type Label } from '../../graders/index.js';
+import { type LabelGroup } from '../../graders/labeling/index.js';
+type Teacher = Grader['teachers'][number];
+export declare function createGraderTeacher(teacher: Teacher, timeoutMs?: number): InferenceProvider;
+export declare function labelCall(label: Label, group: LabelGroup, teacher: Teacher, provider: InferenceProvider, signal?: AbortSignal): Promise<{
+    calls: number;
+    votes: number;
+    missing: number;
+    failures: Partial<Record<InferenceFailureKind, number>>;
+    stopped?: InferenceFailureKind;
+}>;
+export {};

package/dist/training/cli/grader/label-call.js

@@ -0,0 +1,67 @@
+import { graderProvider } from '../../contracts/index.js';
+import { performance } from 'node:perf_hooks';
+import { InferenceFailure } from '../../inference/index.js';
+import { stableDigest } from '../../graders/index.js';
+import { buildLabelPrompt, decodeLabelVotes, teacherPolicyDigest } from '../../graders/labeling/index.js';
+export function createGraderTeacher(teacher, timeoutMs) {
+    return graderProvider(teacher.provider).create(teacher, timeoutMs);
+}
+export async function labelCall(label, group, teacher, provider, signal) {
+    if (provider.capabilities.billing !== 'subscription' || !['disabled', 'contained'].includes(provider.capabilities.tools))
+        throw new Error('Teacher requires a tool-isolated subscription provider');
+    const request = buildLabelPrompt(group);
+    const attempts = [];
+    const failures = {};
+    let refused = false;
+    let stopped;
+    let votes = null;
+    for (let attempt = 0; attempt < 2; attempt++) {
+        signal?.throwIfAborted();
+        const at = new Date().toISOString();
+        const start = performance.now();
+        let result;
+        let invalid;
+        let failure;
+        const prompt = attempt === 0 ? request.prompt : `${request.prompt}\nThe previous response failed the JSON schema. Return exactly the required object with every key, allowed string answers, and no extra keys.`;
+        try {
+            result = await provider.generate({ model: teacher.model, prompt, system: request.system, responseSchema: request.schema, signal });
+            votes = decodeLabelVotes(result.text, group);
+            invalid = votes === null;
+        }
+        catch (error) {
+            signal?.throwIfAborted();
+            failure = error instanceof InferenceFailure ? error : undefined;
+            refused = failure?.detail === 'tool-call';
+            invalid = failure?.kind === 'invalid-response';
+        }
+        const failureKind = votes === null ? failure?.kind ?? (invalid ? 'invalid-response' : 'unknown') : null;
+        if (failureKind)
+            failures[failureKind] = (failures[failureKind] ?? 0) + 1;
+        if (failureKind && ['authentication', 'auth-stale', 'quota'].includes(failureKind))
+            stopped = failureKind;
+        attempts.push({ failure_kind: failureKind, provider: teacher.provider, requested_model: teacher.model,
+            reported_model: result?.modelBasis === 'requested-assumption' ? null : result?.model ?? failure?.responseModel ?? null,
+            cli: null, prompt_digest: stableDigest({ system: request.system, prompt, schema: request.schema }),
+            at, usage: result?.usage ?? failure?.usage ?? null, wall_ms: performance.now() - start });
+        if (votes !== null || !invalid)
+            break;
+    }
+    const provenance = label.provenance[teacher.id] ??= {};
+    const entries = (provenance.votes ??= {});
+    const calls = (provenance.calls ??= {});
+    const receipt = JSON.parse(JSON.stringify({ teacher_policy_digest: teacherPolicyDigest(teacher),
+        projection_digest: group.projectionDigest, instance_keys: group.instances.map(({ instance }) => instance.key),
+        prompt_digest: request.digest, status: votes ? 'ok' : refused ? 'refused' : 'missing', ...(refused ? { reason: 'tool-call' } : {}), attempts }));
+    const call = stableDigest(receipt);
+    calls[call] = receipt;
+    for (const { instance } of group.instances) {
+        const answers = label.votes[instance.key] ??= {};
+        if (votes)
+            answers[teacher.id] = votes[instance.key];
+        else
+            delete answers[teacher.id];
+        entries[instance.key] = { question_digest: instance.definitionDigest, call };
+    }
+    return { calls: attempts.length, votes: votes ? group.instances.length : 0, missing: votes ? 0 : group.instances.length, failures, ...(stopped ? { stopped } : {}) };
+}
+//# sourceMappingURL=label-call.js.map

package/dist/training/cli/grader/label-call.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"label-call.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/label-call.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAA2B,MAAM,wBAAwB,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,mBAAmB,EAAmB,MAAM,iCAAiC,CAAC;AAG3H,MAAM,UAAU,mBAAmB,CAAC,OAAgB,EAAE,SAAkB;IACtE,OAAO,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAY,EAAE,KAAiB,EAAE,OAAgB,EAAE,QAA2B,EAAE,MAAoB;IAClI,IAAI,QAAQ,CAAC,YAAY,CAAC,OAAO,KAAK,cAAc,IAAI,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IACpM,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,MAAM,QAAQ,GAAkD,EAAE,CAAC;IACnE,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,OAAyC,CAAC;IAC9C,IAAI,KAAK,GAAwC,IAAI,CAAC;IACtD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;QAC7C,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,IAAI,MAAmC,CAAC;QACxC,IAAI,OAAgB,CAAC;QACrB,IAAI,OAAqC,CAAC;QAC1C,MAAM,MAAM,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,+IAA+I,CAAC;QACjN,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YACnI,KAAK,GAAG,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7C,OAAO,GAAG,KAAK,KAAK,IAAI,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,OAAO,GAAG,KAAK,YAAY,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,OAAO,GAAG,OAAO,EAAE,MAAM,KAAK,WAAW,CAAC;YAC1C,OAAO,GAAG,OAAO,EAAE,IAAI,KAAK,kBAAkB,CAAC;QACjD,CAAC;QACD,MAAM,WAAW,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxG,IAAI,WAAW;YAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1E,IAAI,WAAW,IAAI,CAAC,gBAAgB,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,OAAO,GAAG,WAAW,CAAC;QAC1G,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,eAAe,EAAE,OAAO,CAAC,KAAK;YACnG,cAAc,EAAE,MAAM,EAAE,UAAU,KAAK,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,IAAI,OAAO,EAAE,aAAa,IAAI,IAAI;YACtH,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAClG,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;QAC5F,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM;IACxC,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IACvD,MAAM,OAAO,GAAG,CAAC,UAAU,CAAC,KAAK,KAAK,EAAE,CAAgD,CAAC;IACzF,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,KAAK,KAAK,EAAE,CAAgD,CAAC;IACvF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,qBAAqB,EAAE,mBAAmB,CAAC,OAAO,CAAC;QAC7F,iBAAiB,EAAE,KAAK,CAAC,gBAAgB,EAAE,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;QAC7G,aAAa,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAgC,CAAC;IAClL,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;IACtB,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACjD,IAAI,KAAK;YAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAE,CAAC;;YACjD,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,eAAe,EAAE,QAAQ,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAC;IAC/E,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACvK,CAAC"}

package/dist/training/cli/grader/label-import.d.ts

@@ -0,0 +1,16 @@
+export interface ImportLabelOptions {
+    graderFile: string;
+    importFile: string;
+    relabel?: boolean;
+    signal?: AbortSignal;
+}
+export interface ImportLabelSummary {
+    labelsFile: string;
+    items: number;
+    labelsAdded: number;
+    labelsMerged: number;
+    votesByLabeler: Record<string, number>;
+    votesByQuestion: Record<string, number>;
+}
+/** Validate and merge the entire transaction before replacing the checkpoint. No inference dependencies. */
+export declare function runLabelImport(options: ImportLabelOptions): Promise<ImportLabelSummary>;

package/dist/training/cli/grader/label-import.js

@@ -0,0 +1,84 @@
+import { randomUUID } from 'node:crypto';
+import { readFile, writeFile, mkdir, rename, rm } from 'node:fs/promises';
+import path from 'node:path';
+import { stringify } from 'yaml';
+import { loadGrader, parseLabelsYaml, parseLabels, labelsSchema, sha256, stableDigest } from '../../graders/index.js';
+import { parseSchema, parseYaml, parseLabelImport } from '../../graders/format/index.js';
+import { resolveStoredLabels } from './label-inputs.js';
+import { mergeImportedLabel } from '../../graders/labeling/index.js';
+/** Validate and merge the entire transaction before replacing the checkpoint. No inference dependencies. */
+export async function runLabelImport(options) {
+    options.signal?.throwIfAborted();
+    const grader = await loadGrader(options.graderFile);
+    if (typeof grader.golden !== 'string')
+        throw new Error('grader label requires a separate golden labels file; inline golden labels are read-only');
+    const labelsFile = grader.golden;
+    const bytes = await readFile(options.importFile, { signal: options.signal });
+    const manifest = parseLabelImport(bytes.toString('utf8'), grader);
+    let labels;
+    let original;
+    try {
+        original = await readFile(labelsFile, 'utf8');
+        labels = options.relabel ? parseSchema(labelsSchema, parseYaml(original)) : parseLabelsYaml(original, grader);
+        if (labels.grader !== grader.id)
+            throw new Error('Labels grader mismatch');
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT')
+            throw error;
+        labels = { schema: 'spawnfile.training.labels.v1', grader: grader.id, labelers: {}, labels: [] };
+    }
+    for (const [id, labeler] of Object.entries(manifest.labelers)) {
+        const previous = labels.labelers[id];
+        if (previous && stableDigest(previous) !== stableDigest(labeler))
+            throw new Error(`Conflicting labeler redefinition: ${id}`);
+        labels.labelers[id] = labeler;
+    }
+    const receipt = { import_digest: sha256(bytes), imported_at: new Date().toISOString() };
+    const added = new Set(), merged = new Set();
+    const summary = { labelsFile, items: manifest.items.length, labelsAdded: 0, labelsMerged: 0,
+        votesByLabeler: {}, votesByQuestion: {} };
+    async function load(from) {
+        const absolute = path.resolve(path.dirname(options.importFile), from);
+        const bytes = await readFile(absolute, { signal: options.signal });
+        const value = from.endsWith('.json') ? JSON.parse(bytes.toString('utf8')) : bytes.toString('utf8');
+        return { reference: { from: path.relative(path.dirname(labelsFile), absolute), sha256: sha256(bytes) }, value };
+    }
+    for (const item of manifest.items) {
+        options.signal?.throwIfAborted();
+        const output = await load(item.output.from);
+        const context = await Promise.all(Object.entries(item.context).map(async ([name, ref]) => [name, await load(ref.from)]));
+        const incoming = { id: item.id, lineage: item.lineage, ...(item.example ? { example: item.example } : {}),
+            output: output.reference, context: Object.fromEntries(context.map(([name, value]) => [name, value.reference])), votes: {}, provenance: {} };
+        const isNew = mergeImportedLabel(grader, labels, item, incoming, { output: output.value, context: Object.fromEntries(context.map(([name, value]) => [name, value.value])) }, receipt, options.relabel);
+        if (isNew)
+            added.add(item.id);
+        else if (!added.has(item.id))
+            merged.add(item.id);
+        for (const key of Object.keys(item.votes)) {
+            summary.votesByLabeler[item.labeler] = (summary.votesByLabeler[item.labeler] ?? 0) + 1;
+            const question = key.split('@')[0];
+            summary.votesByQuestion[question] = (summary.votesByQuestion[question] ?? 0) + 1;
+        }
+    }
+    parseLabels(labels, grader);
+    // Verify saved references too, including historical labels absent from this import.
+    await resolveStoredLabels(labels, labelsFile);
+    summary.labelsAdded = added.size;
+    summary.labelsMerged = merged.size;
+    const snapshot = stringify(labels);
+    if (snapshot === original)
+        return summary;
+    await mkdir(path.dirname(labelsFile), { recursive: true });
+    const temporary = `${labelsFile}.${randomUUID()}.tmp`;
+    try {
+        await writeFile(temporary, snapshot, { mode: 0o600, flag: 'wx', signal: options.signal });
+        options.signal?.throwIfAborted();
+        await rename(temporary, labelsFile);
+    }
+    finally {
+        await rm(temporary, { force: true });
+    }
+    return summary;
+}
+//# sourceMappingURL=label-import.js.map

package/dist/training/cli/grader/label-import.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"label-import.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/label-import.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAA2B,MAAM,wBAAwB,CAAC;AAC/I,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACzF,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAQrE,4GAA4G;AAC5G,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA2B;IAC9D,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yFAAyF,CAAC,CAAC;IAClJ,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IAClE,IAAI,MAAc,CAAC;IACnB,IAAI,QAA4B,CAAC;IACjC,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC9G,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;QACpE,MAAM,GAAG,EAAE,MAAM,EAAE,8BAA8B,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACnG,CAAC;IACD,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC,KAAK,YAAY,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,EAAE,EAAE,CAAC,CAAC;QAC7H,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;IAChC,CAAC;IACD,MAAM,OAAO,GAAG,EAAE,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACxF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,EAAE,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5D,MAAM,OAAO,GAAuB,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC;QAC7G,cAAc,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IAC5C,KAAK,UAAU,IAAI,CAAC,IAAY;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAoB,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtH,OAAO,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;IAClH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAU,CAAC,CAAC,CAAC;QAClI,MAAM,QAAQ,GAAU,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9G,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QAC9I,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAC7D,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACxI,IAAI,KAAK;YAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1C,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACvF,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;YACpC,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IACD,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,oFAAoF;IACpF,MAAM,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,OAAO,CAAC,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;IAAC,OAAO,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC;IACrE,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC1C,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,SAAS,GAAG,GAAG,UAAU,IAAI,UAAU,EAAE,MAAM,CAAC;IACtD,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACtC,CAAC;YAAS,CAAC;QAAC,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IACnD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/training/cli/grader/label-inputs.d.ts

@@ -0,0 +1,9 @@
+import { type Grader, type Label, type Labels } from '../../graders/index.js';
+export interface LabelInput {
+    label: Label;
+    output: Label['output'];
+    context: Label['context'];
+}
+export declare function loadLabelInputs(filename: string, grader: Grader, limit?: number): Promise<LabelInput[]>;
+/** Verify all saved references, including outputs absent from the current manifest. */
+export declare function resolveStoredLabels(labels: Labels, filename: string): Promise<Labels>;

package/dist/training/cli/grader/label-inputs.js

@@ -0,0 +1,56 @@
+import { compatibleWireLiteral } from '../../contracts/index.js';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod';
+import { identifier, parseSchema, parseYaml } from '../../graders/format/index.js';
+import { sha256 } from '../../graders/index.js';
+const reference = z.strictObject({ from: z.string().min(1) });
+const manifestSchema = z.strictObject({ schema: compatibleWireLiteral('spawnfile.training.grader-inputs.v1'), items: z.array(z.strictObject({
+        id: identifier, lineage: identifier, example: z.strictObject({ dataset: identifier, id: identifier }).optional(),
+        output: reference, context: z.record(identifier, reference).default({}),
+    })) });
+export async function loadLabelInputs(filename, grader, limit) {
+    if (typeof grader.golden !== 'string')
+        throw new Error('grader label requires a separate golden labels file; inline golden labels are read-only');
+    const labelsDirectory = path.dirname(grader.golden);
+    const manifest = parseSchema(manifestSchema, parseYaml(await readFile(filename, 'utf8')));
+    const ids = new Set();
+    for (const item of manifest.items) {
+        if (ids.has(item.id))
+            throw new Error(`Duplicate input id ${item.id}`);
+        ids.add(item.id);
+        for (const name of Object.keys(item.context))
+            if (!Object.hasOwn(grader.context, name))
+                throw new Error(`Undeclared context ${name}`);
+    }
+    async function load(from) {
+        const absolute = path.resolve(path.dirname(filename), from);
+        const bytes = await readFile(absolute);
+        const value = from.endsWith('.json') ? JSON.parse(bytes.toString('utf8')) : bytes.toString('utf8');
+        return { reference: { from: path.relative(labelsDirectory, absolute), sha256: sha256(bytes) }, value };
+    }
+    return Promise.all(manifest.items.slice(0, limit).map(async (item) => {
+        const output = await load(item.output.from);
+        const entries = await Promise.all(Object.entries(item.context).map(async ([name, ref]) => [name, await load(ref.from)]));
+        return { output: output.value, context: Object.fromEntries(entries.map(([name, loaded]) => [name, loaded.value])),
+            label: { id: item.id, lineage: item.lineage, ...(item.example ? { example: item.example } : {}), output: output.reference,
+                context: Object.fromEntries(entries.map(([name, loaded]) => [name, loaded.reference])), votes: {}, provenance: {}, asked: [] } };
+    }));
+}
+/** Verify all saved references, including outputs absent from the current manifest. */
+export async function resolveStoredLabels(labels, filename) {
+    async function resolve(value) {
+        if (value === null || typeof value !== 'object' || Array.isArray(value) || !('from' in value))
+            return value;
+        const ref = parseSchema(z.strictObject({ from: z.string().min(1), sha256: z.string() }), value);
+        const bytes = await readFile(path.resolve(path.dirname(filename), ref.from));
+        if (sha256(bytes) !== ref.sha256)
+            throw new Error(`Stored label sha256 mismatch for ${ref.from}; use --relabel with that input`);
+        return ref.from.endsWith('.json') ? JSON.parse(bytes.toString('utf8')) : bytes.toString('utf8');
+    }
+    return { ...labels, labels: await Promise.all(labels.labels.map(async (label) => ({ ...label,
+            output: await resolve(label.output),
+            context: Object.fromEntries(await Promise.all(Object.entries(label.context).map(async ([name, value]) => [name, await resolve(value)]))),
+        }))) };
+}
+//# sourceMappingURL=label-inputs.js.map

package/dist/training/cli/grader/label-inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"label-inputs.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/label-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AACnF,OAAO,EAAE,MAAM,EAAwC,MAAM,wBAAwB,CAAC;AAEtF,MAAM,SAAS,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC9D,MAAM,cAAc,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,qBAAqB,CAAC,qCAAqC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;QAC1I,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE;QAChH,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;KACxE,CAAC,CAAC,EAAE,CAAC,CAAC;AAGP,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB,EAAE,MAAc,EAAE,KAAc;IACpF,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yFAAyF,CAAC,CAAC;IAClJ,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,EAAE,SAAS,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACvE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAC;IACxI,CAAC;IACD,KAAK,UAAU,IAAI,CAAC,IAAY;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAoB,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtH,OAAO,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;IACzG,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,EAAC,IAAI,EAAC,EAAE;QACjE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAU,CAAC,CAAC,CAAC;QAClI,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/G,KAAK,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS;gBACvH,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;IACvI,CAAC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,QAAgB;IACxE,KAAK,UAAU,OAAO,CAAC,KAAsB;QAC3C,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5G,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;QAChG,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7E,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,CAAC,IAAI,iCAAiC,CAAC,CAAC;QACjI,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAoB,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACrH,CAAC;IACD,OAAO,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK;YACxF,MAAM,EAAE,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;YACnC,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;SACzI,CAAC,CAAC,CAAC,EAAE,CAAC;AACT,CAAC"}

package/dist/training/cli/grader/label.d.ts

@@ -0,0 +1,25 @@
+import type { InferenceProvider, InferenceFailureKind } from '../../contracts/execution.js';
+import { type Grader } from '../../graders/index.js';
+export { createGraderTeacher } from './label-call.js';
+export interface LabelOptions {
+    graderFile: string;
+    inputsFile: string;
+    teachers: string[];
+    relabel?: boolean;
+    adjudication?: number;
+    concurrency?: number;
+    limit?: number;
+    signal?: AbortSignal;
+}
+export interface LabelDependencies {
+    provider?: (teacher: Grader['teachers'][number]) => InferenceProvider;
+}
+export declare function runLabel(options: LabelOptions, dependencies?: LabelDependencies): Promise<{
+    labelsFile: string;
+    adjudicationFile?: string;
+    calls: number;
+    votes: number;
+    missing: number;
+    failures: Record<string, Partial<Record<InferenceFailureKind, number>>>;
+    stopped: Record<string, InferenceFailureKind>;
+}>;

package/dist/training/cli/grader/label.js

@@ -0,0 +1,172 @@
+import { ADJUDICATION_SEED } from '../../contracts/index.js';
+import { readFile, writeFile, mkdir, rename } from 'node:fs/promises';
+import path from 'node:path';
+import { stringify } from 'yaml';
+import { askedForInstance, loadGrader, parseLabelsYaml, parseLabels, labelsSchema, stableDigest } from '../../graders/index.js';
+import { parseSchema, parseYaml } from '../../graders/format/index.js';
+import { groupLabelInstances, validateStoredVotes, adjudicationRows, voteProvenance, teacherPolicyDigest } from '../../graders/labeling/index.js';
+import { loadLabelInputs, resolveStoredLabels } from './label-inputs.js';
+import { labelCall, createGraderTeacher } from './label-call.js';
+export { createGraderTeacher } from './label-call.js';
+export async function runLabel(options, dependencies = {}) {
+    const concurrency = options.concurrency ?? 2;
+    for (const [name, value, minimum] of [['concurrency', concurrency, 1], ['limit', options.limit, 1], ['adjudication', options.adjudication, 0]]) {
+        if (value !== undefined && (!Number.isSafeInteger(value) || value < minimum))
+            throw new Error(`Invalid ${name}`);
+    }
+    const grader = await loadGrader(options.graderFile);
+    if (typeof grader.golden !== 'string')
+        throw new Error('grader label requires a separate golden labels file; inline golden labels are read-only');
+    const labelsFile = grader.golden;
+    if (!options.teachers.length || new Set(options.teachers).size !== options.teachers.length)
+        throw new Error('Select unique teachers');
+    const teachers = options.teachers.map(id => {
+        const teacher = grader.teachers.find(t => t.id === id);
+        if (!teacher)
+            throw new Error(`Unknown teacher ${id}`);
+        return teacher;
+    });
+    const inputs = await loadLabelInputs(options.inputsFile, grader, options.limit);
+    let labels;
+    try {
+        const text = await readFile(grader.golden, 'utf8');
+        if (options.relabel) {
+            labels = parseSchema(labelsSchema, parseYaml(text));
+            if (labels.grader !== grader.id)
+                throw new Error('Labels grader mismatch');
+        }
+        else
+            labels = parseLabelsYaml(text, grader);
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT')
+            throw error;
+        labels = { schema: 'spawnfile.training.labels.v1', grader: grader.id, labelers: {}, labels: [] };
+    }
+    for (const teacher of teachers) {
+        const previous = labels.labelers[teacher.id];
+        if (previous && previous.kind !== 'model')
+            throw new Error(`Teacher ${teacher.id} collides with ${previous.kind} labeler`);
+        labels.labelers[teacher.id] = { kind: 'model', provider: teacher.provider, model: teacher.model };
+    }
+    const jobs = [];
+    let checkpoint = Promise.resolve();
+    const persist = () => {
+        parseLabels(labels, grader);
+        const snapshot = stringify(labels);
+        checkpoint = checkpoint.then(async () => {
+            await mkdir(path.dirname(labelsFile), { recursive: true });
+            const temporary = `${labelsFile}.${process.pid}.tmp`;
+            await writeFile(temporary, snapshot, { mode: 0o600 });
+            await rename(temporary, labelsFile);
+        });
+        return checkpoint;
+    };
+    const counts = { calls: 0, votes: 0, missing: 0 };
+    const failures = Object.fromEntries(teachers.map(teacher => [teacher.id, {}]));
+    const stopped = {};
+    const providers = new Map(teachers.map(t => [t.id, (dependencies.provider ?? createGraderTeacher)(t)]));
+    for (const input of inputs) {
+        let label = labels.labels.find(l => l.id === input.label.id);
+        if (label) {
+            const content = (value) => [value.output, value.context, value.lineage, value.example];
+            if (stableDigest(content(label)) !== stableDigest(content(input.label))) {
+                if (!options.relabel)
+                    throw new Error(`Stale content ${label.id}; use --relabel`);
+                Object.assign(label, input.label, { askedByInstance: {} });
+            }
+        }
+        else {
+            label = input.label;
+            labels.labels.push(label);
+        }
+        const target = label;
+        if (options.relabel)
+            for (const name of options.teachers) {
+                for (const votes of Object.values(target.votes))
+                    delete votes[name];
+                delete target.provenance[name];
+                if (target.asked)
+                    target.asked = target.asked.filter(id => id !== name);
+                for (const [key, asked] of Object.entries(target.askedByInstance ?? {}))
+                    target.askedByInstance[key] = asked.filter(id => id !== name);
+            }
+        if (options.relabel)
+            for (const [key, votes] of Object.entries(target.votes)) {
+                if (!Object.keys(votes).length) {
+                    delete target.votes[key];
+                    if (target.askedByInstance?.[key]?.length === 0)
+                        delete target.askedByInstance[key];
+                }
+            }
+        // Snapshot legacy denominators before dispatch can change the output-wide summary.
+        const groups = groupLabelInstances(grader, input.output, input.context);
+        const membership = Object.fromEntries(groups.flatMap(group => group.instances.map(({ instance }) => [instance.key, askedForInstance(target, instance.key)])));
+        target.askedByInstance = { ...target.askedByInstance, ...membership };
+        const outputJobs = [];
+        for (const group of groups) {
+            // Keep empty maps for unanswered instances so adjudication includes missing votes.
+            for (const { instance } of group.instances)
+                target.votes[instance.key] ??= {};
+            for (const teacher of teachers) {
+                const pending = { ...group, instances: group.instances.filter(({ instance }) => {
+                        const receipt = voteProvenance(target, teacher.id, instance.key);
+                        if (receipt?.status === 'refused' && receipt.teacher_policy_digest === teacherPolicyDigest(teacher) &&
+                            receipt.question_digest === instance.definitionDigest && receipt.projection_digest === group.projectionDigest) {
+                            counts.missing++;
+                            return false;
+                        }
+                        return !Object.hasOwn(target.votes[instance.key], teacher.id);
+                    }) };
+                if (!pending.instances.length)
+                    continue;
+                outputJobs.push(async () => {
+                    if (Object.hasOwn(stopped, teacher.id))
+                        return;
+                    for (const { instance } of pending.instances) {
+                        target.askedByInstance[instance.key] = [...new Set([...target.askedByInstance[instance.key], teacher.id])];
+                    }
+                    target.asked = [...new Set([...target.asked ?? [], teacher.id])];
+                    const result = await labelCall(target, pending, teacher, providers.get(teacher.id), options.signal);
+                    if (result.stopped)
+                        stopped[teacher.id] = result.stopped;
+                    const teacherFailures = failures[teacher.id] ??= {};
+                    for (const [kind, count] of Object.entries(result.failures)) {
+                        const failureKind = kind;
+                        teacherFailures[failureKind] = (teacherFailures[failureKind] ?? 0) + count;
+                    }
+                    counts.calls += result.calls;
+                    counts.votes += result.votes;
+                    counts.missing += result.missing;
+                });
+            }
+        }
+        jobs.push(async () => {
+            for (const job of outputJobs) {
+                options.signal?.throwIfAborted();
+                await job();
+            }
+            await persist();
+        });
+    }
+    validateStoredVotes(grader, await resolveStoredLabels(labels, grader.golden), { resolved: true });
+    let next = 0;
+    const results = await Promise.allSettled(Array.from({ length: Math.min(concurrency, jobs.length) }, async () => {
+        while (next < jobs.length) {
+            options.signal?.throwIfAborted();
+            await jobs[next++]();
+        }
+    }));
+    const failed = results.find(result => result.status === 'rejected');
+    if (failed?.status === 'rejected')
+        throw failed.reason;
+    await persist();
+    let adjudicationFile;
+    if (options.adjudication !== undefined) {
+        adjudicationFile = `${grader.golden}.adjudication.yaml`;
+        await writeFile(adjudicationFile, stringify({ schema: 'spawnfile.training.grader-adjudication.v1', grader: grader.id,
+            seed: ADJUDICATION_SEED, items: adjudicationRows(labels, options.adjudication) }), { mode: 0o600 });
+    }
+    return { labelsFile: grader.golden, ...(adjudicationFile ? { adjudicationFile } : {}), ...counts, failures, stopped };
+}
+//# sourceMappingURL=label.js.map

package/dist/training/cli/grader/label.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"label.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/label.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAEjC,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,eAAe,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAA4B,MAAM,wBAAwB,CAAC;AAC1J,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AAClJ,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAOtD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB,EAAE,eAAkC,EAAE;IAIxF,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,cAAc,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAU,EAAE,CAAC;QACxJ,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IACnH,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yFAAyF,CAAC,CAAC;IAClJ,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,QAAQ,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IACtI,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QACzC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACvD,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAChF,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,GAAG,WAAW,CAAC,YAAY,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;YACpD,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC7E,CAAC;;YAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;QACpE,MAAM,GAAG,EAAE,MAAM,EAAE,8BAA8B,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACnG,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,kBAAkB,QAAQ,CAAC,IAAI,UAAU,CAAC,CAAC;QAC3H,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IACpG,CAAC;IACD,MAAM,IAAI,GAA4B,EAAE,CAAC;IACzC,IAAI,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACtC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,GAAG,UAAU,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;YACrD,MAAM,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACtD,MAAM,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QACH,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,EAAmD,CAAC,CAAC,CAAC,CAAC;IAChI,MAAM,OAAO,GAAyC,EAAE,CAAC;IACzD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,IAAI,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxG,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,CAAC,KAAmB,EAAE,EAAE,CAAC,CAAC,KAAM,CAAC,MAAM,EAAE,KAAM,CAAC,OAAO,EAAE,KAAM,CAAC,OAAO,EAAE,KAAM,CAAC,OAAO,CAAC,CAAC;YACzG,IAAI,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACxE,IAAI,CAAC,OAAO,CAAC,OAAO;oBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC;gBAClF,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;aAAM,CAAC;YAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,KAAK,CAAC;QACrB,IAAI,OAAO,CAAC,OAAO;YAAE,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACzD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;oBAAE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpE,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC/B,IAAI,MAAM,CAAC,KAAK;oBAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;gBACxE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC;oBAAE,MAAM,CAAC,eAAgB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;YAC1I,CAAC;QACD,IAAI,OAAO,CAAC,OAAO;YAAE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;oBAC/B,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACzB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,CAAC;wBAAE,OAAO,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;QACD,mFAAmF;QACnF,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjG,CAAC,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,eAAe,GAAG,EAAE,GAAG,MAAM,CAAC,eAAe,EAAE,GAAG,UAAU,EAAE,CAAC;QACtE,MAAM,UAAU,GAA4B,EAAE,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,mFAAmF;YACnF,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,KAAK,CAAC,SAAS;gBAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YAC9E,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;wBAC7E,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;wBACjE,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,qBAAqB,KAAK,mBAAmB,CAAC,OAAO,CAAC;4BACjG,OAAO,CAAC,eAAe,KAAK,QAAQ,CAAC,gBAAgB,IAAI,OAAO,CAAC,iBAAiB,KAAK,KAAK,CAAC,gBAAgB,EAAE,CAAC;4BAChH,MAAM,CAAC,OAAO,EAAE,CAAC;4BACjB,OAAO,KAAK,CAAC;wBACf,CAAC;wBACD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAE,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;oBACjE,CAAC,CAAC,EAAE,CAAC;gBACL,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM;oBAAE,SAAS;gBACxC,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;oBACzB,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC;wBAAE,OAAO;oBAC/C,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;wBAC7C,MAAM,CAAC,eAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,eAAgB,CAAC,QAAQ,CAAC,GAAG,CAAE,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;oBAChH,CAAC;oBACD,MAAM,CAAC,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;oBACjE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;oBACrG,IAAI,MAAM,CAAC,OAAO;wBAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;oBACzD,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;oBACpD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5D,MAAM,WAAW,GAAG,IAA4B,CAAC;wBACjD,eAAe,CAAC,WAAW,CAAC,GAAG,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;oBAC7E,CAAC;oBACD,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC;oBAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC;oBAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC;gBAC/F,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACnB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAAC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;gBAAC,MAAM,GAAG,EAAE,CAAC;YAAC,CAAC;YAChF,MAAM,OAAO,EAAE,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IACD,mBAAmB,CAAC,MAAM,EAAE,MAAM,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAClG,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC7G,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAAC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAE,EAAE,CAAC;QAAC,CAAC;IACzF,CAAC,CAAC,CAAC,CAAC;IACJ,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;IACpE,IAAI,MAAM,EAAE,MAAM,KAAK,UAAU;QAAE,MAAM,MAAM,CAAC,MAAM,CAAC;IACvD,MAAM,OAAO,EAAE,CAAC;IAChB,IAAI,gBAAoC,CAAC;IACzC,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACvC,gBAAgB,GAAG,GAAG,MAAM,CAAC,MAAM,oBAAoB,CAAC;QACxD,MAAM,SAAS,CAAC,gBAAgB,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,2CAA2C,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE;YAClH,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACxG,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AACxH,CAAC"}

package/dist/training/cli/grader/records.d.ts

@@ -0,0 +1,20 @@
+import type { DecisionRequest, DecisionResult } from '../../contracts/grading.js';
+import { type Grader, type Labels, type Label, type QuestionInstance, type DecisionQuestion, type Consensus, type CalibrationRecord, type CascadeRecord } from '../../graders/index.js';
+export interface GoldenInstance {
+    label: Label;
+    instance: QuestionInstance;
+    question: DecisionQuestion;
+    golden: Consensus;
+    request: DecisionRequest;
+}
+export declare function independentConsensus(label: Label, labelers: Labels['labelers'], teacher: string, keys: string[]): Record<string, Consensus>;
+export declare function goldenInstances(grader: Grader, labels: Labels): {
+    rows: GoldenInstance[];
+    disputed: number;
+    unavailable: number;
+};
+export declare function rawGood(row: Pick<GoldenInstance, 'request' | 'question'>, result: DecisionResult): number | undefined;
+export declare function calibrationRecord(row: GoldenInstance, rawPGood: number): CalibrationRecord;
+export declare function cascadeRecord(row: GoldenInstance, record: CalibrationRecord, p: number, labelers: Labels['labelers']): CascadeRecord;
+/** Reject ambiguous associations before any result can overwrite another request. */
+export declare function indexDecisionResults(requests: readonly DecisionRequest[], results: readonly DecisionResult[]): Map<string, DecisionResult>;

package/dist/training/cli/grader/records.js

@@ -0,0 +1,65 @@
+import { allQuestions, askedForInstance, deriveConsensus, expandInstances, expectedOptions, optionsFor } from '../../graders/index.js';
+export function independentConsensus(label, labelers, teacher, keys) {
+    const remaining = Object.fromEntries(Object.entries(labelers).filter(([id]) => id !== teacher));
+    return deriveConsensus({ ...label,
+        askedByInstance: Object.fromEntries(keys.map(key => [key, askedForInstance(label, key).filter(id => id !== teacher)])),
+        asked: label.asked?.filter(id => id !== teacher),
+        votes: Object.fromEntries(Object.entries(label.votes).map(([key, votes]) => [key, Object.fromEntries(Object.entries(votes).filter(([id]) => id !== teacher))])) }, remaining, keys);
+}
+export function goldenInstances(grader, labels) {
+    const questions = new Map(allQuestions(grader).map(question => [question.id, question]));
+    const rows = [];
+    let disputed = 0, unavailable = 0;
+    for (const label of labels.labels) {
+        const instances = expandInstances(grader, label.output, label.context);
+        const keys = instances.map(instance => instance.key);
+        const consensus = grader.model?.backend === 'teacher'
+            ? independentConsensus(label, labels.labelers, grader.model.teacher, keys)
+            : deriveConsensus(label, labels.labelers, keys);
+        for (const instance of instances) {
+            if (instance.state !== 'ready') {
+                if (instance.state === 'missing')
+                    unavailable++;
+                continue;
+            }
+            const golden = consensus[instance.key];
+            if (golden.disputed)
+                disputed++;
+            const question = questions.get(instance.questionId);
+            rows.push({ label, instance, golden, question, request: {
+                    key: JSON.stringify([label.id, instance.key]), type: question.type,
+                    optionLabels: optionsFor(question), expectedOptions: expectedOptions(question),
+                    instructions: question.ask ?? question.id, projection: instance.projection,
+                } });
+        }
+    }
+    return { rows, disputed, unavailable };
+}
+export function rawGood(row, result) {
+    if (result.status !== 'ok' || result.probabilities === null)
+        return undefined;
+    const values = row.request.optionLabels.map(option => result.probabilities[String(option)]);
+    if (values.some(value => !Number.isFinite(value) || value < 0 || value > 1) ||
+        Math.abs(values.reduce((sum, value) => sum + value, 0) - 1) > 1e-5)
+        throw new Error('Backend returned invalid option probabilities');
+    return expectedOptions(row.question).reduce((sum, option) => sum + result.probabilities[String(option)], 0);
+}
+export function calibrationRecord(row, rawPGood) {
+    return { questionId: row.question.id, type: row.question.type, group: row.label.lineage,
+        example: row.label.example, source: row.golden.source,
+        ...(row.golden.disputed ? { disputed: true } : {}),
+        rawPGood, good: expectedOptions(row.question).includes(row.golden.answer) };
+}
+export function cascadeRecord(row, record, p, labelers) {
+    return { ...record, p, outputId: row.label.id, projectionDigest: row.instance.projectionDigest,
+        votes: row.label.votes[row.instance.key] ?? {}, asked: askedForInstance(row.label, row.instance.key), labelers,
+        expected: expectedOptions(row.question), goldenLabelers: Object.keys(row.golden.votes) };
+}
+/** Reject ambiguous associations before any result can overwrite another request. */
+export function indexDecisionResults(requests, results) {
+    if (new Set(requests.map(request => request.key)).size !== requests.length) {
+        throw new Error('Duplicate calibration request key');
+    }
+    return new Map(requests.map((request, index) => [request.key, results[index]]));
+}
+//# sourceMappingURL=records.js.map

package/dist/training/cli/grader/records.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"records.js","sourceRoot":"","sources":["../../../../src/training/cli/grader/records.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAExC,MAAM,wBAAwB,CAAC;AAU7F,MAAM,UAAU,oBAAoB,CAAC,KAAY,EAAE,QAA4B,EAAE,OAAe,EAAE,IAAc;IAC9G,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC;IAChG,OAAO,eAAe,CAAC,EAAE,GAAG,KAAK;QAC/B,eAAe,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;QACtH,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC;QAChD,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACzE,CAAC,GAAG,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;AAC9G,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,MAAc;IAC5D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAqB,EAAE,CAAC;IAClC,IAAI,QAAQ,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACvE,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,OAAO,KAAK,SAAS;YACnD,CAAC,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC;YAC1E,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;gBAAC,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS;oBAAE,WAAW,EAAE,CAAC;gBAAC,SAAS;YAAC,CAAC;YAC9F,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAE,CAAC;YACxC,IAAI,MAAM,CAAC,QAAQ;gBAAE,QAAQ,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAE,CAAC;YACrD,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE;oBACtD,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI;oBAClE,YAAY,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,eAAe,EAAE,eAAe,CAAC,QAAQ,CAAC;oBAC9E,YAAY,EAAE,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU;iBAC3E,EAAE,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,GAAiD,EAAE,MAAsB;IAC/F,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAC9E,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,aAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7F,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAM,GAAG,CAAC,IAAI,KAAM,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IAChJ,OAAO,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,aAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAE,EAAE,CAAC,CAAC,CAAC;AACxH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAmB,EAAE,QAAgB;IACrE,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO;QACrF,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;QACrD,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,QAAQ,EAAE,IAAI,EAAE,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAO,CAAC,EAAE,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAmB,EAAE,MAAyB,EAAE,CAAS,EAAE,QAA4B;IACnH,OAAO,EAAE,GAAG,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,gBAAgB;QAC5F,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ;QAC9G,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC7F,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,oBAAoB,CAAC,QAAoC,EAAE,OAAkC;IAC3G,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,CAAC;AACnF,CAAC"}