spawnfile 0.1.24 → 0.1.27

package/dist/training/cases/fixtures.js

@@ -0,0 +1,134 @@
+import { validateBindingReferences } from './turn-bindings.js';
+import { canonical, fail, object } from './validation.js';
+function isMapping(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+export function mergeFixtureValues(left, right, path = '$') {
+    if (isMapping(left) && isMapping(right)) {
+        const merged = structuredClone(left);
+        for (const [key, value] of Object.entries(right)) {
+            merged[key] = Object.hasOwn(left, key) ? mergeFixtureValues(left[key], value, `${path}.${key}`) : structuredClone(value);
+        }
+        return merged;
+    }
+    if (canonical(left) !== canonical(right))
+        fail(path, 'conflicting selected fixtures');
+    return structuredClone(left);
+}
+export function overlayValues(base, local) {
+    if (isMapping(base) && isMapping(local)) {
+        const result = structuredClone(base);
+        for (const [key, value] of Object.entries(local))
+            result[key] = overlayValues(base[key], value);
+        return result;
+    }
+    return structuredClone(local);
+}
+export function resolveContext(value, context, path, depth = 0) {
+    if (depth > 64)
+        return fail(path, 'context binding cycle or excessive nesting');
+    if (Array.isArray(value))
+        return value.map((item, index) => resolveContext(item, context, `${path}[${index}]`, depth + 1));
+    if (!isMapping(value))
+        return value;
+    if (Object.hasOwn(value, 'context')) {
+        if (Object.keys(value).length !== 1 || typeof value.context !== 'string')
+            return fail(path, 'invalid context binding');
+        let resolved = context;
+        for (const key of value.context.split('.')) {
+            if (!isMapping(resolved) || !Object.hasOwn(resolved, key))
+                return fail(path, `missing context ${value.context}`);
+            resolved = resolved[key];
+        }
+        return resolveContext(resolved, context, path, depth + 1);
+    }
+    const result = {};
+    for (const [key, item] of Object.entries(value))
+        result[key] = resolveContext(item, context, `${path}.${key}`, depth + 1);
+    return result;
+}
+function changedPaths(base, local, prefix = '') {
+    if (isMapping(local) && !isMapping(base) && !Object.keys(local).length)
+        return [prefix];
+    if (isMapping(local))
+        return Object.entries(local).flatMap(([key, value]) => changedPaths(isMapping(base) ? base[key] : undefined, value, prefix ? `${prefix}.${key}` : key));
+    return canonical(base) === canonical(local) ? [] : [prefix];
+}
+function allowed(path, paths) {
+    return paths.some(prefix => path === prefix || path.startsWith(`${prefix}.`));
+}
+export function expandRecord(dataset, record, options, costOnly = false) {
+    let merged = {};
+    for (const name of record.use ?? []) {
+        const fixture = dataset.fixtures?.[name];
+        if (!fixture)
+            fail(`record.${record.id}.use`, `unknown fixture ${name}`);
+        merged = mergeFixtureValues(merged, fixture, `record.${record.id}`);
+    }
+    const expanded = overlayValues(merged, record);
+    const rootSetup = dataset.setup ?? {};
+    const localSetup = expanded.setup ?? {};
+    const changes = changedPaths(rootSetup, localSetup);
+    if (changes.length) {
+        const profile = rootSetup.profile;
+        if (typeof profile !== 'string')
+            fail(`record.${record.id}.setup`, 'record parameters require a dataset preparation profile');
+        const preparation = options.registry?.preparations?.[profile];
+        if (!preparation && !costOnly)
+            fail(`record.${record.id}.setup`, `unsupported preparation ${profile}`);
+        for (const path of changes) {
+            if (path === 'profile' || (!costOnly && (!allowed(path, preparation.unlockedPaths) || !allowed(path, options.unlockedSetupPaths ?? [])))) {
+                fail(`record.${record.id}.setup.${path}`, 'locked preparation parameter');
+            }
+        }
+    }
+    const context = expanded.context ?? {};
+    expanded.context = resolveFields(context, context, `record.${record.id}.context`, false);
+    if (dataset.setup !== undefined || expanded.setup !== undefined) {
+        const setup = overlayValues(rootSetup, localSetup);
+        const { artifacts, ...parameters } = setup;
+        expanded.setup = resolveFields(parameters, expanded.context, `record.${record.id}.setup`, false);
+        if (artifacts !== undefined) {
+            expanded.setup.artifacts = resolveFields(artifacts, expanded.context, `record.${record.id}.setup.artifacts`, false);
+        }
+        const profile = expanded.setup.profile;
+        if (typeof profile !== 'string')
+            fail(`record.${record.id}.setup.profile`, 'required preparation profile');
+        const preparation = options.registry?.preparations?.[profile];
+        if (!preparation && !costOnly)
+            fail(`record.${record.id}.setup`, `unsupported preparation ${profile}`);
+        const { profile: _profile, ...resolvedParameters } = expanded.setup;
+        if (!costOnly)
+            preparation.validate(resolvedParameters, `record.${record.id}.setup`);
+    }
+    if (expanded.input) {
+        expanded.input = resolveFields(expanded.input, expanded.context, `record.${record.id}.input`, false);
+    }
+    for (const turn of expanded.turns ?? []) {
+        turn.input = resolveFields(turn.input, expanded.context, `record.${record.id}.turn.${turn.id}.input`, true);
+    }
+    if (expanded.reference) {
+        expanded.reference = resolveFields(expanded.reference, expanded.context, `record.${record.id}.reference`, false);
+    }
+    if (expanded.outputs && expanded.setup?.artifacts) {
+        const collectors = object(expanded.setup.artifacts, `record.${record.id}.setup.artifacts`);
+        for (const name of Object.keys(expanded.outputs))
+            if (Object.hasOwn(collectors, name))
+                fail(`record.${record.id}.outputs.${name}`, 'conflicts with trusted preparation artifact');
+    }
+    return expanded;
+}
+/** Containers name fields; only their values may carry the reserved binding forms. */
+function resolveFields(values, context, path, allowDeferred) {
+    const result = {};
+    for (const [key, value] of Object.entries(values)) {
+        const location = `${path}.${key}`;
+        if (!allowDeferred)
+            validateBindingReferences(value, new Set(), location);
+        result[key] = resolveContext(value, context, location);
+        if (!allowDeferred)
+            validateBindingReferences(result[key], new Set(), location);
+    }
+    return result;
+}
+//# sourceMappingURL=fixtures.js.map

package/dist/training/cases/fixtures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../../../src/training/cases/fixtures.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAE1D,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AACD,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,KAAc,EAAE,IAAI,GAAG,GAAG;IAC1E,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC3H,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC;QAAE,IAAI,CAAC,IAAI,EAAE,+BAA+B,CAAC,CAAC;IACtF,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;AAC/B,CAAC;AACD,MAAM,UAAU,aAAa,CAAC,IAAa,EAAE,KAAc;IACzD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QAChG,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;AAChC,CAAC;AACD,MAAM,UAAU,cAAc,CAAC,KAAgB,EAAE,OAAgB,EAAE,IAAY,EAAE,KAAK,GAAG,CAAC;IACxF,IAAI,KAAK,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,4CAA4C,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3H,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;QACvH,IAAI,QAAQ,GAAc,OAAO,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC,IAAI,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACjH,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAc,CAAC;QACxC,CAAC;QACD,OAAO,cAAc,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,IAAiB,EAAE,OAAO,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACvI,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,SAAS,YAAY,CAAC,IAAa,EAAE,KAAc,EAAE,MAAM,GAAG,EAAE;IAC9D,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACxF,IAAI,SAAS,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAC1E,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnG,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAC9D,CAAC;AACD,SAAS,OAAO,CAAC,IAAY,EAAE,KAAwB;IACrD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;AAChF,CAAC;AACD,MAAM,UAAU,YAAY,CAAC,OAAgB,EAAE,MAAqB,EAAE,OAA0B,EAAE,QAAQ,GAAG,KAAK;IAChH,IAAI,MAAM,GAAY,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,MAAM,EAAE,mBAAmB,IAAI,EAAE,CAAC,CAAC;QACzE,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAkB,CAAC;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IACtC,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;QAClC,IAAI,OAAO,OAAO,KAAK,QAAQ;YAAE,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,QAAQ,EAAE,yDAAyD,CAAC,CAAC;QAC9H,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC,OAAO,CAAC,CAAC;QAC9D,IAAI,CAAC,WAAW,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,QAAQ,EAAE,2BAA2B,OAAO,EAAE,CAAC,CAAC;QACvG,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,WAAY,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1I,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,UAAU,IAAI,EAAE,EAAE,8BAA8B,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;IACvC,QAAQ,CAAC,OAAO,GAAG,aAAa,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,MAAM,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;IACzF,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAChE,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,EAAE,UAAU,CAAY,CAAC;QAC9D,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,GAAG,KAAK,CAAC;QAC3C,QAAQ,CAAC,KAAK,GAAG,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,OAAO,EAAE,UAAU,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QACjG,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,CAAC,SAAS,GAAG,aAAa,CAAC,SAAoB,EAAE,QAAQ,CAAC,OAAO,EAAE,UAAU,MAAM,CAAC,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;QACjI,CAAC;QACD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC;QACvC,IAAI,OAAO,OAAO,KAAK,QAAQ;YAAE,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,gBAAgB,EAAE,8BAA8B,CAAC,CAAC;QAC3G,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC,OAAO,CAAC,CAAC;QAC9D,IAAI,CAAC,WAAW,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,QAAQ,EAAE,2BAA2B,OAAO,EAAE,CAAC,CAAC;QACvG,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,kBAAkB,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC;QACpE,IAAI,CAAC,QAAQ;YAAE,WAAY,CAAC,QAAQ,CAAC,kBAAkB,EAAE,UAAU,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,QAAQ,CAAC,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,KAA2B,EAAE,QAAQ,CAAC,OAAO,EAAE,UAAU,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC7H,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,KAA2B,EAAE,QAAQ,CAAC,OAAO,EAAE,UAAU,MAAM,CAAC,EAAE,SAAS,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IACpI,CAAC;IACD,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QACvB,QAAQ,CAAC,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,SAA+B,EAAE,QAAQ,CAAC,OAAO,EAAE,UAAU,MAAM,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;IACzI,CAAC;IACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,UAAU,MAAM,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAC3F,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC;gBAAE,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,YAAY,IAAI,EAAE,EAAE,6CAA6C,CAAC,CAAC;IACpL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,sFAAsF;AACtF,SAAS,aAAa,CAAC,MAAe,EAAE,OAAgB,EAAE,IAAY,EAAE,aAAsB;IAC5F,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC,aAAa;YAAE,yBAAyB,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC1E,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QACvD,IAAI,CAAC,aAAa;YAAE,yBAAyB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/training/cases/grader.d.ts

@@ -0,0 +1,14 @@
+import type { CheckNode } from '../contracts/dataset.js';
+import type { PlanResolution } from './plans.js';
+export interface GraderBinding {
+    definitionDigest: string;
+    calibrationDigest?: string;
+    kind?: 'atomic';
+    templatesDigest?: string;
+    gates: {
+        id: string;
+        check: CheckNode;
+    }[];
+}
+export declare function validateGraderParameters(parameters: Readonly<Record<string, unknown>>, location: string): void;
+export declare function resolveGraderCheck(node: CheckNode, context: PlanResolution, location: string, resolveGate: (check: CheckNode, location: string) => CheckNode, isBuiltin: (name: string) => boolean): CheckNode;

package/dist/training/cases/grader.js

@@ -0,0 +1,122 @@
+import { canonicalWireId } from '../contracts/index.js';
+import path from 'node:path';
+import { allQuestions, parseCalibrationJson, parseGraderYaml, sha256, validateGrader } from '../graders/index.js';
+import { checkBinding, parseYaml } from '../graders/format/index.js';
+import { parseAtomicGrader, validateAtomicTemplates } from '../graders/atomic/index.js';
+import { parseCheckTemplatesYaml } from '../graders/decomposition/index.js';
+import { fail, fields, object, safePath, string } from './validation.js';
+export function validateGraderParameters(parameters, location) {
+    const input = string(parameters.input, `${location}.input`);
+    if (!/^artifact:[^#]+(?:#.*)?$/.test(input))
+        fail(`${location}.input`, 'grader requires an artifact selector');
+    if (parameters.grader_binding === undefined)
+        safePath(parameters.grader, `${location}.grader`);
+    else {
+        string(parameters.grader, `${location}.grader`);
+        const binding = object(parameters.grader_binding, `${location}.grader_binding`);
+        fields(binding, ['definitionDigest', 'calibrationDigest', 'kind', 'templatesDigest', 'gates'], `${location}.grader_binding`);
+        if (binding.kind !== undefined && binding.kind !== 'atomic')
+            fail(location, 'invalid grader binding kind');
+        for (const key of ['definitionDigest', binding.kind === 'atomic' ? 'templatesDigest' : 'calibrationDigest']) {
+            if (!/^[a-f0-9]{64}$/.test(string(binding[key], location)))
+                fail(location, `invalid ${key}`);
+        }
+        if (!Array.isArray(binding.gates))
+            fail(location, 'grader binding gates must be an array');
+    }
+    for (const [name, selector] of Object.entries(object(parameters.context ?? {}, `${location}.context`))) {
+        if (!/^(input:files|context:[A-Za-z_][\w-]*|artifact:[^#]+(?:#.*)?)$/.test(string(selector, `${location}.context.${name}`))) {
+            fail(`${location}.context.${name}`, 'unsupported trial context selector');
+        }
+    }
+}
+export function resolveGraderCheck(node, context, location, resolveGate, isBuiltin) {
+    const filename = safePath(node.grader, `${location}.grader`);
+    if (context.resolvingGrader)
+        fail(location, 'nested graders are not supported as gates');
+    context.resolvingGrader = true;
+    try {
+        return resolveGraderFile(node, context, location, resolveGate, filename, isBuiltin);
+    }
+    finally {
+        context.resolvingGrader = false;
+    }
+}
+function resolveGraderFile(node, context, location, resolveGate, filename, isBuiltin) {
+    if (!context.readCriterionFile || !context.criterionFileDigests)
+        fail(location, `grader requires protected bytes: ${filename}`);
+    const read = (name) => {
+        const files = context.criterionFiles ??= new Map();
+        let bytes = files.get(name);
+        if (bytes === undefined) {
+            try {
+                bytes = Buffer.from(context.readCriterionFile(name));
+            }
+            catch (error) {
+                fail(location, `cannot load grader resource ${name}: ${String(error)}`);
+            }
+            files.set(name, bytes);
+        }
+        const digest = sha256(bytes);
+        context.criterionFileDigests[name] = `sha256:${digest}`;
+        context.versions[`criterion-file:${name}`] = `sha256:${digest}`;
+        return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+    };
+    const source = read(filename);
+    const parsed = parseYaml(source);
+    const atomic = canonicalWireId(parsed?.schema) === 'spawnfile.training.atomic-grader.v1' ? parseAtomicGrader(parsed) : undefined;
+    const legacy = atomic ? undefined : validateGrader(parseGraderYaml(source));
+    const identityMode = legacy?.calibration === 'identity';
+    const resourcePath = safePath(path.posix.join(path.posix.dirname(filename), atomic?.templates ?? legacy.calibration), `${location}.resource`);
+    const resourceSource = identityMode ? 'identity' : read(resourcePath);
+    const templates = atomic ? parseCheckTemplatesYaml(resourceSource) : undefined;
+    if (atomic)
+        validateAtomicTemplates(atomic, templates);
+    else if (!identityMode)
+        parseCalibrationJson(resourceSource);
+    const grader = legacy ?? { context: templates.context, gates: atomic.gates };
+    const bindings = object(node.context ?? {}, `${location}.context`);
+    for (const name of Object.keys(bindings)) {
+        if (!Object.hasOwn(grader.context, name))
+            fail(location, `unknown grader context ${name}`);
+    }
+    if (atomic)
+        for (const name of Object.keys(templates.context)) {
+            if (!Object.hasOwn(bindings, name))
+                fail(location, `unbound grader context ${name}`);
+        }
+    for (const question of legacy ? allQuestions(legacy) : []) {
+        for (const selector of [...question.reads, ...(typeof question.when === 'object' ? [question.when.field] : [])]) {
+            if (selector.startsWith('context.') && !Object.hasOwn(bindings, selector.slice(8))) {
+                fail(location, `unbound grader context ${selector.slice(8)} in question ${question.id}`);
+            }
+        }
+    }
+    const gates = grader.gates.flatMap(gate => 'check' in gate
+        ? [{ id: gate.id, check: resolveGate(gate.check, `${location}.gates.${gate.id}`) }] : []);
+    // Rulesets must not hide unbound tool gates from the grader format validator.
+    if (legacy)
+        validateGrader({ ...legacy, gates: legacy.gates.map(gate => 'check' in gate ? gates.find(resolved => resolved.id === gate.id) : gate) });
+    if (atomic)
+        gates.forEach(gate => checkBinding(gate.check, `${location}.gates.${gate.id}`));
+    const artifact = node.input.slice(9).split('#')[0];
+    function assertBinding(check) {
+        const inputs = Array.isArray(check.input) ? check.input : [check.input];
+        if (check.eval && isBuiltin(check.eval) && inputs.some(input => typeof input === 'string' &&
+            input.split('#')[0] !== `artifact:${artifact}`)) {
+            fail(location, `grader gate input must select graded artifact ${artifact}`);
+        }
+        if (check.eval === 'tool-sequence' && check.ending_at.artifact !== artifact) {
+            fail(location, `grader gate must end at graded artifact ${artifact}`);
+        }
+        for (const child of [...(check.all ?? []), ...(check.any ?? []), ...(check.not ? [check.not] : [])])
+            assertBinding(child);
+    }
+    gates.forEach(gate => assertBinding(gate.check));
+    const grader_binding = { definitionDigest: context.criterionFileDigests[filename].slice(7),
+        ...(atomic ? { kind: 'atomic', templatesDigest: context.criterionFileDigests[resourcePath].slice(7) }
+            : { calibrationDigest: identityMode ? sha256('identity') : context.criterionFileDigests[resourcePath].slice(7) }), gates };
+    context.versions[`grader:${filename}`] = `${grader_binding.definitionDigest}:${grader_binding.templatesDigest ?? grader_binding.calibrationDigest}`;
+    return { ...structuredClone(node), grader: path.resolve(path.dirname(context.sourceName ?? 'dataset.yaml'), filename), grader_binding };
+}
+//# sourceMappingURL=grader.js.map

package/dist/training/cases/grader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grader.js","sourceRoot":"","sources":["../../../src/training/cases/grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAClH,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AACxF,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAUzE,MAAM,UAAU,wBAAwB,CAAC,UAA6C,EAAE,QAAgB;IACtG,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,GAAG,QAAQ,QAAQ,CAAC,CAAC;IAC5D,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,IAAI,CAAC,GAAG,QAAQ,QAAQ,EAAE,sCAAsC,CAAC,CAAC;IAC/G,IAAI,UAAU,CAAC,cAAc,KAAK,SAAS;QAAE,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,QAAQ,SAAS,CAAC,CAAC;SAC1F,CAAC;QACJ,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,QAAQ,SAAS,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,EAAE,GAAG,QAAQ,iBAAiB,CAAC,CAAC;QAChF,MAAM,CAAC,OAAO,EAAE,CAAC,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,GAAG,QAAQ,iBAAiB,CAAC,CAAC;QAC7H,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ;YAAE,IAAI,CAAC,QAAQ,EAAE,6BAA6B,CAAC,CAAC;QAC3G,KAAK,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,mBAAmB,CAAC,EAAE,CAAC;YAC5G,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAC;gBAAE,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC,CAAC;IAC7F,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,IAAI,EAAE,EAAE,GAAG,QAAQ,UAAU,CAAC,CAAC,EAAE,CAAC;QACvG,IAAI,CAAC,gEAAgE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,QAAQ,YAAY,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YAC5H,IAAI,CAAC,GAAG,QAAQ,YAAY,IAAI,EAAE,EAAE,oCAAoC,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAe,EAAE,OAAuB,EAAE,QAAgB,EAC3F,WAA8D,EAAE,SAAoC;IACpG,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,SAAS,CAAC,CAAC;IAC7D,IAAI,OAAO,CAAC,eAAe;QAAE,IAAI,CAAC,QAAQ,EAAE,2CAA2C,CAAC,CAAC;IACzF,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC;QAAC,OAAO,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAAC,CAAC;YACpF,CAAC;QAAC,OAAO,CAAC,eAAe,GAAG,KAAK,CAAC;IAAC,CAAC;AAC9C,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAe,EAAE,OAAuB,EAAE,QAAgB,EACnF,WAA8D,EAAE,QAAgB,EAAE,SAAoC;IACtH,IAAI,CAAC,OAAO,CAAC,iBAAiB,IAAI,CAAC,OAAO,CAAC,oBAAoB;QAAE,IAAI,CAAC,QAAQ,EAAE,oCAAoC,QAAQ,EAAE,CAAC,CAAC;IAChI,MAAM,IAAI,GAAG,CAAC,IAAY,EAAU,EAAE;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,KAAK,IAAI,GAAG,EAAsB,CAAC;QACvE,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,IAAI,CAAC;gBAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;YAAC,CAAC;YAC9D,OAAO,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,QAAQ,EAAE,+BAA+B,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAAC,CAAC;YAC1F,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,CAAC,oBAAqB,CAAC,IAAI,CAAC,GAAG,UAAU,MAAM,EAAE,CAAC;QACzD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,IAAI,EAAE,CAAC,GAAG,UAAU,MAAM,EAAE,CAAC;QAChE,OAAO,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACjE,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAwB,CAAC;IACxD,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,qCAAqC,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACjI,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5E,MAAM,YAAY,GAAG,MAAM,EAAE,WAAW,KAAK,UAAU,CAAC;IACxD,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,IAAI,MAAO,CAAC,WAAW,CAAC,EAAE,GAAG,QAAQ,WAAW,CAAC,CAAC;IAC/I,MAAM,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/E,IAAI,MAAM;QAAE,uBAAuB,CAAC,MAAM,EAAE,SAAU,CAAC,CAAC;SACnD,IAAI,CAAC,YAAY;QAAE,oBAAoB,CAAC,cAAc,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,IAAI,EAAE,OAAO,EAAE,SAAU,CAAC,OAAO,EAAE,KAAK,EAAE,MAAO,CAAC,KAAK,EAAE,CAAC;IAC/E,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,GAAG,QAAQ,UAAU,CAAC,CAAC;IACnE,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;YAAE,IAAI,CAAC,QAAQ,EAAE,0BAA0B,IAAI,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,MAAM;QAAE,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,SAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC;gBAAE,IAAI,CAAC,QAAQ,EAAE,0BAA0B,IAAI,EAAE,CAAC,CAAC;QACvF,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC1D,KAAK,MAAM,QAAQ,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAChH,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,IAAI,CAAC,QAAQ,EAAE,0BAA0B,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,IAAI,IAAI;QACxD,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,CAAC,KAAkB,EAAE,GAAG,QAAQ,UAAU,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzG,8EAA8E;IAC9E,IAAI,MAAM;QAAE,cAAc,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtJ,IAAI,MAAM;QAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,QAAQ,UAAU,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC5F,MAAM,QAAQ,GAAI,IAAI,CAAC,KAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IAChE,SAAS,aAAa,CAAC,KAAgB;QACrC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxE,IAAI,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ;YACvF,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,YAAY,QAAQ,EAAE,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,QAAQ,EAAE,iDAAiD,QAAQ,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,IAAK,KAAK,CAAC,SAAkC,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACtG,IAAI,CAAC,QAAQ,EAAE,2CAA2C,QAAQ,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAAE,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5H,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,MAAM,cAAc,GAAkB,EAAE,gBAAgB,EAAE,OAAO,CAAC,oBAAqB,CAAC,QAAQ,CAAE,CAAC,KAAK,CAAC,CAAC,CAAC;QACzG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAiB,EAAE,eAAe,EAAE,OAAO,CAAC,oBAAqB,CAAC,YAAY,CAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC9G,CAAC,CAAC,EAAE,iBAAiB,EAAE,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAqB,CAAC,YAAY,CAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;IACjI,OAAO,CAAC,QAAQ,CAAC,UAAU,QAAQ,EAAE,CAAC,GAAG,GAAG,cAAc,CAAC,gBAAgB,IAAI,cAAc,CAAC,eAAe,IAAI,cAAc,CAAC,iBAAiB,EAAE,CAAC;IACpJ,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,IAAI,cAAc,CAAC,EAAE,QAAQ,CAAC,EAAE,cAAc,EAAE,CAAC;AAC1I,CAAC"}

package/dist/training/cases/index.d.ts

@@ -0,0 +1,12 @@
+export { parseDatasetYaml } from './parser.js';
+export { resolveDataset, resolveDatasetForCost } from './resolver.js';
+export { validateDatasetSplit, validateCostDatasetSplit, selectedFiles } from './splits.js';
+export { normalizeStages } from './plans.js';
+export { builtinEvaluatorDefinitions } from './builtin-evaluators.js';
+export { compileJsonSchema } from './json-schema.js';
+export { DatasetError, fail, fields, number, string } from './validation.js';
+export { validateTrainingConfiguration, trainingJudgeVersions } from './training/index.js';
+export { projectDatasetCost } from './cost-projection.js';
+export type { CostDataset, CostRecord } from './cost-projection.js';
+export type { GraderBinding } from './grader.js';
+export { isTextSelector, textSelectors } from './text-checks.js';

package/dist/training/cases/index.js

@@ -0,0 +1,11 @@
+export { parseDatasetYaml } from './parser.js';
+export { resolveDataset, resolveDatasetForCost } from './resolver.js';
+export { validateDatasetSplit, validateCostDatasetSplit, selectedFiles } from './splits.js';
+export { normalizeStages } from './plans.js';
+export { builtinEvaluatorDefinitions } from './builtin-evaluators.js';
+export { compileJsonSchema } from './json-schema.js';
+export { DatasetError, fail, fields, number, string } from './validation.js';
+export { validateTrainingConfiguration, trainingJudgeVersions } from './training/index.js';
+export { projectDatasetCost } from './cost-projection.js';
+export { isTextSelector, textSelectors } from './text-checks.js';
+//# sourceMappingURL=index.js.map

package/dist/training/cases/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/training/cases/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtE,OAAO,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5F,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,6BAA6B,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAK1D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/training/cases/json-schema.d.ts

@@ -0,0 +1,3 @@
+import { type ValidateFunction } from 'ajv';
+/** Shared authoring/runtime policy. `required` need not also declare properties. */
+export declare function compileJsonSchema(schema: unknown): ValidateFunction;

package/dist/training/cases/json-schema.js

@@ -0,0 +1,6 @@
+import { Ajv } from 'ajv';
+/** Shared authoring/runtime policy. `required` need not also declare properties. */
+export function compileJsonSchema(schema) {
+    return new Ajv({ strict: true, strictRequired: false, allErrors: true }).compile(schema);
+}
+//# sourceMappingURL=json-schema.js.map

package/dist/training/cases/json-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-schema.js","sourceRoot":"","sources":["../../../src/training/cases/json-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAyC,MAAM,KAAK,CAAC;AAEjE,oFAAoF;AACpF,MAAM,UAAU,iBAAiB,CAAC,MAAe;IAC/C,OAAO,IAAI,GAAG,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAmB,CAAC,CAAC;AACxG,CAAC"}

package/dist/training/cases/parser.d.ts

@@ -0,0 +1,6 @@
+import type { InferenceProviderRegistry } from '../contracts/inference-registry.js';
+import type { DatasetDocument } from '../contracts/dataset.js';
+export declare function parseDatasetYaml(source: string, options?: {
+    sourceName?: string;
+    inferenceProviders?: InferenceProviderRegistry;
+}): DatasetDocument;

package/dist/training/cases/parser.js

@@ -0,0 +1,45 @@
+import { createHash } from 'node:crypto';
+import { isAlias, isMap, isScalar, isSeq, LineCounter, parseDocument } from 'yaml';
+import { validateDataset } from './schema.js';
+import { fail } from './validation.js';
+export function parseDatasetYaml(source, options = {}) {
+    const sourceName = options.sourceName ?? '<dataset>';
+    if (Buffer.byteLength(source) > 2_000_000)
+        fail(sourceName, 'dataset exceeds 2 MB');
+    const lineCounter = new LineCounter();
+    const document = parseDocument(source, { lineCounter, uniqueKeys: true, version: '1.2', merge: false });
+    const problems = [...document.errors, ...document.warnings];
+    if (problems.length)
+        fail(sourceName, problems.map(error => error.message).join('\n'));
+    const locations = {};
+    let count = 0;
+    function visit(node, path, depth) {
+        if (++count > 50_000 || depth > 64)
+            fail(path, 'dataset node/depth resource limit exceeded');
+        if (!node)
+            return;
+        if (isAlias(node))
+            fail(path, 'YAML aliases are unsupported; use named fixtures');
+        if (node.range) {
+            const position = lineCounter.linePos(node.range[0]);
+            locations[path] = { line: position.line, column: position.col };
+        }
+        if (isMap(node)) {
+            for (const pair of node.items) {
+                if (!isScalar(pair.key) || typeof pair.key.value !== 'string')
+                    fail(path, 'mapping keys must be strings');
+                visit(pair.value, `${path}.${pair.key.value}`, depth + 1);
+            }
+        }
+        if (isSeq(node))
+            node.items.forEach((item, index) => visit(item, `${path}[${index}]`, depth + 1));
+    }
+    visit(document.contents, '$', 0);
+    return {
+        dataset: validateDataset(document.toJS({ maxAliasCount: 0 }), options.inferenceProviders),
+        sourceName,
+        sourceDigest: `sha256:${createHash('sha256').update(source).digest('hex')}`,
+        locations,
+    };
+}
+//# sourceMappingURL=parser.js.map

package/dist/training/cases/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../../src/training/cases/parser.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AAGnF,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAEvC,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,UAAmF,EAAE;IACpI,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,WAAW,CAAC;IACrD,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,SAAS;QAAE,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;IACpF,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC5D,IAAI,QAAQ,CAAC,MAAM;QAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,MAAM,SAAS,GAAiC,EAAE,CAAC;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,SAAS,KAAK,CAAC,IAA6B,EAAE,IAAY,EAAE,KAAa;QACvE,IAAI,EAAE,KAAK,GAAG,MAAM,IAAI,KAAK,GAAG,EAAE;YAAE,IAAI,CAAC,IAAI,EAAE,4CAA4C,CAAC,CAAC;QAC7F,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,OAAO,CAAC,IAAI,CAAC;YAAE,IAAI,CAAC,IAAI,EAAE,kDAAkD,CAAC,CAAC;QAClF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACpD,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,GAAG,EAAE,CAAC;QAClE,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAChB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,QAAQ;oBAAE,IAAI,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC;gBAC1G,KAAK,CAAC,IAAI,CAAC,KAAa,EAAE,GAAG,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAY,EAAE,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAC5G,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IACjC,OAAO;QACL,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;QACzF,UAAU;QACV,YAAY,EAAE,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;QAC3E,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/training/cases/plans.d.ts

@@ -0,0 +1,14 @@
+import type { Dataset, DatasetRegistry, LiveCheck, Stage } from '../contracts/dataset.js';
+export interface PlanResolution {
+    registry: DatasetRegistry;
+    sourceName?: string;
+    resolvingGrader?: boolean;
+    versions: Record<string, string>;
+    artifacts?: ReadonlySet<string>;
+    criterionFileDigests?: Record<string, string>;
+    criterionFiles?: Map<string, Uint8Array>;
+    readCriterionFile?: (path: string) => string | Uint8Array;
+}
+export declare function normalizeStages(plan: Pick<Dataset, 'checks' | 'stages'>, shorthandId?: string): Stage[];
+export declare function resolveStages(stages: Stage[], context: PlanResolution, mechanicalOnly?: boolean): Stage[];
+export declare function resolveLive(checks: LiveCheck[], context: PlanResolution): LiveCheck[];

package/dist/training/cases/plans.js

@@ -0,0 +1,186 @@
+import { isTextSelector, textSelectors } from './text-checks.js';
+import { resolveGraderCheck } from './grader.js';
+import { createHash } from 'node:crypto';
+import { builtinEvaluatorDefinitions } from './builtin-evaluators.js';
+import { compileJsonSchema } from './json-schema.js';
+import { validateBindingReferences } from './turn-bindings.js';
+import { checkShape } from './check-schema.js';
+import { fail, object, string } from './validation.js';
+function combine(values) {
+    return {
+        mechanical: values.every(value => value.mechanical),
+        incremental: values.every(value => value.incremental),
+        stopProof: values.every(value => value.stopProof),
+    };
+}
+export function normalizeStages(plan, shorthandId = 'shared-checks') {
+    if (plan.stages)
+        return structuredClone(plan.stages);
+    if (!plan.checks)
+        return [];
+    return [{ id: shorthandId, checks: structuredClone(plan.checks), gate: { all: plan.checks.map(check => ({ passed: check.id })) } }];
+}
+function artifacts(node, context, path) {
+    if (!context.artifacts)
+        return;
+    const names = [];
+    for (const key of ['input', 'evidence', 'reference', 'source']) {
+        for (const selector of textSelectors(node[key])) {
+            if (typeof selector === 'string' && selector.startsWith('artifact:'))
+                names.push(selector.slice(9).split('#')[0]);
+        }
+    }
+    if (node.ending_at)
+        names.push(string(object(node.ending_at, path).artifact, path));
+    for (const name of names)
+        if (!context.artifacts.has(name))
+            fail(path, `undeclared artifact ${name}`);
+}
+function expand(node, context, known, path, ruleStack = []) {
+    checkShape(node, path);
+    if (node.use) {
+        for (const [key, value] of Object.entries(node.with ?? {}))
+            validateBindingReferences(value, new Set(), `${path}.with.${key}`);
+        if (ruleStack.includes(node.use) || ruleStack.length > 32)
+            fail(path, 'ruleset expansion cycle or depth limit');
+        const ruleset = context.registry.rulesets?.[node.use];
+        if (!ruleset)
+            fail(path, `unsupported ruleset ${node.use}`);
+        context.versions[`ruleset:${node.use}`] = ruleset.version;
+        const expanded = ruleset.expand(structuredClone(node.with ?? {}));
+        const result = expand(expanded, context, known, path, [...ruleStack, node.use]);
+        return { ...result, node: { ...result.node, ...(node.id ? { id: node.id } : {}) } };
+    }
+    if (node.eval) {
+        const definition = context.registry.evaluators?.[node.eval] ?? builtinEvaluatorDefinitions[node.eval];
+        if (!definition)
+            fail(path, `unsupported evaluator ${node.eval}`);
+        context.versions[`evaluator:${node.eval}`] = definition.version;
+        const { id: _id, eval: _eval, ...parameters } = node;
+        for (const [key, value] of Object.entries(parameters))
+            validateBindingReferences(value, new Set(), `${path}.${key}`);
+        definition.validate(parameters, path);
+        artifacts(node, context, path);
+        const sidecar = node.eval === 'json-schema' && typeof node.schema === 'string' ? node.schema :
+            node.eval === 'llm-rubric' && node.rubric && typeof node.rubric === 'object' ?
+                node.rubric.file : undefined;
+        const sidecars = node.eval === 'values-in-source' ? textSelectors(node.source).filter(source => !isTextSelector(source)) : sidecar ? [sidecar] : [];
+        for (const sidecar of sidecars) {
+            if (!context.readCriterionFile || !context.criterionFileDigests)
+                fail(path, `criterion sidecar requires protected bytes: ${sidecar}`);
+            const files = context.criterionFiles ??= new Map();
+            let bytes = files.get(sidecar);
+            if (bytes === undefined) {
+                bytes = Buffer.from(context.readCriterionFile(sidecar));
+                files.set(sidecar, bytes);
+            }
+            if (node.eval === 'json-schema')
+                compileJsonSchema(JSON.parse(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes)));
+            context.criterionFileDigests[sidecar] = `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
+            context.versions[`criterion-file:${sidecar}`] = context.criterionFileDigests[sidecar];
+        }
+        if (node.eval === 'llm-rubric') {
+            const name = string(node.judge, `${path}.judge`);
+            const route = context.registry.judges?.[name];
+            if (!route)
+                fail(path, `unresolved judge route ${name}`);
+            context.versions[`judge:${name}`] = route.version;
+        }
+        if (node.eval === 'embedding-similarity') {
+            const name = string(node.provider, `${path}.provider`);
+            const route = context.registry.embeddings?.[name];
+            if (!route)
+                fail(path, `unresolved embedding route ${name}`);
+            context.versions[`embedding:${name}`] = route.version;
+        }
+        const resolvedNode = node.eval === 'grader' ? resolveGraderCheck(node, context, path, (gate, gatePath) => expand(gate, context, new Map(), gatePath, ruleStack).node, name => Object.hasOwn(builtinEvaluatorDefinitions, name)) : structuredClone(node);
+        return { node: resolvedNode, nature: {
+                mechanical: definition.kind === 'mechanical', incremental: definition.incremental,
+                stopProof: definition.stopProof,
+            } };
+    }
+    if (node.passed || node.score) {
+        const id = node.passed ?? node.score;
+        const nature = known.get(id);
+        if (!nature)
+            fail(path, `unresolved, forward or cyclic result reference ${id}`);
+        return { node: structuredClone(node), nature: node.score ? { ...nature, stopProof: false } : nature };
+    }
+    const operation = node.all ? 'all' : node.any ? 'any' : 'not';
+    const children = operation === 'not' ? [node.not] : node[operation];
+    const resolved = children.map((child, index) => expand(child, context, known, `${path}.${operation}[${index}]`, ruleStack));
+    return {
+        node: { ...(node.id ? { id: node.id } : {}), [operation]: operation === 'not' ? resolved[0].node : resolved.map(result => result.node) },
+        nature: combine(resolved.map(result => result.nature)),
+    };
+}
+export function resolveStages(stages, context, mechanicalOnly = false) {
+    const known = new Map();
+    const identities = new Set();
+    return stages.map((stage, index) => {
+        const path = `stages[${index}]`;
+        if (identities.has(stage.id))
+            fail(path, `duplicate plan ID ${stage.id}`);
+        identities.add(stage.id);
+        const when = stage.when ? expand(stage.when, context, known, `${path}.when`) : undefined;
+        if (when && mechanicalOnly && !when.nature.mechanical)
+            fail(path, 'semantic conditions cannot gate later work');
+        const checks = stage.checks.map((check, checkIndex) => {
+            const id = check.id;
+            if (identities.has(id))
+                fail(path, `duplicate plan ID ${id}`);
+            identities.add(id);
+            const result = expand(check, context, known, `${path}.checks[${checkIndex}]`);
+            if (mechanicalOnly && !result.nature.mechanical)
+                fail(path, 'semantic/embedding checks cannot gate a later turn');
+            known.set(id, result.nature);
+            return result;
+        });
+        let nature = combine(checks.map(check => check.nature));
+        if (when)
+            nature = combine([nature, when.nature]);
+        if (stage.score) {
+            for (const id of Object.keys(stage.score.linear)) {
+                if (!checks.some(check => check.node.id === id))
+                    fail(path, `score component is not a check in this stage: ${id}`);
+            }
+        }
+        known.set(stage.id, nature);
+        const gate = stage.gate ? expand(stage.gate, context, known, `${path}.gate`) : undefined;
+        if (gate && mechanicalOnly && !gate.nature.mechanical)
+            fail(path, 'semantic/embedding gates cannot block later work');
+        if (gate)
+            for (const check of checks) {
+                const id = check.node.id;
+                if (!Object.hasOwn(stage.score?.linear ?? {}, id) && !referencesPassed(gate.node, id)) {
+                    fail(path, `non-gate criterion ${id} requires an explicit score weight`);
+                }
+            }
+        if (gate && referencesPassed(gate.node, stage.id))
+            fail(path, 'stage gate cannot depend on its own verdict');
+        known.set(stage.id, combine([nature, ...(gate ? [gate.nature] : [])]));
+        return { id: stage.id, checks: checks.map(check => check.node),
+            ...(when ? { when: when.node } : {}), ...(stage.score ? { score: structuredClone(stage.score) } : {}),
+            ...(gate ? { gate: gate.node } : {}) };
+    });
+}
+export function resolveLive(checks, context) {
+    return checks.map((check, index) => {
+        const path = `live.checks[${index}]`;
+        const { on, on_fail: action = 'continue', ...criterion } = check;
+        const resolved = expand(criterion, context, new Map(), path);
+        if (!resolved.nature.incremental)
+            fail(path, 'evaluator does not support incremental observation');
+        if (action === 'stop' && (!resolved.nature.mechanical || !resolved.nature.stopProof)) {
+            fail(path, 'hard stopping requires mechanical finality proofs throughout the criterion');
+        }
+        if (typeof on !== 'string' && context.artifacts && !context.artifacts.has(on.artifact))
+            fail(path, `undeclared live artifact ${on.artifact}`);
+        return { ...resolved.node, id: check.id, on: structuredClone(on), on_fail: action };
+    });
+}
+function referencesPassed(node, id) {
+    return node.passed === id || (node.not ? referencesPassed(node.not, id) : false) ||
+        [...(node.all ?? []), ...(node.any ?? [])].some(child => referencesPassed(child, id));
+}
+//# sourceMappingURL=plans.js.map