spawnfile 0.1.24 → 0.1.27

package/dist/training/evaluation/grader.js

@@ -0,0 +1,67 @@
+import { evaluateAtomicGrader } from './atomic-grader.js';
+import { allQuestions, DecisionAccountingError, gradeOutput } from '../graders/index.js';
+import { collectJudgeCalls, result } from './logic/index.js';
+import { selectEvidence } from './selectors.js';
+/** All provider construction belongs to the host; this evaluator only consumes injected routes. */
+export async function evaluateGrader(check, context) {
+    const runtime = context.graders?.[check.grader];
+    if (!runtime)
+        throw new Error(`Unresolved grader runtime ${String(check.grader)}`);
+    if ('kind' in runtime && runtime.kind === 'atomic')
+        return evaluateAtomicGrader(check, context, runtime);
+    const legacy = runtime;
+    const binding = check.grader_binding;
+    if (!binding || legacy.definitionDigest !== binding.definitionDigest || legacy.calibrationDigest !== binding.calibrationDigest) {
+        throw new Error('Grader runtime identity does not match resolved grader/calibration files');
+    }
+    const selected = selectEvidence(check.input, context.snapshot);
+    if (!selected.found)
+        return { ...result('unknown', 'Graded artifact was not observed', context.snapshot.closed), evidence: selected.evidence };
+    const bound = {};
+    const evidence = [...selected.evidence];
+    const selectors = (check.context ?? {});
+    for (const name of Object.keys(selectors)) {
+        if (!Object.hasOwn(legacy.grader.context, name))
+            throw new Error(`Unknown grader context ${name}`);
+        const selection = selectEvidence(selectors[name], context.snapshot);
+        if (selection.found)
+            bound[name] = selection.value;
+        evidence.push(...selection.evidence);
+    }
+    // Direct engine clients must not bypass the resolution guarantee; absent trial data
+    // stays absent so instance expansion records missing coverage instead of empty evidence.
+    const unbound = allQuestions(legacy.grader).flatMap(question => [...question.reads, ...(typeof question.when === 'object' ? [question.when.field] : [])])
+        .filter(read => read.startsWith('context.') && !Object.hasOwn(selectors, read.slice(8)));
+    if (unbound.length)
+        return { ...result('unknown', `Unbound grader context: ${unbound.join(', ')}`, context.snapshot.closed), evidence };
+    const children = [];
+    const gateOutcomes = {};
+    const artifact = context.snapshot.artifacts[check.input.slice(9).split('#')[0]];
+    for (const gate of binding.gates) {
+        if (!context.evaluateCheck)
+            throw new Error('Grader gates require the evaluation engine');
+        const evaluated = await context.evaluateCheck(gate.check, artifact);
+        children.push({ ...evaluated, id: gate.id });
+        evidence.push(...evaluated.evidence);
+        gateOutcomes[gate.id] = evaluated.status === 'skipped' ? 'unknown' : evaluated.status;
+        if (evaluated.status === 'fail')
+            break;
+    }
+    let graded;
+    try {
+        graded = await gradeOutput({ ...legacy, output: selected.value, context: bound, gateOutcomes, signal: context.signal });
+    }
+    catch (error) {
+        if (!(error instanceof DecisionAccountingError))
+            throw error;
+        const judgeCalls = collectJudgeCalls(children);
+        return { ...result('error', error.message, context.snapshot.closed), evidence, children, judgeCalls,
+            decisionAccounting: error.decisionAccounting,
+            usage: judgeCalls.flatMap(call => call.usage ? [call.usage] : []) };
+    }
+    const judgeCalls = collectJudgeCalls([...children, { ...result(graded.status, 'Grader'), judgeCalls: graded.judgeCalls }]);
+    return { ...result(graded.status, `Grader ${legacy.grader.id}; coverage ${graded.coverage}`, context.snapshot.closed),
+        score: graded.score, grading: graded, evidence, children, judgeCalls,
+        usage: judgeCalls.flatMap(call => call.usage ? [call.usage] : []) };
+}
+//# sourceMappingURL=grader.js.map

package/dist/training/evaluation/grader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grader.js","sourceRoot":"","sources":["../../../src/training/evaluation/grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAI1D,OAAO,EAAE,YAAY,EAAE,uBAAuB,EAAE,WAAW,EAAoB,MAAM,qBAAqB,CAAC;AAC3G,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,mGAAmG;AACnG,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAgB,EAAE,OAA0B;IAC/E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,MAAgB,CAAC,CAAC;IAC1D,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACnF,IAAI,MAAM,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,oBAAoB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACzG,MAAM,MAAM,GAAG,OAAuD,CAAC;IACvE,MAAM,OAAO,GAAG,KAAK,CAAC,cAA2C,CAAC;IAClE,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,gBAAgB,KAAK,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,iBAAiB,KAAK,OAAO,CAAC,iBAAiB,EAAE,CAAC;QAC/H,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;IAC9F,CAAC;IACD,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,KAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzE,IAAI,CAAC,QAAQ,CAAC,KAAK;QAAE,OAAO,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,kCAAkC,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAC/I,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAA2B,CAAC;IAClE,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;QACnG,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrE,IAAI,SAAS,CAAC,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC;QACnD,QAAQ,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IACD,oFAAoF;IACpF,yFAAyF;IACzF,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtJ,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,2BAA2B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACxI,MAAM,QAAQ,GAAuB,EAAE,CAAC;IACxC,MAAM,YAAY,GAAgC,EAAE,CAAC;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAE,KAAK,CAAC,KAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IAC7F,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACjC,IAAI,CAAC,OAAO,CAAC,aAAa;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC1F,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACpE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7C,QAAQ,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;QACtF,IAAI,SAAS,CAAC,MAAM,KAAK,MAAM;YAAE,MAAM;IACzC,CAAC;IACD,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QAAC,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAAC,CAAC;IAChI,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,CAAC,KAAK,YAAY,uBAAuB,CAAC;YAAE,MAAM,KAAK,CAAC;QAC7D,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU;YACjG,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;YAC5C,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACxE,CAAC;IACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,GAAG,QAAQ,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC3H,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,MAAM,CAAC,MAAM,CAAC,EAAE,cAAc,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QACnH,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU;QACpE,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACxE,CAAC"}

package/dist/training/evaluation/index.d.ts

@@ -0,0 +1,10 @@
+export { createEvaluationRegistry } from './registry.js';
+export { evaluateNode, evaluatePlan, mayStop } from './engine.js';
+export { selectEvidence, evidenceText, observedAbsence, ownPath } from './selectors.js';
+export { result, combine, negate, weightedScore, checkThreshold } from './logic/index.js';
+export { predicate, bindingValue } from './mechanical/index.js';
+export { createJudgeCache, evidenceUnits } from './semantic/index.js';
+export { renderJudgeTemplate, validateJudgeTemplate } from './semantic/index.js';
+export type { EvidenceUnit, EvidencePacket } from './semantic/index.js';
+export type { PlanMeasurementRepair } from './repair.js';
+export { validateMeasurementRepair } from './repair.js';

package/dist/training/evaluation/index.js

@@ -0,0 +1,9 @@
+export { createEvaluationRegistry } from './registry.js';
+export { evaluateNode, evaluatePlan, mayStop } from './engine.js';
+export { selectEvidence, evidenceText, observedAbsence, ownPath } from './selectors.js';
+export { result, combine, negate, weightedScore, checkThreshold } from './logic/index.js';
+export { predicate, bindingValue } from './mechanical/index.js';
+export { createJudgeCache, evidenceUnits } from './semantic/index.js';
+export { renderJudgeTemplate, validateJudgeTemplate } from './semantic/index.js';
+export { validateMeasurementRepair } from './repair.js';
+//# sourceMappingURL=index.js.map

package/dist/training/evaluation/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/training/evaluation/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACxF,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC1F,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAGjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC"}

package/dist/training/evaluation/logic/index.d.ts

@@ -0,0 +1 @@
+export { result, negate, combine, weightedScore, checkThreshold, collectJudgeCalls } from './results.js';

package/dist/training/evaluation/logic/index.js

@@ -0,0 +1,2 @@
+export { result, negate, combine, weightedScore, checkThreshold, collectJudgeCalls } from './results.js';
+//# sourceMappingURL=index.js.map

package/dist/training/evaluation/logic/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/training/evaluation/logic/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC"}

package/dist/training/evaluation/logic/results.d.ts

@@ -0,0 +1,7 @@
+import type { EvaluationResult, JudgeCall, Verdict, WeightedScore } from '../../contracts/evaluation.js';
+export declare function result(status: Verdict, reason: string, final?: boolean): EvaluationResult;
+export declare function negate(value: EvaluationResult): EvaluationResult;
+export declare function collectJudgeCalls(results: readonly EvaluationResult[]): JudgeCall[];
+export declare function combine(operation: 'all' | 'any', children: EvaluationResult[]): EvaluationResult;
+export declare function weightedScore(weights: Readonly<Record<string, number>>, checks: Readonly<Record<string, EvaluationResult>>): WeightedScore;
+export declare function checkThreshold(score: WeightedScore | undefined, minimum: number): EvaluationResult;

package/dist/training/evaluation/logic/results.js

@@ -0,0 +1,70 @@
+export function result(status, reason, final = true) {
+    return { status, finality: final ? 'final' : 'provisional', reason, evidence: [] };
+}
+export function negate(value) {
+    return { ...value, status: value.status === 'pass' ? 'fail' : value.status === 'fail' ? 'pass' : value.status,
+        reason: `not: ${value.reason}`, children: [value],
+        ...(value.score === undefined ? {} : { score: 1 - value.score }) };
+}
+export function collectJudgeCalls(results) {
+    const calls = new Map();
+    for (const value of results) {
+        for (const call of value.judgeCalls ?? [])
+            if (!calls.has(call.id))
+                calls.set(call.id, call);
+    }
+    return [...calls.values()];
+}
+export function combine(operation, children) {
+    if (children.length === 0)
+        throw new Error('A logical composition cannot be empty');
+    const decisive = operation === 'all' ? 'fail' : 'pass';
+    const unanimous = operation === 'all' ? 'pass' : 'fail';
+    const witnesses = children.filter(child => child.status === decisive);
+    const status = witnesses.length ? decisive
+        : children.every(child => child.status === unanimous) ? unanimous
+            : children.some(child => child.status === 'error') ? 'error'
+                : children.some(child => child.status === 'unknown') ? 'unknown' : 'skipped';
+    const final = witnesses.length ? witnesses.some(child => child.finality === 'final')
+        : children.every(child => child.finality === 'final');
+    const judgeCalls = collectJudgeCalls(children);
+    // A named judge failure nested under all:/any: is the same finding as a top-level
+    // one; dropping it here would make composition the new place errors go anonymous.
+    const failure = status === 'error' ? children.find(child => child.judgeFailure)?.judgeFailure : undefined;
+    return { ...result(status, `${operation}: ${children.map(child => child.status).join(', ')}`, final),
+        children, evidence: children.flatMap(child => child.evidence), ...(judgeCalls.length ? { judgeCalls } : {}),
+        ...(failure ? { judgeFailure: structuredClone(failure) } : {}) };
+}
+export function weightedScore(weights, checks) {
+    const entries = Object.entries(weights);
+    const total = entries.reduce((sum, [, weight]) => sum + weight, 0);
+    if (entries.length === 0 || entries.some(([, weight]) => !Number.isFinite(weight) || weight < 0) ||
+        !Number.isFinite(total) || total <= 0) {
+        throw new Error('Score weights must be finite, nonnegative with a positive total');
+    }
+    let lower = 0;
+    let assessedWeight = 0;
+    for (const [id, weight] of entries) {
+        const check = checks[id];
+        if (!check || (check.status !== 'pass' && check.status !== 'fail'))
+            continue;
+        const score = check.score ?? (check.status === 'pass' ? 1 : 0);
+        if (!Number.isFinite(score) || score < 0 || score > 1)
+            throw new Error('Invalid evaluator score');
+        lower += (weight / total) * score;
+        assessedWeight += weight / total;
+    }
+    const missingWeight = Math.max(0, 1 - assessedWeight);
+    const finality = entries.every(([id, weight]) => weight === 0 || checks[id]?.finality === 'final')
+        ? 'final' : 'provisional';
+    return { lower, upper: lower + missingWeight, assessedWeight, missingWeight, finality };
+}
+export function checkThreshold(score, minimum) {
+    if (!Number.isFinite(minimum) || minimum < 0 || minimum > 1)
+        throw new Error('Invalid score threshold');
+    if (!score || score.missingWeight > 1e-9)
+        return result('unknown', 'Scoring coverage is incomplete', score?.finality === 'final');
+    return { ...result(score.lower >= minimum ? 'pass' : 'fail', `Score ${score.lower}; minimum ${minimum}`, score.finality === 'final'),
+        score: score.lower };
+}
+//# sourceMappingURL=results.js.map

package/dist/training/evaluation/logic/results.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"results.js","sourceRoot":"","sources":["../../../../src/training/evaluation/logic/results.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,MAAM,CAAC,MAAe,EAAE,MAAc,EAAE,KAAK,GAAG,IAAI;IAClE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,KAAuB;IAC5C,OAAO,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM;QAC3G,MAAM,EAAE,QAAQ,KAAK,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC;QACjD,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAoC;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,UAAU,IAAI,EAAE;YAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,SAAwB,EAAE,QAA4B;IAC5E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACpF,MAAM,QAAQ,GAAY,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,SAAS,GAAY,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACtE,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;QACxC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;YACjE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;gBAC5D,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/E,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC;QAClF,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC;IACxD,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC/C,kFAAkF;IAClF,kFAAkF;IAClF,MAAM,OAAO,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1G,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,SAAS,KAAK,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC;QAClG,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3G,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAyC,EACrE,MAAkD;IAClD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC;IACnE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;QAC5F,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;YAAE,SAAS;QAC7E,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAClG,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC;QAClC,cAAc,IAAI,MAAM,GAAG,KAAK,CAAC;IACnC,CAAC;IACD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,EAAE,QAAQ,KAAK,OAAO,CAAC;QAChG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC;IAC5B,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,GAAG,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAC1F,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAgC,EAAE,OAAe;IAC9E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACxG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,aAAa,GAAG,IAAI;QAAE,OAAO,MAAM,CAAC,SAAS,EAAE,gCAAgC,EAAE,KAAK,EAAE,QAAQ,KAAK,OAAO,CAAC,CAAC;IAClI,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,KAAK,CAAC,KAAK,aAAa,OAAO,EAAE,EAAE,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC;QAClI,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;AACzB,CAAC"}

package/dist/training/evaluation/mechanical/index.d.ts

@@ -0,0 +1,6 @@
+export { evaluateText } from './text.js';
+export { evaluateTool } from './tools.js';
+export { evaluateSequence } from './sequence.js';
+export { evaluateTokenEfficiency } from './token-efficiency.js';
+export { predicate, bindingValue } from './predicates.js';
+export { evaluateTextCheck } from './text-checks.js';

package/dist/training/evaluation/mechanical/index.js

@@ -0,0 +1,7 @@
+export { evaluateText } from './text.js';
+export { evaluateTool } from './tools.js';
+export { evaluateSequence } from './sequence.js';
+export { evaluateTokenEfficiency } from './token-efficiency.js';
+export { predicate, bindingValue } from './predicates.js';
+export { evaluateTextCheck } from './text-checks.js';
+//# sourceMappingURL=index.js.map

package/dist/training/evaluation/mechanical/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/training/evaluation/mechanical/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/training/evaluation/mechanical/predicates.d.ts

@@ -0,0 +1,2 @@
+export declare function predicate(expression: string, values: Record<string, unknown>): boolean | undefined;
+export declare function bindingValue(selector: string, values: Record<string, unknown>): unknown;

package/dist/training/evaluation/mechanical/predicates.js

@@ -0,0 +1,27 @@
+import { Environment } from '@marcbachmann/cel-js';
+import { ownPath } from '../selectors.js';
+const environment = new Environment({ limits: { maxAstNodes: 1_000, maxDepth: 40,
+        maxListElements: 1_000, maxMapEntries: 1_000, maxCallArguments: 32 } })
+    .registerVariable('call', 'dyn').registerVariable('context', 'dyn')
+    .registerVariable('bindings', 'dyn').registerVariable('event', 'dyn');
+export function predicate(expression, values) {
+    if (expression.length > 16_384)
+        throw new Error('CEL expression exceeds limit');
+    const parsed = environment.parse(expression);
+    try {
+        const value = parsed(values);
+        if (typeof value !== 'boolean')
+            throw new Error('CEL predicate must return boolean');
+        return value;
+    }
+    catch (error) {
+        // Missing observation data is uncertainty, not a successful match or invented field.
+        if (error instanceof Error && /no such key|not found|undefined|undeclared|missing/i.test(error.message))
+            return undefined;
+        throw error;
+    }
+}
+export function bindingValue(selector, values) {
+    return ownPath(values, selector.split('.'));
+}
+//# sourceMappingURL=predicates.js.map

package/dist/training/evaluation/mechanical/predicates.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"predicates.js","sourceRoot":"","sources":["../../../../src/training/evaluation/mechanical/predicates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE1C,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC9E,eAAe,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE,EAAE,EAAE,CAAC;KACtE,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC;KAClE,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;AAExE,MAAM,UAAU,SAAS,CAAC,UAAkB,EAAE,MAA+B;IAC3E,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAChF,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,KAAK,GAAY,MAAM,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,OAAO,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACrF,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,qFAAqF;QACrF,IAAI,KAAK,YAAY,KAAK,IAAI,qDAAqD,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,SAAS,CAAC;QAC1H,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,MAA+B;IAC5E,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9C,CAAC"}

package/dist/training/evaluation/mechanical/sequence.d.ts

@@ -0,0 +1,3 @@
+import type { CheckNode } from '../../contracts/dataset.js';
+import type { EvaluationContext, EvaluationResult } from '../../contracts/evaluation.js';
+export declare function evaluateSequence(check: CheckNode, context: EvaluationContext): Promise<EvaluationResult>;

package/dist/training/evaluation/mechanical/sequence.js

@@ -0,0 +1,75 @@
+import { capturedCalls, completeToolScope, happensBefore } from '../../capture/index.js';
+import { result } from '../logic/index.js';
+import { bindingValue, predicate } from './predicates.js';
+export async function evaluateSequence(check, context) {
+    const { snapshot } = context;
+    const steps = check.steps;
+    const ending = check.ending_at;
+    const artifact = ending ? snapshot.artifacts[ending.artifact] : undefined;
+    const accepted = artifact?.acceptedCallId;
+    const evidence = artifact ? [{ kind: 'artifact', id: artifact.checkpointId, digest: artifact.digest }] : [];
+    if (ending && !accepted)
+        return { ...result('unknown', 'Accepted artifact has no authentic call lineage', snapshot.closed), evidence };
+    const calls = capturedCalls(snapshot.events).filter(call => call.success === true && call.dispatch && call.completion)
+        .sort((a, b) => (a.dispatch.source < b.dispatch.source ? -1 : a.dispatch.source > b.dispatch.source ? 1 : 0) || a.dispatch.seq - b.dispatch.seq);
+    const matches = [];
+    let uncertain = false;
+    let visited = 0;
+    function search(index, prior, bindings) {
+        if (++visited > 20_000)
+            throw new Error('Sequence matching budget exceeded');
+        const step = steps[index];
+        for (const call of calls) {
+            if (call.tool !== step.tool || prior.some(item => item.id === call.id))
+                continue;
+            if (index === steps.length - 1 && accepted && call.id !== accepted)
+                continue;
+            const previous = prior.at(-1);
+            if (previous) {
+                const order = happensBefore(previous.completion, call.dispatch, snapshot.events, snapshot.coverage);
+                if (order === undefined)
+                    uncertain = true;
+                if (order !== true)
+                    continue;
+            }
+            const values = { call, context: snapshot.context, bindings };
+            const pass = step.where ? predicate(step.where.cel, values) : true;
+            if (pass === undefined)
+                uncertain = true;
+            if (pass !== true)
+                continue;
+            const nextBindings = { ...bindings };
+            let missing = false;
+            for (const [name, selector] of Object.entries(step.bind ?? {})) {
+                const value = bindingValue(selector, values);
+                if (value === undefined) {
+                    uncertain = true;
+                    missing = true;
+                    break;
+                }
+                nextBindings[name] = value;
+            }
+            if (missing)
+                continue;
+            const tuple = [...prior, call];
+            if (index + 1 === steps.length)
+                matches.push(tuple);
+            else
+                search(index + 1, tuple, nextBindings);
+        }
+    }
+    if (!steps.length)
+        throw new Error('Sequence must have steps');
+    search(0, [], {});
+    if (matches.length) {
+        // Every matching tuple is retained; the first witness is deterministic by native source/sequence.
+        return { ...result('pass', `${matches.length} valid ordered tuple(s)`, !ending || snapshot.closed), score: 1,
+            evidence: [...evidence, ...matches.flatMap(tuple => tuple.flatMap(call => [
+                    { kind: 'event', id: call.dispatch.id }, { kind: 'event', id: call.completion.id },
+                ]))] };
+    }
+    const complete = completeToolScope(snapshot.closed, snapshot.coverage);
+    return { ...result(complete && !uncertain ? 'fail' : 'unknown', uncertain
+            ? 'Call ordering or predicate evidence is incomplete' : 'No successful bound sequence', snapshot.closed), evidence };
+}
+//# sourceMappingURL=sequence.js.map

package/dist/training/evaluation/mechanical/sequence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sequence.js","sourceRoot":"","sources":["../../../../src/training/evaluation/mechanical/sequence.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,aAAa,EAAqB,MAAM,wBAAwB,CAAC;AAC5G,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI1D,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAgB,EAAE,OAA0B;IACjF,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAe,CAAC;IACpC,MAAM,MAAM,GAAG,KAAK,CAAC,SAA4D,CAAC;IAClF,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,MAAM,QAAQ,GAAG,QAAQ,EAAE,cAAc,CAAC;IAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAmB,EAAE,EAAE,EAAE,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACrH,IAAI,MAAM,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,iDAAiD,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvI,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC;SACnH,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAS,CAAC,MAAM,GAAG,CAAC,CAAC,QAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAS,CAAC,MAAM,GAAG,CAAC,CAAC,QAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAS,CAAC,GAAG,GAAG,CAAC,CAAC,QAAS,CAAC,GAAG,CAAC,CAAC;IACzJ,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,SAAS,MAAM,CAAC,KAAa,EAAE,KAAqB,EAAE,QAAiC;QACrF,IAAI,EAAE,OAAO,GAAG,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC7E,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC;QAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;gBAAE,SAAS;YACjF,IAAI,KAAK,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,IAAI,IAAI,CAAC,EAAE,KAAK,QAAQ;gBAAE,SAAS;YAC7E,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,UAAW,EAAE,IAAI,CAAC,QAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACtG,IAAI,KAAK,KAAK,SAAS;oBAAE,SAAS,GAAG,IAAI,CAAC;gBAC1C,IAAI,KAAK,KAAK,IAAI;oBAAE,SAAS;YAC/B,CAAC;YACD,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC;YAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACnE,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS,GAAG,IAAI,CAAC;YACzC,IAAI,IAAI,KAAK,IAAI;gBAAE,SAAS;YAC5B,MAAM,YAAY,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;YACrC,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC/D,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAC7C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBAAC,SAAS,GAAG,IAAI,CAAC;oBAAC,OAAO,GAAG,IAAI,CAAC;oBAAC,MAAM;gBAAC,CAAC;gBACrE,YAAY,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC7B,CAAC;YACD,IAAI,OAAO;gBAAE,SAAS;YACtB,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;YAC/B,IAAI,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM;gBAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;gBAC/C,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC/D,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAClB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,kGAAkG;QAClG,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,yBAAyB,EAAE,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;YAC1G,QAAQ,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;oBACxE,EAAE,IAAI,EAAE,OAAgB,EAAE,EAAE,EAAE,IAAI,CAAC,QAAS,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAgB,EAAE,EAAE,EAAE,IAAI,CAAC,UAAW,CAAC,EAAE,EAAE;iBACvG,CAAC,CAAC,CAAC,EAAE,CAAC;IACX,CAAC;IACD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvE,OAAO,EAAE,GAAG,MAAM,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,SAAS;YACvE,CAAC,CAAC,mDAAmD,CAAC,CAAC,CAAC,8BAA8B,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;AACzH,CAAC"}

package/dist/training/evaluation/mechanical/text-checks.d.ts

@@ -0,0 +1,3 @@
+import type { CheckNode } from '../../contracts/dataset.js';
+import type { EvaluationContext, EvaluationResult } from '../../contracts/evaluation.js';
+export declare function evaluateTextCheck(check: CheckNode, context: EvaluationContext): Promise<EvaluationResult>;

package/dist/training/evaluation/mechanical/text-checks.js

@@ -0,0 +1,65 @@
+import { isTextSelector, textSelectors } from '../../cases/index.js';
+import { result } from '../logic/index.js';
+import { observedAbsence, selectEvidence, textReference } from '../selectors.js';
+import { extractedNumbers, normalizeQuote, quotedSpans, stripIgnored } from './text-values.js';
+function textLeaves(value) {
+    if (typeof value === 'string')
+        return [value];
+    if (typeof value === 'number' && Number.isFinite(value))
+        return [String(value)];
+    if (Array.isArray(value))
+        return value.flatMap(textLeaves);
+    if (value && typeof value === 'object')
+        return Object.values(value).flatMap(textLeaves);
+    return [];
+}
+async function selectTexts(selectors, context, sources = false) {
+    const texts = [], evidence = [];
+    for (const selector of selectors) {
+        if (sources && !isTextSelector(selector)) {
+            const text = await context.readText(selector);
+            texts.push(text);
+            evidence.push(textReference('sidecar', selector, text));
+            continue;
+        }
+        const selected = selectEvidence(selector, context.snapshot);
+        if (!selected.found) {
+            const missingInput = !sources && observedAbsence(selector, context.snapshot);
+            return { failure: { ...result(missingInput ? 'fail' : 'unknown', `Missing ${selector}`, context.snapshot.closed),
+                    ...(missingInput ? { score: 0 } : {}), evidence: [...evidence, ...selected.evidence] } };
+        }
+        const strings = textLeaves(selected.value);
+        if (!strings.length)
+            return { failure: result('error', `Text selector requires string or number leaves: ${selector}`, context.snapshot.closed) };
+        texts.push(...strings);
+        evidence.push(...selected.evidence);
+    }
+    return { text: texts.join('\n'), evidence };
+}
+export async function evaluateTextCheck(check, context) {
+    const selected = await selectTexts(textSelectors(check.input), context);
+    if ('failure' in selected)
+        return selected.failure;
+    const ignore = (check.ignore ?? []);
+    const input = stripIgnored(selected.text, ignore);
+    if (check.eval === 'word-count') {
+        const count = input.match(/[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu)?.length ?? 0;
+        const passed = count >= (check.min ?? 0) && count <= (check.max ?? Infinity);
+        return { ...result(passed ? 'pass' : 'fail', `word-count: ${count}`, context.snapshot.closed), score: Number(passed),
+            evidence: selected.evidence, mechanicalEvidence: { count } };
+    }
+    const sources = await selectTexts(textSelectors(check.source), context, true);
+    if ('failure' in sources)
+        return sources.failure;
+    const source = stripIgnored(sources.text, ignore);
+    const numbers = extractedNumbers(input), quotes = quotedSpans(input);
+    const available = new Set(extractedNumbers(source).map(value => value.normalized));
+    const missing = numbers.filter(value => !available.has(value.normalized)).map(value => value.raw);
+    const normalizedSource = normalizeQuote(source);
+    missing.push(...quotes.filter(value => !normalizeQuote(value) || !normalizedSource.includes(normalizeQuote(value))).map(value => `"${value}"`));
+    const total = numbers.length + quotes.length, found = total - missing.length;
+    return { ...result(missing.length <= (check.max_missing ?? 0) ? 'pass' : 'fail', `values-in-source: ${found}/${total}; missing: ${JSON.stringify(missing)}`, context.snapshot.closed),
+        score: total ? found / total : 1, evidence: [...selected.evidence, ...sources.evidence],
+        mechanicalEvidence: { found, total, missing } };
+}
+//# sourceMappingURL=text-checks.js.map

package/dist/training/evaluation/mechanical/text-checks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-checks.js","sourceRoot":"","sources":["../../../../src/training/evaluation/mechanical/text-checks.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAG/F,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3D,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACxF,OAAO,EAAE,CAAC;AACZ,CAAC;AACD,KAAK,UAAU,WAAW,CAAC,SAAmB,EAAE,OAA0B,EAAE,OAAO,GAAG,KAAK;IACzF,MAAM,KAAK,GAAa,EAAE,EAAE,QAAQ,GAAwB,EAAE,CAAC;IAC/D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;YAC1E,SAAS;QACX,CAAC;QACD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,YAAY,GAAG,CAAC,OAAO,IAAI,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7E,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,WAAW,QAAQ,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC9G,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAC7F,CAAC;QACD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,mDAAmD,QAAQ,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACjJ,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAAgB,EAAE,OAA0B;IAClF,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;IACxE,IAAI,SAAS,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAa,CAAC;IAChD,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,wCAAwC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,KAAK,IAAI,CAAE,KAAK,CAAC,GAA0B,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAE,KAAK,CAAC,GAA0B,IAAI,QAAQ,CAAC,CAAC;QAC7H,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,eAAe,KAAK,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC;YAClH,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC9E,IAAI,SAAS,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC;IACjD,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClG,MAAM,gBAAgB,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAChD,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IAChJ,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAC7E,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,CAAE,KAAK,CAAC,WAAkC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EACrG,qBAAqB,KAAK,IAAI,KAAK,cAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpG,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;QACvF,kBAAkB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC;AACpD,CAAC"}

package/dist/training/evaluation/mechanical/text-values.d.ts

@@ -0,0 +1,7 @@
+export declare function stripIgnored(text: string, patterns: readonly string[]): string;
+export declare function extractedNumbers(text: string): {
+    raw: string;
+    normalized: string;
+}[];
+export declare function quotedSpans(text: string): string[];
+export declare function normalizeQuote(text: string): string;

package/dist/training/evaluation/mechanical/text-values.js

@@ -0,0 +1,28 @@
+/** Deliberately lexical: English grouped numbers, dot decimals, units and quoted spans. */
+const currency = { '$': 'USD', '€': 'EUR', '£': 'GBP', '¥': 'JPY' };
+const unit = '(?:USD|EUR|GBP|JPY|[$€£¥])';
+const digits = "(?:\\d{1,3}(?:[, '\\u00a0\\u202f]\\d{3})+|\\d+)(?:\\.\\d+)?";
+const numeric = new RegExp(`(?<![\\p{L}\\p{N}_])(?:${unit}\\s*)?[+−-]?${digits}(?:\\s*(?:%|${unit}))?(?![\\p{L}\\p{N}_])`, 'gu');
+export function stripIgnored(text, patterns) {
+    return patterns.reduce((value, pattern) => value.replace(new RegExp(pattern, 'gu'), ' '), text);
+}
+function normalizeNumber(raw) {
+    const units = raw.match(/USD|EUR|GBP|JPY|[$€£¥%]/g) ?? [];
+    const value = raw.replace(/USD|EUR|GBP|JPY|[$€£¥%]|[, '\u00a0\u202f\s]/g, '').replace('−', '-');
+    // String arithmetic preserves integers above Number.MAX_SAFE_INTEGER.
+    const negative = value.startsWith('-');
+    const [whole, fraction = ''] = value.replace(/^[+-]/, '').split('.');
+    const normalized = `${whole.replace(/^0+(?=\d)/, '')}${fraction.replace(/0+$/, '') ? `.${fraction.replace(/0+$/, '')}` : ''}`;
+    return `${negative && normalized !== '0' ? '-' : ''}${normalized}:${units.map(item => currency[item] ?? item).join(':')}`;
+}
+export function extractedNumbers(text) {
+    return [...text.normalize('NFKC').matchAll(numeric)].map(match => ({ raw: match[0], normalized: normalizeNumber(match[0]) }));
+}
+export function quotedSpans(text) {
+    return [...text.matchAll(/"([^"\r\n]+)"|“([^”\r\n]+)”/gu)].map(match => (match[1] ?? match[2]));
+}
+export function normalizeQuote(text) {
+    return text.normalize('NFKC').replace(/[‘’]/gu, "'").replace(/[“”]/gu, '"').replace(/[‐‑‒–—−]/gu, '-')
+        .replace(/\s+/gu, ' ').replace(/^[\p{P}\s]+|[\p{P}\s]+$/gu, '');
+}
+//# sourceMappingURL=text-values.js.map

package/dist/training/evaluation/mechanical/text-values.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-values.js","sourceRoot":"","sources":["../../../../src/training/evaluation/mechanical/text-values.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,MAAM,QAAQ,GAAqC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;AACtG,MAAM,IAAI,GAAG,4BAA4B,CAAC;AAC1C,MAAM,MAAM,GAAG,6DAA6D,CAAC;AAC7E,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,0BAA0B,IAAI,eAAe,MAAM,eAAe,IAAI,wBAAwB,EAAE,IAAI,CAAC,CAAC;AAEjI,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,QAA2B;IACpE,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;AAClG,CAAC;AACD,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,0BAA0B,CAAC,IAAI,EAAE,CAAC;IAC1D,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAChG,sEAAsE;IACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,CAAC,KAAK,EAAE,QAAQ,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,GAAG,KAAM,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAC/H,OAAO,GAAG,QAAQ,IAAI,UAAU,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AAC5H,CAAC;AACD,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAChI,CAAC;AACD,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;AACnG,CAAC;AACD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC;SACnG,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC;AACpE,CAAC"}

package/dist/training/evaluation/mechanical/text.d.ts

@@ -0,0 +1,3 @@
+import type { CheckNode } from '../../contracts/dataset.js';
+import type { EvaluationContext, EvaluationResult } from '../../contracts/evaluation.js';
+export declare function evaluateText(check: CheckNode, context: EvaluationContext): Promise<EvaluationResult>;

package/dist/training/evaluation/mechanical/text.js

@@ -0,0 +1,40 @@
+import { isDeepStrictEqual } from 'node:util';
+import { compileJsonSchema } from '../../cases/index.js';
+import { result } from '../logic/index.js';
+import { observedAbsence, selectEvidence, textReference } from '../selectors.js';
+export async function evaluateText(check, context) {
+    const selected = selectEvidence(check.input, context.snapshot);
+    if (!selected.found) {
+        // An absence the run observed is a behavioural failure, not missing evidence: the
+        // subject closed its own execution, every channel it declared is accounted for, and
+        // the artifact it had to produce is simply not there. It fails as a measured zero and
+        // keeps its weight, so nothing is scored better by omission. An absence the run could
+        // not observe stays unknown and keeps the trial unscorable, exactly as before.
+        if (observedAbsence(check.input, context.snapshot)) {
+            return { ...result('fail', `Missing ${String(check.input)} in a completed, fully accounted subject execution`), score: 0 };
+        }
+        return result('unknown', `Missing ${String(check.input)}`, context.snapshot.closed);
+    }
+    let passed;
+    if (check.eval === 'equals')
+        passed = isDeepStrictEqual(selected.value, check.value);
+    else if (check.eval === 'contains-any') {
+        const strings = typeof selected.value === 'string' ? [selected.value] : selected.value;
+        if (!Array.isArray(strings) || strings.some(item => typeof item !== 'string')) {
+            return result('error', 'contains-any requires a string or array of strings', context.snapshot.closed);
+        }
+        passed = strings.some(item => check.values.some(value => item.includes(value)));
+    }
+    else if (check.eval === 'json-schema') {
+        const schemaBytes = typeof check.schema === 'string' ? await context.readText(check.schema) : undefined;
+        if (schemaBytes !== undefined)
+            selected.evidence.push(textReference('sidecar', check.schema, schemaBytes));
+        const schema = schemaBytes === undefined ? check.schema : JSON.parse(schemaBytes);
+        const validator = compileJsonSchema(schema);
+        passed = validator(selected.value);
+    }
+    else
+        throw new Error(`Unknown text evaluator ${check.eval}`);
+    return { ...result(passed ? 'pass' : 'fail', `${check.eval}: ${passed ? 'matched' : 'not matched'}`, context.snapshot.closed), evidence: selected.evidence, score: passed ? 1 : 0 };
+}
+//# sourceMappingURL=text.js.map

package/dist/training/evaluation/mechanical/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/training/evaluation/mechanical/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEjF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,OAA0B;IAC7E,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,KAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACpB,kFAAkF;QAClF,oFAAoF;QACpF,sFAAsF;QACtF,sFAAsF;QACtF,+EAA+E;QAC/E,IAAI,eAAe,CAAC,KAAK,CAAC,KAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7D,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,WAAW,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,oDAAoD,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC7H,CAAC;QACD,OAAO,MAAM,CAAC,SAAS,EAAE,WAAW,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,MAAe,CAAC;IACpB,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;SAChF,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;QACvF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC;YAC9E,OAAO,MAAM,CAAC,OAAO,EAAE,oDAAoD,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxG,CAAC;QACD,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAE,KAAK,CAAC,MAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAChG,CAAC;SAAM,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACxG,IAAI,WAAW,KAAK,SAAS;YAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,KAAK,CAAC,MAAgB,EAAE,WAAW,CAAC,CAAC,CAAC;QACrH,MAAM,MAAM,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAClF,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5C,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;;QAAM,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/D,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,EAAE,EACjG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACnF,CAAC"}

package/dist/training/evaluation/mechanical/token-efficiency.d.ts

@@ -0,0 +1,4 @@
+import type { CheckNode } from '../../contracts/dataset.js';
+import type { EvaluationContext, EvaluationResult } from '../../contracts/evaluation.js';
+/** A normalized reward over complete native totals, never an execution budget. */
+export declare function evaluateTokenEfficiency(check: CheckNode, context: EvaluationContext): Promise<EvaluationResult>;

package/dist/training/evaluation/mechanical/token-efficiency.js

@@ -0,0 +1,28 @@
+import { result } from '../logic/index.js';
+const count = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
+/** A normalized reward over complete native totals, never an execution budget. */
+export async function evaluateTokenEfficiency(check, context) {
+    const { snapshot } = context;
+    const target = check.target_tokens, metric = check.metric;
+    if (!count(target) || target === 0 || typeof metric !== 'string' || !['input', 'output', 'total'].includes(metric)) {
+        return result('error', 'Invalid token-efficiency parameters', snapshot.closed);
+    }
+    if (!snapshot.closed)
+        return result('unknown', 'Subject token usage is not final', false);
+    const subject = snapshot.subjectUsage;
+    if (subject?.status !== 'complete' || !subject.usage) {
+        return result('unknown', 'Complete subject token usage is unavailable');
+    }
+    const usage = subject.usage;
+    const values = metric === 'input' ? [usage.inputTokens] : metric === 'output' ? [usage.outputTokens]
+        : [usage.inputTokens, usage.outputTokens];
+    if (!values.every(count))
+        return result('unknown', 'Relevant subject token totals are unknown or invalid');
+    const measured = values.reduce((sum, value) => sum + value, 0);
+    if (!count(measured))
+        return result('unknown', 'Subject token total exceeds the safe integer range');
+    // Ratio form avoids overflowing the denominator; cache/reasoning are subsets.
+    return { ...result('pass', `token-efficiency: ${metric} subject tokens=${measured}; reference=${target}`),
+        score: 1 / (1 + measured / target) };
+}
+//# sourceMappingURL=token-efficiency.js.map

package/dist/training/evaluation/mechanical/token-efficiency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-efficiency.js","sourceRoot":"","sources":["../../../../src/training/evaluation/mechanical/token-efficiency.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,KAAK,GAAG,CAAC,KAAc,EAAmB,EAAE,CAChD,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AAEzE,kFAAkF;AAClF,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAAgB,EAAE,OAA0B;IACxF,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC1D,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACnH,OAAO,MAAM,CAAC,OAAO,EAAE,qCAAqC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC,SAAS,EAAE,kCAAkC,EAAE,KAAK,CAAC,CAAC;IAC1F,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC;IACtC,IAAI,OAAO,EAAE,MAAM,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrD,OAAO,MAAM,CAAC,SAAS,EAAE,6CAA6C,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;QAClG,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,SAAS,EAAE,sDAAsD,CAAC,CAAC;IAC3G,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/D,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC,SAAS,EAAE,oDAAoD,CAAC,CAAC;IACrG,8EAA8E;IAC9E,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,qBAAqB,MAAM,mBAAmB,QAAQ,eAAe,MAAM,EAAE,CAAC;QACvG,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;AACzC,CAAC"}

package/dist/training/evaluation/mechanical/tools.d.ts

@@ -0,0 +1,3 @@
+import type { CheckNode } from '../../contracts/dataset.js';
+import type { EvaluationContext, EvaluationResult } from '../../contracts/evaluation.js';
+export declare function evaluateTool(check: CheckNode, context: EvaluationContext): Promise<EvaluationResult>;