vieval 0.0.4 → 0.0.6

package/dist/core/assertions/index.d.mts

@@ -1,2 +1,314 @@
-import { C as expectToolCallArgs, S as expectStructuredOutput, T as toRunScores, _ as expectMustExclude, a as CustomAssertionOptions, b as expectRegex, c as RegexAssertionOptions, d as StructuredOutputAssertionOptions, f as ToolCall, g as expectCustom, h as evaluateAssertions, i as AssertionState, l as RubricAssertionOptions, m as collectFailedAssertions, n as AssertionContext, o as MustExcludeAssertionOptions, p as ToolCallArgsAssertionOptions, r as AssertionOutcome, s as MustIncludeAssertionOptions, t as Assertion, u as RubricJudgeResult, v as expectMustInclude, w as normalizeMatchText, x as expectRubric, y as expectNot } from "../../index-C3gPFmcR.mjs";
-export { Assertion, AssertionContext, AssertionOutcome, AssertionState, CustomAssertionOptions, MustExcludeAssertionOptions, MustIncludeAssertionOptions, RegexAssertionOptions, RubricAssertionOptions, RubricJudgeResult, StructuredOutputAssertionOptions, ToolCall, ToolCallArgsAssertionOptions, collectFailedAssertions, evaluateAssertions, expectCustom, expectMustExclude, expectMustInclude, expectNot, expectRegex, expectRubric, expectStructuredOutput, expectToolCallArgs, normalizeMatchText, toRunScores };
+import { G as RunScore, K as RunScoreKind } from "../../index-DBZKkpBe.mjs";
+
+//#region src/core/assertions/index.d.ts
+/**
+ * Stores mutable evaluation state for stateful assertion flows.
+ *
+ * Use when:
+ * - assertions need to share counters, rolling metrics, or memoized values
+ * - a scenario evaluates multiple steps and expects state-aware checks
+ */
+type AssertionState = Map<string, unknown>;
+/**
+ * Represents one tool call emitted by a model response.
+ */
+interface ToolCall {
+  /**
+   * Tool name used by the call.
+   */
+  name: string;
+  /**
+   * Tool arguments payload.
+   */
+  args: unknown;
+}
+/**
+ * Normalized assertion context for one model output.
+ */
+interface AssertionContext {
+  /**
+   * Plain text model output used by text assertions.
+   */
+  text: string;
+  /**
+   * Optional structured output parsed from the model response.
+   */
+  structuredOutput?: unknown;
+  /**
+   * Optional tool calls extracted from the model response.
+   */
+  toolCalls?: readonly ToolCall[];
+  /**
+   * Shared mutable state for stateful assertion measurement.
+   */
+  state: AssertionState;
+}
+/**
+ * Result for one assertion evaluation.
+ */
+interface AssertionOutcome {
+  /**
+   * Stable assertion id.
+   */
+  id: string;
+  /**
+   * Assertion family emitted as run score kind.
+   */
+  scoreKind: RunScoreKind;
+  /**
+   * Whether the assertion passed.
+   */
+  pass: boolean;
+  /**
+   * Normalized score in the `0..1` range.
+   */
+  score: number;
+  /**
+   * Human-readable reason for logs and reports.
+   */
+  reason: string;
+}
+/**
+ * Async assertion function used by eval scenarios.
+ */
+type Assertion = (context: AssertionContext) => Promise<AssertionOutcome>;
+/**
+ * Normalizes text for matching.
+ *
+ * Before: `"  Hello\nWorld  "`
+ * After: `"hello world"`
+ */
+declare function normalizeMatchText(value: string, caseSensitive: boolean): string;
+/**
+ * Options for include-keyword assertions.
+ */
+interface MustIncludeAssertionOptions {
+  /**
+   * Stable assertion id.
+   */
+  id: string;
+  /**
+   * Keywords that must be present.
+   */
+  keywords: readonly string[];
+  /**
+   * Match mode for keywords.
+   *
+   * @default 'all'
+   */
+  mode?: 'all' | 'any';
+  /**
+   * Case-sensitive matching toggle.
+   *
+   * @default false
+   */
+  caseSensitive?: boolean;
+}
+/**
+ * Creates an assertion that requires specific keywords in model text.
+ *
+ * Example:
+ * `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`
+ */
+declare function expectMustInclude(options: MustIncludeAssertionOptions): Assertion;
+/**
+ * Options for exclude-keyword assertions.
+ */
+interface MustExcludeAssertionOptions {
+  /**
+   * Stable assertion id.
+   */
+  id: string;
+  /**
+   * Keywords that must not appear.
+   */
+  keywords: readonly string[];
+  /**
+   * Case-sensitive matching toggle.
+   *
+   * @default false
+   */
+  caseSensitive?: boolean;
+}
+/**
+ * Creates an assertion that forbids specific keywords.
+ *
+ * Example:
+ * `expectMustExclude({ id: 'no-engine-dump', keywords: ['bestmove', 'ponder'] })`
+ */
+declare function expectMustExclude(options: MustExcludeAssertionOptions): Assertion;
+/**
+ * Options for regular-expression assertions.
+ */
+interface RegexAssertionOptions {
+  /**
+   * Stable assertion id.
+   */
+  id: string;
+  /**
+   * Pattern to apply to model text.
+   */
+  pattern: RegExp;
+}
+/**
+ * Creates an assertion based on a regular expression.
+ *
+ * Example:
+ * `expectRegex({ id: 'starts-with-act', pattern: /^<\|ACT:/ })`
+ */
+declare function expectRegex(options: RegexAssertionOptions): Assertion;
+/**
+ * Options for structured-output assertions.
+ */
+interface StructuredOutputAssertionOptions<TValue> {
+  /**
+   * Stable assertion id.
+   */
+  id: string;
+  /**
+   * Runtime validator for structured output.
+   */
+  validate: (value: unknown) => value is TValue;
+  /**
+   * Optional failure reason.
+   */
+  failureReason?: string;
+}
+/**
+ * Creates an assertion for structured model output.
+ *
+ * Example:
+ * `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`
+ */
+declare function expectStructuredOutput<TValue>(options: StructuredOutputAssertionOptions<TValue>): Assertion;
+/**
+ * Options for tool-call argument assertions.
+ */
+interface ToolCallArgsAssertionOptions {
+  /**
+   * Stable assertion id.
+   */
+  id: string;
+  /**
+   * Tool name to inspect.
+   */
+  toolName: string;
+  /**
+   * Runtime validator for tool arguments.
+   */
+  validate: (args: unknown) => boolean;
+}
+/**
+ * Creates an assertion for validating tool-call arguments.
+ *
+ * Example:
+ * `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`
+ */
+declare function expectToolCallArgs(options: ToolCallArgsAssertionOptions): Assertion;
+/**
+ * Rubric judge result returned by teacher-model or rubric logic.
+ */
+interface RubricJudgeResult {
+  /**
+   * Normalized score in the `0..1` range.
+   */
+  score: number;
+  /**
+   * Judge explanation text.
+   */
+  reason: string;
+  /**
+   * Optional judge model id.
+   */
+  judgeModel?: string;
+}
+/**
+ * Options for rubric assertions.
+ */
+interface RubricAssertionOptions {
+  /**
+   * Stable assertion id.
+   */
+  id: string;
+  /**
+   * Async rubric judge callback.
+   */
+  judge: (context: AssertionContext) => Promise<RubricJudgeResult>;
+  /**
+   * Minimum passing score.
+   *
+   * @default 0.7
+   */
+  minScore?: number;
+}
+/**
+ * Creates a rubric assertion driven by teacher-model style scoring.
+ *
+ * Example:
+ * `expectRubric({ id: 'human-like-tone', judge: judgeFn, minScore: 0.8 })`
+ */
+declare function expectRubric(options: RubricAssertionOptions): Assertion;
+/**
+ * Options for custom assertions.
+ */
+interface CustomAssertionOptions {
+  /**
+   * Stable assertion id.
+   */
+  id: string;
+  /**
+   * Score family emitted by this custom assertion.
+   */
+  scoreKind: RunScoreKind;
+  /**
+   * Custom evaluator callback.
+   */
+  evaluate: (context: AssertionContext) => Promise<{
+    pass: boolean;
+    reason: string;
+    score: number;
+  }> | {
+    pass: boolean;
+    reason: string;
+    score: number;
+  };
+}
+/**
+ * Creates a custom assertion with fully user-defined logic.
+ *
+ * Example:
+ * `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`
+ */
+declare function expectCustom(options: CustomAssertionOptions): Assertion;
+/**
+ * Creates an inverse assertion.
+ *
+ * Example:
+ * `expectNot(expectMustInclude({ id: 'contains-engine-word', keywords: ['bestmove'] }), { id: 'no-engine-word' })`
+ */
+declare function expectNot(assertion: Assertion, options: {
+  id: string;
+}): Assertion;
+/**
+ * Executes assertion list and returns all outcomes.
+ *
+ * Call stack:
+ *
+ * {@link evaluateAssertions}
+ *   -> `assertion(context)`
+ *     -> {@link AssertionOutcome}[]
+ */
+declare function evaluateAssertions(assertions: readonly Assertion[], context: Omit<AssertionContext, 'state'> & {
+  state?: AssertionState;
+}): Promise<AssertionOutcome[]>;
+/**
+ * Converts assertion outcomes to run-score tuples consumed by aggregation.
+ */
+declare function toRunScores(outcomes: readonly AssertionOutcome[]): RunScore[];
+/**
+ * Returns failing assertion outcomes in original order.
+ */
+declare function collectFailedAssertions(outcomes: readonly AssertionOutcome[]): AssertionOutcome[];
+//#endregion
+export { Assertion, AssertionContext, AssertionOutcome, AssertionState, CustomAssertionOptions, MustExcludeAssertionOptions, MustIncludeAssertionOptions, RegexAssertionOptions, RubricAssertionOptions, RubricJudgeResult, StructuredOutputAssertionOptions, ToolCall, ToolCallArgsAssertionOptions, collectFailedAssertions, evaluateAssertions, expectCustom, expectMustExclude, expectMustInclude, expectNot, expectRegex, expectRubric, expectStructuredOutput, expectToolCallArgs, normalizeMatchText, toRunScores };
+//# sourceMappingURL=index.d.mts.map

package/dist/core/assertions/index.mjs

@@ -1,2 +1,183 @@
-import { a as expectMustInclude, c as expectRubric, d as normalizeMatchText, f as toRunScores, i as expectMustExclude, l as expectStructuredOutput, n as evaluateAssertions, o as expectNot, r as expectCustom, s as expectRegex, t as collectFailedAssertions, u as expectToolCallArgs } from "../../assertions-DcAjfVDA.mjs";
+//#region src/core/assertions/index.ts
+/**
+* Normalizes text for matching.
+*
+* Before: `"  Hello\nWorld  "`
+* After: `"hello world"`
+*/
+function normalizeMatchText(value, caseSensitive) {
+	const compactedWhitespace = value.trim().replaceAll(/\s+/g, " ");
+	if (caseSensitive) return compactedWhitespace;
+	return compactedWhitespace.toLowerCase();
+}
+function clampScore(score) {
+	if (Number.isNaN(score)) return 0;
+	if (score < 0) return 0;
+	if (score > 1) return 1;
+	return score;
+}
+function createOutcome(id, scoreKind, pass, score, reason) {
+	return {
+		id,
+		pass,
+		reason,
+		score: clampScore(score),
+		scoreKind
+	};
+}
+/**
+* Creates an assertion that requires specific keywords in model text.
+*
+* Example:
+* `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`
+*/
+function expectMustInclude(options) {
+	return async (context) => {
+		if (options.keywords.length === 0) return createOutcome(options.id, "exact", true, 1, "No required keywords configured.");
+		const caseSensitive = options.caseSensitive ?? false;
+		const normalizedText = normalizeMatchText(context.text, caseSensitive);
+		const matches = options.keywords.filter((keyword) => {
+			const normalizedKeyword = normalizeMatchText(keyword, caseSensitive);
+			return normalizedText.includes(normalizedKeyword);
+		});
+		const pass = (options.mode ?? "all") === "all" ? matches.length === options.keywords.length : matches.length > 0;
+		const score = options.keywords.length === 0 ? 1 : matches.length / options.keywords.length;
+		return createOutcome(options.id, "exact", pass, score, pass ? `Matched ${matches.length}/${options.keywords.length} required keywords.` : `Matched ${matches.length}/${options.keywords.length} required keywords.`);
+	};
+}
+/**
+* Creates an assertion that forbids specific keywords.
+*
+* Example:
+* `expectMustExclude({ id: 'no-engine-dump', keywords: ['bestmove', 'ponder'] })`
+*/
+function expectMustExclude(options) {
+	return async (context) => {
+		if (options.keywords.length === 0) return createOutcome(options.id, "exact", true, 1, "No excluded keywords configured.");
+		const caseSensitive = options.caseSensitive ?? false;
+		const normalizedText = normalizeMatchText(context.text, caseSensitive);
+		const forbiddenMatches = options.keywords.filter((keyword) => {
+			const normalizedKeyword = normalizeMatchText(keyword, caseSensitive);
+			return normalizedText.includes(normalizedKeyword);
+		});
+		const pass = forbiddenMatches.length === 0;
+		const score = pass ? 1 : 0;
+		return createOutcome(options.id, "exact", pass, score, pass ? "No forbidden keywords found." : `Forbidden keywords found: ${forbiddenMatches.join(", ")}`);
+	};
+}
+/**
+* Creates an assertion based on a regular expression.
+*
+* Example:
+* `expectRegex({ id: 'starts-with-act', pattern: /^<\|ACT:/ })`
+*/
+function expectRegex(options) {
+	return async (context) => {
+		const pass = options.pattern.test(context.text);
+		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? "Regex matched response text." : `Regex did not match: ${options.pattern}`);
+	};
+}
+/**
+* Creates an assertion for structured model output.
+*
+* Example:
+* `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`
+*/
+function expectStructuredOutput(options) {
+	return async (context) => {
+		const pass = options.validate(context.structuredOutput);
+		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? "Structured output matched validator." : options.failureReason ?? "Structured output validation failed.");
+	};
+}
+/**
+* Creates an assertion for validating tool-call arguments.
+*
+* Example:
+* `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`
+*/
+function expectToolCallArgs(options) {
+	return async (context) => {
+		const targetCall = (context.toolCalls ?? []).find((call) => call.name === options.toolName);
+		if (targetCall == null) return createOutcome(options.id, "exact", false, 0, `Missing tool call: ${options.toolName}`);
+		const pass = options.validate(targetCall.args);
+		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? `Tool call args validated for ${options.toolName}.` : `Tool call args validation failed for ${options.toolName}.`);
+	};
+}
+/**
+* Creates a rubric assertion driven by teacher-model style scoring.
+*
+* Example:
+* `expectRubric({ id: 'human-like-tone', judge: judgeFn, minScore: 0.8 })`
+*/
+function expectRubric(options) {
+	return async (context) => {
+		const result = await options.judge(context);
+		const minScore = options.minScore ?? .7;
+		const normalizedScore = clampScore(result.score);
+		const pass = normalizedScore >= minScore;
+		return createOutcome(options.id, "judge", pass, normalizedScore, `${result.reason}${result.judgeModel ? ` (judge: ${result.judgeModel})` : ""}`);
+	};
+}
+/**
+* Creates a custom assertion with fully user-defined logic.
+*
+* Example:
+* `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`
+*/
+function expectCustom(options) {
+	return async (context) => {
+		const result = await options.evaluate(context);
+		return createOutcome(options.id, options.scoreKind, result.pass, result.score, result.reason);
+	};
+}
+/**
+* Creates an inverse assertion.
+*
+* Example:
+* `expectNot(expectMustInclude({ id: 'contains-engine-word', keywords: ['bestmove'] }), { id: 'no-engine-word' })`
+*/
+function expectNot(assertion, options) {
+	return async (context) => {
+		const baseOutcome = await assertion(context);
+		return createOutcome(options.id, baseOutcome.scoreKind, !baseOutcome.pass, 1 - baseOutcome.score, `NOT(${baseOutcome.id}): ${baseOutcome.reason}`);
+	};
+}
+/**
+* Executes assertion list and returns all outcomes.
+*
+* Call stack:
+*
+* {@link evaluateAssertions}
+*   -> `assertion(context)`
+*     -> {@link AssertionOutcome}[]
+*/
+async function evaluateAssertions(assertions, context) {
+	const normalizedContext = {
+		state: context.state ?? /* @__PURE__ */ new Map(),
+		structuredOutput: context.structuredOutput,
+		text: context.text,
+		toolCalls: context.toolCalls
+	};
+	const outcomes = [];
+	for (const assertion of assertions) outcomes.push(await assertion(normalizedContext));
+	return outcomes;
+}
+/**
+* Converts assertion outcomes to run-score tuples consumed by aggregation.
+*/
+function toRunScores(outcomes) {
+	return outcomes.map((outcome) => ({
+		kind: outcome.scoreKind,
+		score: outcome.score
+	}));
+}
+/**
+* Returns failing assertion outcomes in original order.
+*/
+function collectFailedAssertions(outcomes) {
+	return outcomes.filter((outcome) => !outcome.pass);
+}
+//#endregion
 export { collectFailedAssertions, evaluateAssertions, expectCustom, expectMustExclude, expectMustInclude, expectNot, expectRegex, expectRubric, expectStructuredOutput, expectToolCallArgs, normalizeMatchText, toRunScores };
+
+//# sourceMappingURL=index.mjs.map

package/dist/core/assertions/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../../../src/core/assertions/index.ts"],"sourcesContent":["import type { RunScore, RunScoreKind } from '../runner/aggregate'\n\n/**\n * Stores mutable evaluation state for stateful assertion flows.\n *\n * Use when:\n * - assertions need to share counters, rolling metrics, or memoized values\n * - a scenario evaluates multiple steps and expects state-aware checks\n */\nexport type AssertionState = Map<string, unknown>\n\n/**\n * Represents one tool call emitted by a model response.\n */\nexport interface ToolCall {\n  /**\n   * Tool name used by the call.\n   */\n  name: string\n  /**\n   * Tool arguments payload.\n   */\n  args: unknown\n}\n\n/**\n * Normalized assertion context for one model output.\n */\nexport interface AssertionContext {\n  /**\n   * Plain text model output used by text assertions.\n   */\n  text: string\n  /**\n   * Optional structured output parsed from the model response.\n   */\n  structuredOutput?: unknown\n  /**\n   * Optional tool calls extracted from the model response.\n   */\n  toolCalls?: readonly ToolCall[]\n  /**\n   * Shared mutable state for stateful assertion measurement.\n   */\n  state: AssertionState\n}\n\n/**\n * Result for one assertion evaluation.\n */\nexport interface AssertionOutcome {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Assertion family emitted as run score kind.\n   */\n  scoreKind: RunScoreKind\n  /**\n   * Whether the assertion passed.\n   */\n  pass: boolean\n  /**\n   * Normalized score in the `0..1` range.\n   */\n  score: number\n  /**\n   * Human-readable reason for logs and reports.\n   */\n  reason: string\n}\n\n/**\n * Async assertion function used by eval scenarios.\n */\nexport type Assertion = (context: AssertionContext) => Promise<AssertionOutcome>\n\n/**\n * Normalizes text for matching.\n *\n * Before: `\"  Hello\\nWorld  \"`\n * After: `\"hello world\"`\n */\nexport function normalizeMatchText(value: string, caseSensitive: boolean): string {\n  const compactedWhitespace = value.trim().replaceAll(/\\s+/g, ' ')\n\n  if (caseSensitive) {\n    return compactedWhitespace\n  }\n\n  return compactedWhitespace.toLowerCase()\n}\n\nfunction clampScore(score: number): number {\n  if (Number.isNaN(score)) {\n    return 0\n  }\n\n  if (score < 0) {\n    return 0\n  }\n\n  if (score > 1) {\n    return 1\n  }\n\n  return score\n}\n\nfunction createOutcome(\n  id: string,\n  scoreKind: RunScoreKind,\n  pass: boolean,\n  score: number,\n  reason: string,\n): AssertionOutcome {\n  return {\n    id,\n    pass,\n    reason,\n    score: clampScore(score),\n    scoreKind,\n  }\n}\n\n/**\n * Options for include-keyword assertions.\n */\nexport interface MustIncludeAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Keywords that must be present.\n   */\n  keywords: readonly string[]\n  /**\n   * Match mode for keywords.\n   *\n   * @default 'all'\n   */\n  mode?: 'all' | 'any'\n  /**\n   * Case-sensitive matching toggle.\n   *\n   * @default false\n   */\n  caseSensitive?: boolean\n}\n\n/**\n * Creates an assertion that requires specific keywords in model text.\n *\n * Example:\n * `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`\n */\nexport function expectMustInclude(options: MustIncludeAssertionOptions): Assertion {\n  return async (context) => {\n    if (options.keywords.length === 0) {\n      return createOutcome(options.id, 'exact', true, 1, 'No required keywords configured.')\n    }\n\n    const caseSensitive = options.caseSensitive ?? false\n    const normalizedText = normalizeMatchText(context.text, caseSensitive)\n    const matches = options.keywords.filter((keyword) => {\n      const normalizedKeyword = normalizeMatchText(keyword, caseSensitive)\n      return normalizedText.includes(normalizedKeyword)\n    })\n\n    const mode = options.mode ?? 'all'\n    const pass = mode === 'all'\n      ? matches.length === options.keywords.length\n      : matches.length > 0\n\n    const score = options.keywords.length === 0 ? 1 : matches.length / options.keywords.length\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      score,\n      pass\n        ? `Matched ${matches.length}/${options.keywords.length} required keywords.`\n        : `Matched ${matches.length}/${options.keywords.length} required keywords.`,\n    )\n  }\n}\n\n/**\n * Options for exclude-keyword assertions.\n */\nexport interface MustExcludeAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Keywords that must not appear.\n   */\n  keywords: readonly string[]\n  /**\n   * Case-sensitive matching toggle.\n   *\n   * @default false\n   */\n  caseSensitive?: boolean\n}\n\n/**\n * Creates an assertion that forbids specific keywords.\n *\n * Example:\n * `expectMustExclude({ id: 'no-engine-dump', keywords: ['bestmove', 'ponder'] })`\n */\nexport function expectMustExclude(options: MustExcludeAssertionOptions): Assertion {\n  return async (context) => {\n    if (options.keywords.length === 0) {\n      return createOutcome(options.id, 'exact', true, 1, 'No excluded keywords configured.')\n    }\n\n    const caseSensitive = options.caseSensitive ?? false\n    const normalizedText = normalizeMatchText(context.text, caseSensitive)\n    const forbiddenMatches = options.keywords.filter((keyword) => {\n      const normalizedKeyword = normalizeMatchText(keyword, caseSensitive)\n      return normalizedText.includes(normalizedKeyword)\n    })\n\n    const pass = forbiddenMatches.length === 0\n    const score = pass ? 1 : 0\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      score,\n      pass\n        ? 'No forbidden keywords found.'\n        : `Forbidden keywords found: ${forbiddenMatches.join(', ')}`,\n    )\n  }\n}\n\n/**\n * Options for regular-expression assertions.\n */\nexport interface RegexAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Pattern to apply to model text.\n   */\n  pattern: RegExp\n}\n\n/**\n * Creates an assertion based on a regular expression.\n *\n * Example:\n * `expectRegex({ id: 'starts-with-act', pattern: /^<\\|ACT:/ })`\n */\nexport function expectRegex(options: RegexAssertionOptions): Assertion {\n  return async (context) => {\n    const pass = options.pattern.test(context.text)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? 'Regex matched response text.' : `Regex did not match: ${options.pattern}`,\n    )\n  }\n}\n\n/**\n * Options for structured-output assertions.\n */\nexport interface StructuredOutputAssertionOptions<TValue> {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Runtime validator for structured output.\n   */\n  validate: (value: unknown) => value is TValue\n  /**\n   * Optional failure reason.\n   */\n  failureReason?: string\n}\n\n/**\n * Creates an assertion for structured model output.\n *\n * Example:\n * `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`\n */\nexport function expectStructuredOutput<TValue>(options: StructuredOutputAssertionOptions<TValue>): Assertion {\n  return async (context) => {\n    const pass = options.validate(context.structuredOutput)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? 'Structured output matched validator.' : (options.failureReason ?? 'Structured output validation failed.'),\n    )\n  }\n}\n\n/**\n * Options for tool-call argument assertions.\n */\nexport interface ToolCallArgsAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Tool name to inspect.\n   */\n  toolName: string\n  /**\n   * Runtime validator for tool arguments.\n   */\n  validate: (args: unknown) => boolean\n}\n\n/**\n * Creates an assertion for validating tool-call arguments.\n *\n * Example:\n * `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`\n */\nexport function expectToolCallArgs(options: ToolCallArgsAssertionOptions): Assertion {\n  return async (context) => {\n    const targetCall = (context.toolCalls ?? []).find(call => call.name === options.toolName)\n\n    if (targetCall == null) {\n      return createOutcome(options.id, 'exact', false, 0, `Missing tool call: ${options.toolName}`)\n    }\n\n    const pass = options.validate(targetCall.args)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? `Tool call args validated for ${options.toolName}.` : `Tool call args validation failed for ${options.toolName}.`,\n    )\n  }\n}\n\n/**\n * Rubric judge result returned by teacher-model or rubric logic.\n */\nexport interface RubricJudgeResult {\n  /**\n   * Normalized score in the `0..1` range.\n   */\n  score: number\n  /**\n   * Judge explanation text.\n   */\n  reason: string\n  /**\n   * Optional judge model id.\n   */\n  judgeModel?: string\n}\n\n/**\n * Options for rubric assertions.\n */\nexport interface RubricAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Async rubric judge callback.\n   */\n  judge: (context: AssertionContext) => Promise<RubricJudgeResult>\n  /**\n   * Minimum passing score.\n   *\n   * @default 0.7\n   */\n  minScore?: number\n}\n\n/**\n * Creates a rubric assertion driven by teacher-model style scoring.\n *\n * Example:\n * `expectRubric({ id: 'human-like-tone', judge: judgeFn, minScore: 0.8 })`\n */\nexport function expectRubric(options: RubricAssertionOptions): Assertion {\n  return async (context) => {\n    const result = await options.judge(context)\n    const minScore = options.minScore ?? 0.7\n    const normalizedScore = clampScore(result.score)\n    const pass = normalizedScore >= minScore\n\n    return createOutcome(\n      options.id,\n      'judge',\n      pass,\n      normalizedScore,\n      `${result.reason}${result.judgeModel ? ` (judge: ${result.judgeModel})` : ''}`,\n    )\n  }\n}\n\n/**\n * Options for custom assertions.\n */\nexport interface CustomAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Score family emitted by this custom assertion.\n   */\n  scoreKind: RunScoreKind\n  /**\n   * Custom evaluator callback.\n   */\n  evaluate: (context: AssertionContext) => Promise<{ pass: boolean, reason: string, score: number }> | { pass: boolean, reason: string, score: number }\n}\n\n/**\n * Creates a custom assertion with fully user-defined logic.\n *\n * Example:\n * `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`\n */\nexport function expectCustom(options: CustomAssertionOptions): Assertion {\n  return async (context) => {\n    const result = await options.evaluate(context)\n\n    return createOutcome(options.id, options.scoreKind, result.pass, result.score, result.reason)\n  }\n}\n\n/**\n * Creates an inverse assertion.\n *\n * Example:\n * `expectNot(expectMustInclude({ id: 'contains-engine-word', keywords: ['bestmove'] }), { id: 'no-engine-word' })`\n */\nexport function expectNot(assertion: Assertion, options: { id: string }): Assertion {\n  return async (context) => {\n    const baseOutcome = await assertion(context)\n\n    return createOutcome(\n      options.id,\n      baseOutcome.scoreKind,\n      !baseOutcome.pass,\n      1 - baseOutcome.score,\n      `NOT(${baseOutcome.id}): ${baseOutcome.reason}`,\n    )\n  }\n}\n\n/**\n * Executes assertion list and returns all outcomes.\n *\n * Call stack:\n *\n * {@link evaluateAssertions}\n *   -> `assertion(context)`\n *     -> {@link AssertionOutcome}[]\n */\nexport async function evaluateAssertions(\n  assertions: readonly Assertion[],\n  context: Omit<AssertionContext, 'state'> & { state?: AssertionState },\n): Promise<AssertionOutcome[]> {\n  const state = context.state ?? new Map<string, unknown>()\n  const normalizedContext: AssertionContext = {\n    state,\n    structuredOutput: context.structuredOutput,\n    text: context.text,\n    toolCalls: context.toolCalls,\n  }\n\n  const outcomes: AssertionOutcome[] = []\n\n  for (const assertion of assertions) {\n    outcomes.push(await assertion(normalizedContext))\n  }\n\n  return outcomes\n}\n\n/**\n * Converts assertion outcomes to run-score tuples consumed by aggregation.\n */\nexport function toRunScores(outcomes: readonly AssertionOutcome[]): RunScore[] {\n  return outcomes.map(outcome => ({\n    kind: outcome.scoreKind,\n    score: outcome.score,\n  }))\n}\n\n/**\n * Returns failing assertion outcomes in original order.\n */\nexport function collectFailedAssertions(outcomes: readonly AssertionOutcome[]): AssertionOutcome[] {\n  return outcomes.filter(outcome => !outcome.pass)\n}\n"],"mappings":";;;;;;;AAoFA,SAAgB,mBAAmB,OAAe,eAAgC;CAChF,MAAM,sBAAsB,MAAM,MAAM,CAAC,WAAW,QAAQ,IAAI;AAEhE,KAAI,cACF,QAAO;AAGT,QAAO,oBAAoB,aAAa;;AAG1C,SAAS,WAAW,OAAuB;AACzC,KAAI,OAAO,MAAM,MAAM,CACrB,QAAO;AAGT,KAAI,QAAQ,EACV,QAAO;AAGT,KAAI,QAAQ,EACV,QAAO;AAGT,QAAO;;AAGT,SAAS,cACP,IACA,WACA,MACA,OACA,QACkB;AAClB,QAAO;EACL;EACA;EACA;EACA,OAAO,WAAW,MAAM;EACxB;EACD;;;;;;;;AAmCH,SAAgB,kBAAkB,SAAiD;AACjF,QAAO,OAAO,YAAY;AACxB,MAAI,QAAQ,SAAS,WAAW,EAC9B,QAAO,cAAc,QAAQ,IAAI,SAAS,MAAM,GAAG,mCAAmC;EAGxF,MAAM,gBAAgB,QAAQ,iBAAiB;EAC/C,MAAM,iBAAiB,mBAAmB,QAAQ,MAAM,cAAc;EACtE,MAAM,UAAU,QAAQ,SAAS,QAAQ,YAAY;GACnD,MAAM,oBAAoB,mBAAmB,SAAS,cAAc;AACpE,UAAO,eAAe,SAAS,kBAAkB;IACjD;EAGF,MAAM,QADO,QAAQ,QAAQ,WACP,QAClB,QAAQ,WAAW,QAAQ,SAAS,SACpC,QAAQ,SAAS;EAErB,MAAM,QAAQ,QAAQ,SAAS,WAAW,IAAI,IAAI,QAAQ,SAAS,QAAQ,SAAS;AAEpF,SAAO,cACL,QAAQ,IACR,SACA,MACA,OACA,OACI,WAAW,QAAQ,OAAO,GAAG,QAAQ,SAAS,OAAO,uBACrD,WAAW,QAAQ,OAAO,GAAG,QAAQ,SAAS,OAAO,qBAC1D;;;;;;;;;AA8BL,SAAgB,kBAAkB,SAAiD;AACjF,QAAO,OAAO,YAAY;AACxB,MAAI,QAAQ,SAAS,WAAW,EAC9B,QAAO,cAAc,QAAQ,IAAI,SAAS,MAAM,GAAG,mCAAmC;EAGxF,MAAM,gBAAgB,QAAQ,iBAAiB;EAC/C,MAAM,iBAAiB,mBAAmB,QAAQ,MAAM,cAAc;EACtE,MAAM,mBAAmB,QAAQ,SAAS,QAAQ,YAAY;GAC5D,MAAM,oBAAoB,mBAAmB,SAAS,cAAc;AACpE,UAAO,eAAe,SAAS,kBAAkB;IACjD;EAEF,MAAM,OAAO,iBAAiB,WAAW;EACzC,MAAM,QAAQ,OAAO,IAAI;AAEzB,SAAO,cACL,QAAQ,IACR,SACA,MACA,OACA,OACI,iCACA,6BAA6B,iBAAiB,KAAK,KAAK,GAC7D;;;;;;;;;AAwBL,SAAgB,YAAY,SAA2C;AACrE,QAAO,OAAO,YAAY;EACxB,MAAM,OAAO,QAAQ,QAAQ,KAAK,QAAQ,KAAK;AAE/C,SAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,iCAAiC,wBAAwB,QAAQ,UACzE;;;;;;;;;AA4BL,SAAgB,uBAA+B,SAA8D;AAC3G,QAAO,OAAO,YAAY;EACxB,MAAM,OAAO,QAAQ,SAAS,QAAQ,iBAAiB;AAEvD,SAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,yCAA0C,QAAQ,iBAAiB,uCAC3E;;;;;;;;;AA4BL,SAAgB,mBAAmB,SAAkD;AACnF,QAAO,OAAO,YAAY;EACxB,MAAM,cAAc,QAAQ,aAAa,EAAE,EAAE,MAAK,SAAQ,KAAK,SAAS,QAAQ,SAAS;AAEzF,MAAI,cAAc,KAChB,QAAO,cAAc,QAAQ,IAAI,SAAS,OAAO,GAAG,sBAAsB,QAAQ,WAAW;EAG/F,MAAM,OAAO,QAAQ,SAAS,WAAW,KAAK;AAE9C,SAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,gCAAgC,QAAQ,SAAS,KAAK,wCAAwC,QAAQ,SAAS,GACvH;;;;;;;;;AAgDL,SAAgB,aAAa,SAA4C;AACvE,QAAO,OAAO,YAAY;EACxB,MAAM,SAAS,MAAM,QAAQ,MAAM,QAAQ;EAC3C,MAAM,WAAW,QAAQ,YAAY;EACrC,MAAM,kBAAkB,WAAW,OAAO,MAAM;EAChD,MAAM,OAAO,mBAAmB;AAEhC,SAAO,cACL,QAAQ,IACR,SACA,MACA,iBACA,GAAG,OAAO,SAAS,OAAO,aAAa,YAAY,OAAO,WAAW,KAAK,KAC3E;;;;;;;;;AA4BL,SAAgB,aAAa,SAA4C;AACvE,QAAO,OAAO,YAAY;EACxB,MAAM,SAAS,MAAM,QAAQ,SAAS,QAAQ;AAE9C,SAAO,cAAc,QAAQ,IAAI,QAAQ,WAAW,OAAO,MAAM,OAAO,OAAO,OAAO,OAAO;;;;;;;;;AAUjG,SAAgB,UAAU,WAAsB,SAAoC;AAClF,QAAO,OAAO,YAAY;EACxB,MAAM,cAAc,MAAM,UAAU,QAAQ;AAE5C,SAAO,cACL,QAAQ,IACR,YAAY,WACZ,CAAC,YAAY,MACb,IAAI,YAAY,OAChB,OAAO,YAAY,GAAG,KAAK,YAAY,SACxC;;;;;;;;;;;;AAaL,eAAsB,mBACpB,YACA,SAC6B;CAE7B,MAAM,oBAAsC;EAC1C,OAFY,QAAQ,yBAAS,IAAI,KAAsB;EAGvD,kBAAkB,QAAQ;EAC1B,MAAM,QAAQ;EACd,WAAW,QAAQ;EACpB;CAED,MAAM,WAA+B,EAAE;AAEvC,MAAK,MAAM,aAAa,WACtB,UAAS,KAAK,MAAM,UAAU,kBAAkB,CAAC;AAGnD,QAAO;;;;;AAMT,SAAgB,YAAY,UAAmD;AAC7E,QAAO,SAAS,KAAI,aAAY;EAC9B,MAAM,QAAQ;EACd,OAAO,QAAQ;EAChB,EAAE;;;;;AAML,SAAgB,wBAAwB,UAA2D;AACjG,QAAO,SAAS,QAAO,YAAW,CAAC,QAAQ,KAAK"}

package/dist/core/inference-executors/index.d.mts

@@ -1,4 +1,4 @@
-import { a as requiredEnvFrom, i as envFrom, n as EnvValueType, r as RequiredEnvFromOptions, t as EnvFromOptions } from "../../env-DtpjACOW.mjs";
+import { a as requiredEnvFrom, i as envFrom, n as EnvValueType, r as RequiredEnvFromOptions, t as EnvFromOptions } from "../../env-BeHv_5mo.mjs";
 import { createOpenAI } from "@xsai-ext/providers/create";
 
 //#region src/core/inference-executors/retry-policy.d.ts

package/dist/core/inference-executors/index.mjs

@@ -1,4 +1,4 @@
-import { n as requiredEnvFrom, t as envFrom } from "../../env-C7X81PWa.mjs";
+import { n as requiredEnvFrom, t as envFrom } from "../../env--94B0UtW.mjs";
 import process from "node:process";
 import { errorMessageFrom, errorNameFrom, sleep } from "@moeru/std";
 import { createOpenAI } from "@xsai-ext/providers/create";

package/dist/core/processors/results/index.d.mts

@@ -1,4 +1,4 @@
-import { B as AggregatedRunResults } from "../../../index-BDMEAmf2.mjs";
+import { H as AggregatedRunResults } from "../../../index-DBZKkpBe.mjs";
 
 //#region src/core/processors/results/policies/hybrid-threshold.d.ts
 /**

package/dist/core/runner/index.d.mts

@@ -1,2 +1,3 @@
-import { $ as ScheduledTaskMatrixMeta, A as TaskModelSelectionOptions, B as AggregatedRunResults, C as RunScheduledTasksOptions, D as runScheduledTasks, E as ScheduledTaskExecutor, F as collectEvalEntries, G as aggregateRunResults, H as RunResult, I as CreateVievalRunnerRuntimeContextOptions, J as RunnerMatrixDefinition, K as CreateRunnerScheduleOptions, L as RunnerRuntimeContext, O as CreateTaskExecutionContextOptions, P as asProjectRelativePath, Q as ScheduledTaskMatrix, R as createRunnerRuntimeContext, T as RunnerTaskState, U as RunScore, V as AggregatedRunSummary, W as RunScoreKind, X as RunnerMatrixSelection, Y as RunnerMatrixInput, Z as ScheduledTask, at as CacheFileOptions, et as createRunnerSchedule, it as CacheFileHandle, j as createTaskExecutionContext, k as TaskExecutionContext, nt as createFilesystemTaskCacheRuntime, ot as CacheNamespace, q as InferenceExecutor, rt as normalizeCacheFilePathSegments, st as TaskCacheRuntime, tt as CreateFilesystemTaskCacheRuntimeOptions, w as RunnerExecutionError, z as AggregatedProviderSummary } from "../../index-BDMEAmf2.mjs";
-export { AggregatedProviderSummary, AggregatedRunResults, AggregatedRunSummary, CacheFileHandle, CacheFileOptions, CacheNamespace, CreateFilesystemTaskCacheRuntimeOptions, CreateRunnerScheduleOptions, CreateTaskExecutionContextOptions, CreateVievalRunnerRuntimeContextOptions, InferenceExecutor, RunResult, RunScheduledTasksOptions, RunScore, RunScoreKind, RunnerExecutionError, RunnerMatrixDefinition, RunnerMatrixInput, RunnerMatrixSelection, RunnerRuntimeContext, RunnerTaskState, ScheduledTask, ScheduledTaskExecutor, ScheduledTaskMatrix, ScheduledTaskMatrixMeta, TaskCacheRuntime, TaskExecutionContext, TaskModelSelectionOptions, aggregateRunResults, asProjectRelativePath, collectEvalEntries, createFilesystemTaskCacheRuntime, createRunnerRuntimeContext, createRunnerSchedule, createTaskExecutionContext, normalizeCacheFilePathSegments, runScheduledTasks };
+import { $ as ScheduledTask, A as CreateTaskExecutionContextOptions, B as createRunnerRuntimeContext, D as RunnerTaskState, E as RunnerExecutionError, G as RunScore, H as AggregatedRunResults, I as asProjectRelativePath, J as CreateRunnerScheduleOptions, K as RunScoreKind, L as collectEvalEntries, M as TaskModelSelectionOptions, N as createTaskExecutionContext, O as ScheduledTaskExecutor, Q as RunnerMatrixSelection, R as CreateVievalRunnerRuntimeContextOptions, T as RunScheduledTasksOptions, U as AggregatedRunSummary, V as AggregatedProviderSummary, W as RunResult, X as RunnerMatrixDefinition, Y as InferenceExecutor, Z as RunnerMatrixInput, at as normalizeCacheFilePathSegments, ct as CacheNamespace, et as ScheduledTaskMatrix, it as createFilesystemTaskCacheRuntime, j as TaskExecutionContext, k as runScheduledTasks, lt as TaskCacheRuntime, nt as createRunnerSchedule, ot as CacheFileHandle, q as aggregateRunResults, rt as CreateFilesystemTaskCacheRuntimeOptions, st as CacheFileOptions, tt as ScheduledTaskMatrixMeta, z as RunnerRuntimeContext } from "../../index-DBZKkpBe.mjs";
+import { a as SchedulerMiddleware, c as SchedulerScopeContext, i as SchedulerConcurrencyConfig, n as getActiveScopes, o as SchedulerRuntime, r as CreateSchedulerRuntimeOptions, s as SchedulerScope, t as createSchedulerRuntime } from "../../index-fakXoZEe.mjs";
+export { AggregatedProviderSummary, AggregatedRunResults, AggregatedRunSummary, CacheFileHandle, CacheFileOptions, CacheNamespace, CreateFilesystemTaskCacheRuntimeOptions, CreateRunnerScheduleOptions, CreateSchedulerRuntimeOptions, CreateTaskExecutionContextOptions, CreateVievalRunnerRuntimeContextOptions, InferenceExecutor, RunResult, RunScheduledTasksOptions, RunScore, RunScoreKind, RunnerExecutionError, RunnerMatrixDefinition, RunnerMatrixInput, RunnerMatrixSelection, RunnerRuntimeContext, RunnerTaskState, ScheduledTask, ScheduledTaskExecutor, ScheduledTaskMatrix, ScheduledTaskMatrixMeta, SchedulerConcurrencyConfig, SchedulerMiddleware, SchedulerRuntime, SchedulerScope, SchedulerScopeContext, TaskCacheRuntime, TaskExecutionContext, TaskModelSelectionOptions, aggregateRunResults, asProjectRelativePath, collectEvalEntries, createFilesystemTaskCacheRuntime, createRunnerRuntimeContext, createRunnerSchedule, createSchedulerRuntime, createTaskExecutionContext, getActiveScopes, normalizeCacheFilePathSegments, runScheduledTasks };