vieval 0.0.11 → 0.0.12

package/dist/core/assertions/index.mjs

@@ -1,48 +1,40 @@
 //#region src/core/assertions/index.ts
 /**
-* Normalizes text for matching.
-*
-* Before: `"  Hello\nWorld  "`
-* After: `"hello world"`
+* Returns failing assertion outcomes in original order.
 */
-function normalizeMatchText(value, caseSensitive) {
-	const compactedWhitespace = value.trim().replaceAll(/\s+/g, " ");
-	if (caseSensitive) return compactedWhitespace;
-	return compactedWhitespace.toLowerCase();
-}
-function clampScore(score) {
-	if (Number.isNaN(score)) return 0;
-	if (score < 0) return 0;
-	if (score > 1) return 1;
-	return score;
+function collectFailedAssertions(outcomes) {
+	return outcomes.filter((outcome) => !outcome.pass);
 }
-function createOutcome(id, scoreKind, pass, score, reason) {
-	return {
-		id,
-		pass,
-		reason,
-		score: clampScore(score),
-		scoreKind
+/**
+* Executes assertion list and returns all outcomes.
+*
+* Call stack:
+*
+* {@link evaluateAssertions}
+*   -> `assertion(context)`
+*     -> {@link AssertionOutcome}[]
+*/
+async function evaluateAssertions(assertions, context) {
+	const normalizedContext = {
+		state: context.state ?? /* @__PURE__ */ new Map(),
+		structuredOutput: context.structuredOutput,
+		text: context.text,
+		toolCalls: context.toolCalls
 	};
+	const outcomes = [];
+	for (const assertion of assertions) outcomes.push(await assertion(normalizedContext));
+	return outcomes;
 }
 /**
-* Creates an assertion that requires specific keywords in model text.
+* Creates a custom assertion with fully user-defined logic.
 *
 * Example:
-* `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`
+* `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`
 */
-function expectMustInclude(options) {
+function expectCustom(options) {
 	return async (context) => {
-		if (options.keywords.length === 0) return createOutcome(options.id, "exact", true, 1, "No required keywords configured.");
-		const caseSensitive = options.caseSensitive ?? false;
-		const normalizedText = normalizeMatchText(context.text, caseSensitive);
-		const matches = options.keywords.filter((keyword) => {
-			const normalizedKeyword = normalizeMatchText(keyword, caseSensitive);
-			return normalizedText.includes(normalizedKeyword);
-		});
-		const pass = (options.mode ?? "all") === "all" ? matches.length === options.keywords.length : matches.length > 0;
-		const score = options.keywords.length === 0 ? 1 : matches.length / options.keywords.length;
-		return createOutcome(options.id, "exact", pass, score, pass ? `Matched ${matches.length}/${options.keywords.length} required keywords.` : `Matched ${matches.length}/${options.keywords.length} required keywords.`);
+		const result = await options.evaluate(context);
+		return createOutcome(options.id, options.scoreKind, result.pass, result.score, result.reason);
 	};
 }
 /**
@@ -66,41 +58,47 @@ function expectMustExclude(options) {
 	};
 }
 /**
-* Creates an assertion based on a regular expression.
+* Creates an assertion that requires specific keywords in model text.
 *
 * Example:
-* `expectRegex({ id: 'starts-with-act', pattern: /^<\|ACT:/ })`
+* `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`
 */
-function expectRegex(options) {
+function expectMustInclude(options) {
 	return async (context) => {
-		const pass = options.pattern.test(context.text);
-		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? "Regex matched response text." : `Regex did not match: ${options.pattern}`);
+		if (options.keywords.length === 0) return createOutcome(options.id, "exact", true, 1, "No required keywords configured.");
+		const caseSensitive = options.caseSensitive ?? false;
+		const normalizedText = normalizeMatchText(context.text, caseSensitive);
+		const matches = options.keywords.filter((keyword) => {
+			const normalizedKeyword = normalizeMatchText(keyword, caseSensitive);
+			return normalizedText.includes(normalizedKeyword);
+		});
+		const pass = (options.mode ?? "all") === "all" ? matches.length === options.keywords.length : matches.length > 0;
+		const score = options.keywords.length === 0 ? 1 : matches.length / options.keywords.length;
+		return createOutcome(options.id, "exact", pass, score, pass ? `Matched ${matches.length}/${options.keywords.length} required keywords.` : `Matched ${matches.length}/${options.keywords.length} required keywords.`);
 	};
 }
 /**
-* Creates an assertion for structured model output.
+* Creates an inverse assertion.
 *
 * Example:
-* `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`
+* `expectNot(expectMustInclude({ id: 'contains-engine-word', keywords: ['bestmove'] }), { id: 'no-engine-word' })`
 */
-function expectStructuredOutput(options) {
+function expectNot(assertion, options) {
 	return async (context) => {
-		const pass = options.validate(context.structuredOutput);
-		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? "Structured output matched validator." : options.failureReason ?? "Structured output validation failed.");
+		const baseOutcome = await assertion(context);
+		return createOutcome(options.id, baseOutcome.scoreKind, !baseOutcome.pass, 1 - baseOutcome.score, `NOT(${baseOutcome.id}): ${baseOutcome.reason}`);
 	};
 }
 /**
-* Creates an assertion for validating tool-call arguments.
+* Creates an assertion based on a regular expression.
 *
 * Example:
-* `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`
+* `expectRegex({ id: 'starts-with-act', pattern: /^<\|ACT:/ })`
 */
-function expectToolCallArgs(options) {
+function expectRegex(options) {
 	return async (context) => {
-		const targetCall = (context.toolCalls ?? []).find((call) => call.name === options.toolName);
-		if (targetCall == null) return createOutcome(options.id, "exact", false, 0, `Missing tool call: ${options.toolName}`);
-		const pass = options.validate(targetCall.args);
-		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? `Tool call args validated for ${options.toolName}.` : `Tool call args validation failed for ${options.toolName}.`);
+		const pass = options.pattern.test(context.text);
+		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? "Regex matched response text." : `Regex did not match: ${options.pattern}`);
 	};
 }
 /**
@@ -119,48 +117,41 @@ function expectRubric(options) {
 	};
 }
 /**
-* Creates a custom assertion with fully user-defined logic.
+* Creates an assertion for structured model output.
 *
 * Example:
-* `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`
+* `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`
 */
-function expectCustom(options) {
+function expectStructuredOutput(options) {
 	return async (context) => {
-		const result = await options.evaluate(context);
-		return createOutcome(options.id, options.scoreKind, result.pass, result.score, result.reason);
+		const pass = options.validate(context.structuredOutput);
+		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? "Structured output matched validator." : options.failureReason ?? "Structured output validation failed.");
 	};
 }
 /**
-* Creates an inverse assertion.
+* Creates an assertion for validating tool-call arguments.
 *
 * Example:
-* `expectNot(expectMustInclude({ id: 'contains-engine-word', keywords: ['bestmove'] }), { id: 'no-engine-word' })`
+* `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`
 */
-function expectNot(assertion, options) {
+function expectToolCallArgs(options) {
 	return async (context) => {
-		const baseOutcome = await assertion(context);
-		return createOutcome(options.id, baseOutcome.scoreKind, !baseOutcome.pass, 1 - baseOutcome.score, `NOT(${baseOutcome.id}): ${baseOutcome.reason}`);
+		const targetCall = (context.toolCalls ?? []).find((call) => call.name === options.toolName);
+		if (targetCall == null) return createOutcome(options.id, "exact", false, 0, `Missing tool call: ${options.toolName}`);
+		const pass = options.validate(targetCall.args);
+		return createOutcome(options.id, "exact", pass, pass ? 1 : 0, pass ? `Tool call args validated for ${options.toolName}.` : `Tool call args validation failed for ${options.toolName}.`);
 	};
 }
 /**
-* Executes assertion list and returns all outcomes.
-*
-* Call stack:
+* Normalizes text for matching.
 *
-* {@link evaluateAssertions}
-*   -> `assertion(context)`
-*     -> {@link AssertionOutcome}[]
+* Before: `"  Hello\nWorld  "`
+* After: `"hello world"`
 */
-async function evaluateAssertions(assertions, context) {
-	const normalizedContext = {
-		state: context.state ?? /* @__PURE__ */ new Map(),
-		structuredOutput: context.structuredOutput,
-		text: context.text,
-		toolCalls: context.toolCalls
-	};
-	const outcomes = [];
-	for (const assertion of assertions) outcomes.push(await assertion(normalizedContext));
-	return outcomes;
+function normalizeMatchText(value, caseSensitive) {
+	const compactedWhitespace = value.trim().replaceAll(/\s+/g, " ");
+	if (caseSensitive) return compactedWhitespace;
+	return compactedWhitespace.toLowerCase();
 }
 /**
 * Converts assertion outcomes to run-score tuples consumed by aggregation.
@@ -171,11 +162,20 @@ function toRunScores(outcomes) {
 		score: outcome.score
 	}));
 }
-/**
-* Returns failing assertion outcomes in original order.
-*/
-function collectFailedAssertions(outcomes) {
-	return outcomes.filter((outcome) => !outcome.pass);
+function clampScore(score) {
+	if (Number.isNaN(score)) return 0;
+	if (score < 0) return 0;
+	if (score > 1) return 1;
+	return score;
+}
+function createOutcome(id, scoreKind, pass, score, reason) {
+	return {
+		id,
+		pass,
+		reason,
+		score: clampScore(score),
+		scoreKind
+	};
 }
 //#endregion
 export { collectFailedAssertions, evaluateAssertions, expectCustom, expectMustExclude, expectMustInclude, expectNot, expectRegex, expectRubric, expectStructuredOutput, expectToolCallArgs, normalizeMatchText, toRunScores };

package/dist/core/assertions/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","names":[],"sources":["../../../src/core/assertions/index.ts"],"sourcesContent":["import type { RunScore, RunScoreKind } from '../runner/aggregate'\n\n/**\n * Stores mutable evaluation state for stateful assertion flows.\n *\n * Use when:\n * - assertions need to share counters, rolling metrics, or memoized values\n * - a scenario evaluates multiple steps and expects state-aware checks\n */\nexport type AssertionState = Map<string, unknown>\n\n/**\n * Represents one tool call emitted by a model response.\n */\nexport interface ToolCall {\n  /**\n   * Tool name used by the call.\n   */\n  name: string\n  /**\n   * Tool arguments payload.\n   */\n  args: unknown\n}\n\n/**\n * Normalized assertion context for one model output.\n */\nexport interface AssertionContext {\n  /**\n   * Plain text model output used by text assertions.\n   */\n  text: string\n  /**\n   * Optional structured output parsed from the model response.\n   */\n  structuredOutput?: unknown\n  /**\n   * Optional tool calls extracted from the model response.\n   */\n  toolCalls?: readonly ToolCall[]\n  /**\n   * Shared mutable state for stateful assertion measurement.\n   */\n  state: AssertionState\n}\n\n/**\n * Result for one assertion evaluation.\n */\nexport interface AssertionOutcome {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Assertion family emitted as run score kind.\n   */\n  scoreKind: RunScoreKind\n  /**\n   * Whether the assertion passed.\n   */\n  pass: boolean\n  /**\n   * Normalized score in the `0..1` range.\n   */\n  score: number\n  /**\n   * Human-readable reason for logs and reports.\n   */\n  reason: string\n}\n\n/**\n * Async assertion function used by eval scenarios.\n */\nexport type Assertion = (context: AssertionContext) => Promise<AssertionOutcome>\n\n/**\n * Normalizes text for matching.\n *\n * Before: `\"  Hello\\nWorld  \"`\n * After: `\"hello world\"`\n */\nexport function normalizeMatchText(value: string, caseSensitive: boolean): string {\n  const compactedWhitespace = value.trim().replaceAll(/\\s+/g, ' ')\n\n  if (caseSensitive) {\n    return compactedWhitespace\n  }\n\n  return compactedWhitespace.toLowerCase()\n}\n\nfunction clampScore(score: number): number {\n  if (Number.isNaN(score)) {\n    return 0\n  }\n\n  if (score < 0) {\n    return 0\n  }\n\n  if (score > 1) {\n    return 1\n  }\n\n  return score\n}\n\nfunction createOutcome(\n  id: string,\n  scoreKind: RunScoreKind,\n  pass: boolean,\n  score: number,\n  reason: string,\n): AssertionOutcome {\n  return {\n    id,\n    pass,\n    reason,\n    score: clampScore(score),\n    scoreKind,\n  }\n}\n\n/**\n * Options for include-keyword assertions.\n */\nexport interface MustIncludeAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Keywords that must be present.\n   */\n  keywords: readonly string[]\n  /**\n   * Match mode for keywords.\n   *\n   * @default 'all'\n   */\n  mode?: 'all' | 'any'\n  /**\n   * Case-sensitive matching toggle.\n   *\n   * @default false\n   */\n  caseSensitive?: boolean\n}\n\n/**\n * Creates an assertion that requires specific keywords in model text.\n *\n * Example:\n * `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`\n */\nexport function expectMustInclude(options: MustIncludeAssertionOptions): Assertion {\n  return async (context) => {\n    if (options.keywords.length === 0) {\n      return createOutcome(options.id, 'exact', true, 1, 'No required keywords configured.')\n    }\n\n    const caseSensitive = options.caseSensitive ?? false\n    const normalizedText = normalizeMatchText(context.text, caseSensitive)\n    const matches = options.keywords.filter((keyword) => {\n      const normalizedKeyword = normalizeMatchText(keyword, caseSensitive)\n      return normalizedText.includes(normalizedKeyword)\n    })\n\n    const mode = options.mode ?? 'all'\n    const pass = mode === 'all'\n      ? matches.length === options.keywords.length\n      : matches.length > 0\n\n    const score = options.keywords.length === 0 ? 1 : matches.length / options.keywords.length\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      score,\n      pass\n        ? `Matched ${matches.length}/${options.keywords.length} required keywords.`\n        : `Matched ${matches.length}/${options.keywords.length} required keywords.`,\n    )\n  }\n}\n\n/**\n * Options for exclude-keyword assertions.\n */\nexport interface MustExcludeAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Keywords that must not appear.\n   */\n  keywords: readonly string[]\n  /**\n   * Case-sensitive matching toggle.\n   *\n   * @default false\n   */\n  caseSensitive?: boolean\n}\n\n/**\n * Creates an assertion that forbids specific keywords.\n *\n * Example:\n * `expectMustExclude({ id: 'no-engine-dump', keywords: ['bestmove', 'ponder'] })`\n */\nexport function expectMustExclude(options: MustExcludeAssertionOptions): Assertion {\n  return async (context) => {\n    if (options.keywords.length === 0) {\n      return createOutcome(options.id, 'exact', true, 1, 'No excluded keywords configured.')\n    }\n\n    const caseSensitive = options.caseSensitive ?? false\n    const normalizedText = normalizeMatchText(context.text, caseSensitive)\n    const forbiddenMatches = options.keywords.filter((keyword) => {\n      const normalizedKeyword = normalizeMatchText(keyword, caseSensitive)\n      return normalizedText.includes(normalizedKeyword)\n    })\n\n    const pass = forbiddenMatches.length === 0\n    const score = pass ? 1 : 0\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      score,\n      pass\n        ? 'No forbidden keywords found.'\n        : `Forbidden keywords found: ${forbiddenMatches.join(', ')}`,\n    )\n  }\n}\n\n/**\n * Options for regular-expression assertions.\n */\nexport interface RegexAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Pattern to apply to model text.\n   */\n  pattern: RegExp\n}\n\n/**\n * Creates an assertion based on a regular expression.\n *\n * Example:\n * `expectRegex({ id: 'starts-with-act', pattern: /^<\\|ACT:/ })`\n */\nexport function expectRegex(options: RegexAssertionOptions): Assertion {\n  return async (context) => {\n    const pass = options.pattern.test(context.text)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? 'Regex matched response text.' : `Regex did not match: ${options.pattern}`,\n    )\n  }\n}\n\n/**\n * Options for structured-output assertions.\n */\nexport interface StructuredOutputAssertionOptions<TValue> {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Runtime validator for structured output.\n   */\n  validate: (value: unknown) => value is TValue\n  /**\n   * Optional failure reason.\n   */\n  failureReason?: string\n}\n\n/**\n * Creates an assertion for structured model output.\n *\n * Example:\n * `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`\n */\nexport function expectStructuredOutput<TValue>(options: StructuredOutputAssertionOptions<TValue>): Assertion {\n  return async (context) => {\n    const pass = options.validate(context.structuredOutput)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? 'Structured output matched validator.' : (options.failureReason ?? 'Structured output validation failed.'),\n    )\n  }\n}\n\n/**\n * Options for tool-call argument assertions.\n */\nexport interface ToolCallArgsAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Tool name to inspect.\n   */\n  toolName: string\n  /**\n   * Runtime validator for tool arguments.\n   */\n  validate: (args: unknown) => boolean\n}\n\n/**\n * Creates an assertion for validating tool-call arguments.\n *\n * Example:\n * `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`\n */\nexport function expectToolCallArgs(options: ToolCallArgsAssertionOptions): Assertion {\n  return async (context) => {\n    const targetCall = (context.toolCalls ?? []).find(call => call.name === options.toolName)\n\n    if (targetCall == null) {\n      return createOutcome(options.id, 'exact', false, 0, `Missing tool call: ${options.toolName}`)\n    }\n\n    const pass = options.validate(targetCall.args)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? `Tool call args validated for ${options.toolName}.` : `Tool call args validation failed for ${options.toolName}.`,\n    )\n  }\n}\n\n/**\n * Rubric judge result returned by teacher-model or rubric logic.\n */\nexport interface RubricJudgeResult {\n  /**\n   * Normalized score in the `0..1` range.\n   */\n  score: number\n  /**\n   * Judge explanation text.\n   */\n  reason: string\n  /**\n   * Optional judge model id.\n   */\n  judgeModel?: string\n}\n\n/**\n * Options for rubric assertions.\n */\nexport interface RubricAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Async rubric judge callback.\n   */\n  judge: (context: AssertionContext) => Promise<RubricJudgeResult>\n  /**\n   * Minimum passing score.\n   *\n   * @default 0.7\n   */\n  minScore?: number\n}\n\n/**\n * Creates a rubric assertion driven by teacher-model style scoring.\n *\n * Example:\n * `expectRubric({ id: 'human-like-tone', judge: judgeFn, minScore: 0.8 })`\n */\nexport function expectRubric(options: RubricAssertionOptions): Assertion {\n  return async (context) => {\n    const result = await options.judge(context)\n    const minScore = options.minScore ?? 0.7\n    const normalizedScore = clampScore(result.score)\n    const pass = normalizedScore >= minScore\n\n    return createOutcome(\n      options.id,\n      'judge',\n      pass,\n      normalizedScore,\n      `${result.reason}${result.judgeModel ? ` (judge: ${result.judgeModel})` : ''}`,\n    )\n  }\n}\n\n/**\n * Options for custom assertions.\n */\nexport interface CustomAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Score family emitted by this custom assertion.\n   */\n  scoreKind: RunScoreKind\n  /**\n   * Custom evaluator callback.\n   */\n  evaluate: (context: AssertionContext) => Promise<{ pass: boolean, reason: string, score: number }> | { pass: boolean, reason: string, score: number }\n}\n\n/**\n * Creates a custom assertion with fully user-defined logic.\n *\n * Example:\n * `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`\n */\nexport function expectCustom(options: CustomAssertionOptions): Assertion {\n  return async (context) => {\n    const result = await options.evaluate(context)\n\n    return createOutcome(options.id, options.scoreKind, result.pass, result.score, result.reason)\n  }\n}\n\n/**\n * Creates an inverse assertion.\n *\n * Example:\n * `expectNot(expectMustInclude({ id: 'contains-engine-word', keywords: ['bestmove'] }), { id: 'no-engine-word' })`\n */\nexport function expectNot(assertion: Assertion, options: { id: string }): Assertion {\n  return async (context) => {\n    const baseOutcome = await assertion(context)\n\n    return createOutcome(\n      options.id,\n      baseOutcome.scoreKind,\n      !baseOutcome.pass,\n      1 - baseOutcome.score,\n      `NOT(${baseOutcome.id}): ${baseOutcome.reason}`,\n    )\n  }\n}\n\n/**\n * Executes assertion list and returns all outcomes.\n *\n * Call stack:\n *\n * {@link evaluateAssertions}\n *   -> `assertion(context)`\n *     -> {@link AssertionOutcome}[]\n */\nexport async function evaluateAssertions(\n  assertions: readonly Assertion[],\n  context: Omit<AssertionContext, 'state'> & { state?: AssertionState },\n): Promise<AssertionOutcome[]> {\n  const state = context.state ?? new Map<string, unknown>()\n  const normalizedContext: AssertionContext = {\n    state,\n    structuredOutput: context.structuredOutput,\n    text: context.text,\n    toolCalls: context.toolCalls,\n  }\n\n  const outcomes: AssertionOutcome[] = []\n\n  for (const assertion of assertions) {\n    outcomes.push(await assertion(normalizedContext))\n  }\n\n  return outcomes\n}\n\n/**\n * Converts assertion outcomes to run-score tuples consumed by aggregation.\n */\nexport function toRunScores(outcomes: readonly AssertionOutcome[]): RunScore[] {\n  return outcomes.map(outcome => ({\n    kind: outcome.scoreKind,\n    score: outcome.score,\n  }))\n}\n\n/**\n * Returns failing assertion outcomes in original order.\n */\nexport function collectFailedAssertions(outcomes: readonly AssertionOutcome[]): AssertionOutcome[] {\n  return outcomes.filter(outcome => !outcome.pass)\n}\n"],"mappings":";;;;;;;AAoFA,SAAgB,mBAAmB,OAAe,eAAgC;CAChF,MAAM,sBAAsB,MAAM,KAAK,CAAC,CAAC,WAAW,QAAQ,GAAG;CAE/D,IAAI,eACF,OAAO;CAGT,OAAO,oBAAoB,YAAY;AACzC;AAEA,SAAS,WAAW,OAAuB;CACzC,IAAI,OAAO,MAAM,KAAK,GACpB,OAAO;CAGT,IAAI,QAAQ,GACV,OAAO;CAGT,IAAI,QAAQ,GACV,OAAO;CAGT,OAAO;AACT;AAEA,SAAS,cACP,IACA,WACA,MACA,OACA,QACkB;CAClB,OAAO;EACL;EACA;EACA;EACA,OAAO,WAAW,KAAK;EACvB;CACF;AACF;;;;;;;AAkCA,SAAgB,kBAAkB,SAAiD;CACjF,OAAO,OAAO,YAAY;EACxB,IAAI,QAAQ,SAAS,WAAW,GAC9B,OAAO,cAAc,QAAQ,IAAI,SAAS,MAAM,GAAG,kCAAkC;EAGvF,MAAM,gBAAgB,QAAQ,iBAAiB;EAC/C,MAAM,iBAAiB,mBAAmB,QAAQ,MAAM,aAAa;EACrE,MAAM,UAAU,QAAQ,SAAS,QAAQ,YAAY;GACnD,MAAM,oBAAoB,mBAAmB,SAAS,aAAa;GACnE,OAAO,eAAe,SAAS,iBAAiB;EAClD,CAAC;EAGD,MAAM,QADO,QAAQ,QAAQ,WACP,QAClB,QAAQ,WAAW,QAAQ,SAAS,SACpC,QAAQ,SAAS;EAErB,MAAM,QAAQ,QAAQ,SAAS,WAAW,IAAI,IAAI,QAAQ,SAAS,QAAQ,SAAS;EAEpF,OAAO,cACL,QAAQ,IACR,SACA,MACA,OACA,OACI,WAAW,QAAQ,OAAO,GAAG,QAAQ,SAAS,OAAO,uBACrD,WAAW,QAAQ,OAAO,GAAG,QAAQ,SAAS,OAAO,oBAC3D;CACF;AACF;;;;;;;AA4BA,SAAgB,kBAAkB,SAAiD;CACjF,OAAO,OAAO,YAAY;EACxB,IAAI,QAAQ,SAAS,WAAW,GAC9B,OAAO,cAAc,QAAQ,IAAI,SAAS,MAAM,GAAG,kCAAkC;EAGvF,MAAM,gBAAgB,QAAQ,iBAAiB;EAC/C,MAAM,iBAAiB,mBAAmB,QAAQ,MAAM,aAAa;EACrE,MAAM,mBAAmB,QAAQ,SAAS,QAAQ,YAAY;GAC5D,MAAM,oBAAoB,mBAAmB,SAAS,aAAa;GACnE,OAAO,eAAe,SAAS,iBAAiB;EAClD,CAAC;EAED,MAAM,OAAO,iBAAiB,WAAW;EACzC,MAAM,QAAQ,OAAO,IAAI;EAEzB,OAAO,cACL,QAAQ,IACR,SACA,MACA,OACA,OACI,iCACA,6BAA6B,iBAAiB,KAAK,IAAI,GAC7D;CACF;AACF;;;;;;;AAsBA,SAAgB,YAAY,SAA2C;CACrE,OAAO,OAAO,YAAY;EACxB,MAAM,OAAO,QAAQ,QAAQ,KAAK,QAAQ,IAAI;EAE9C,OAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,iCAAiC,wBAAwB,QAAQ,SAC1E;CACF;AACF;;;;;;;AA0BA,SAAgB,uBAA+B,SAA8D;CAC3G,OAAO,OAAO,YAAY;EACxB,MAAM,OAAO,QAAQ,SAAS,QAAQ,gBAAgB;EAEtD,OAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,yCAA0C,QAAQ,iBAAiB,sCAC5E;CACF;AACF;;;;;;;AA0BA,SAAgB,mBAAmB,SAAkD;CACnF,OAAO,OAAO,YAAY;EACxB,MAAM,cAAc,QAAQ,aAAa,CAAC,EAAA,CAAG,MAAK,SAAQ,KAAK,SAAS,QAAQ,QAAQ;EAExF,IAAI,cAAc,MAChB,OAAO,cAAc,QAAQ,IAAI,SAAS,OAAO,GAAG,sBAAsB,QAAQ,UAAU;EAG9F,MAAM,OAAO,QAAQ,SAAS,WAAW,IAAI;EAE7C,OAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,gCAAgC,QAAQ,SAAS,KAAK,wCAAwC,QAAQ,SAAS,EACxH;CACF;AACF;;;;;;;AA8CA,SAAgB,aAAa,SAA4C;CACvE,OAAO,OAAO,YAAY;EACxB,MAAM,SAAS,MAAM,QAAQ,MAAM,OAAO;EAC1C,MAAM,WAAW,QAAQ,YAAY;EACrC,MAAM,kBAAkB,WAAW,OAAO,KAAK;EAC/C,MAAM,OAAO,mBAAmB;EAEhC,OAAO,cACL,QAAQ,IACR,SACA,MACA,iBACA,GAAG,OAAO,SAAS,OAAO,aAAa,YAAY,OAAO,WAAW,KAAK,IAC5E;CACF;AACF;;;;;;;AA0BA,SAAgB,aAAa,SAA4C;CACvE,OAAO,OAAO,YAAY;EACxB,MAAM,SAAS,MAAM,QAAQ,SAAS,OAAO;EAE7C,OAAO,cAAc,QAAQ,IAAI,QAAQ,WAAW,OAAO,MAAM,OAAO,OAAO,OAAO,MAAM;CAC9F;AACF;;;;;;;AAQA,SAAgB,UAAU,WAAsB,SAAoC;CAClF,OAAO,OAAO,YAAY;EACxB,MAAM,cAAc,MAAM,UAAU,OAAO;EAE3C,OAAO,cACL,QAAQ,IACR,YAAY,WACZ,CAAC,YAAY,MACb,IAAI,YAAY,OAChB,OAAO,YAAY,GAAG,KAAK,YAAY,QACzC;CACF;AACF;;;;;;;;;;AAWA,eAAsB,mBACpB,YACA,SAC6B;CAE7B,MAAM,oBAAsC;EAC1C,OAFY,QAAQ,yBAAS,IAAI,IAAqB;EAGtD,kBAAkB,QAAQ;EAC1B,MAAM,QAAQ;EACd,WAAW,QAAQ;CACrB;CAEA,MAAM,WAA+B,CAAC;CAEtC,KAAK,MAAM,aAAa,YACtB,SAAS,KAAK,MAAM,UAAU,iBAAiB,CAAC;CAGlD,OAAO;AACT;;;;AAKA,SAAgB,YAAY,UAAmD;CAC7E,OAAO,SAAS,KAAI,aAAY;EAC9B,MAAM,QAAQ;EACd,OAAO,QAAQ;CACjB,EAAE;AACJ;;;;AAKA,SAAgB,wBAAwB,UAA2D;CACjG,OAAO,SAAS,QAAO,YAAW,CAAC,QAAQ,IAAI;AACjD"}
+{"version":3,"file":"index.mjs","names":[],"sources":["../../../src/core/assertions/index.ts"],"sourcesContent":["import type { RunScore, RunScoreKind } from '../runner/aggregate'\n\n/**\n * Async assertion function used by eval scenarios.\n */\nexport type Assertion = (context: AssertionContext) => Promise<AssertionOutcome>\n\n/**\n * Normalized assertion context for one model output.\n */\nexport interface AssertionContext {\n  /**\n   * Shared mutable state for stateful assertion measurement.\n   */\n  state: AssertionState\n  /**\n   * Optional structured output parsed from the model response.\n   */\n  structuredOutput?: unknown\n  /**\n   * Plain text model output used by text assertions.\n   */\n  text: string\n  /**\n   * Optional tool calls extracted from the model response.\n   */\n  toolCalls?: readonly ToolCall[]\n}\n\n/**\n * Result for one assertion evaluation.\n */\nexport interface AssertionOutcome {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Whether the assertion passed.\n   */\n  pass: boolean\n  /**\n   * Human-readable reason for logs and reports.\n   */\n  reason: string\n  /**\n   * Normalized score in the `0..1` range.\n   */\n  score: number\n  /**\n   * Assertion family emitted as run score kind.\n   */\n  scoreKind: RunScoreKind\n}\n\n/**\n * Stores mutable evaluation state for stateful assertion flows.\n *\n * Use when:\n * - assertions need to share counters, rolling metrics, or memoized values\n * - a scenario evaluates multiple steps and expects state-aware checks\n */\nexport type AssertionState = Map<string, unknown>\n\n/**\n * Options for custom assertions.\n */\nexport interface CustomAssertionOptions {\n  /**\n   * Custom evaluator callback.\n   */\n  evaluate: (context: AssertionContext) => Promise<{ pass: boolean, reason: string, score: number }> | { pass: boolean, reason: string, score: number }\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Score family emitted by this custom assertion.\n   */\n  scoreKind: RunScoreKind\n}\n\n/**\n * Options for exclude-keyword assertions.\n */\nexport interface MustExcludeAssertionOptions {\n  /**\n   * Case-sensitive matching toggle.\n   *\n   * @default false\n   */\n  caseSensitive?: boolean\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Keywords that must not appear.\n   */\n  keywords: readonly string[]\n}\n\n/**\n * Options for include-keyword assertions.\n */\nexport interface MustIncludeAssertionOptions {\n  /**\n   * Case-sensitive matching toggle.\n   *\n   * @default false\n   */\n  caseSensitive?: boolean\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Keywords that must be present.\n   */\n  keywords: readonly string[]\n  /**\n   * Match mode for keywords.\n   *\n   * @default 'all'\n   */\n  mode?: 'all' | 'any'\n}\n\n/**\n * Options for regular-expression assertions.\n */\nexport interface RegexAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Pattern to apply to model text.\n   */\n  pattern: RegExp\n}\n\n/**\n * Options for rubric assertions.\n */\nexport interface RubricAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Async rubric judge callback.\n   */\n  judge: (context: AssertionContext) => Promise<RubricJudgeResult>\n  /**\n   * Minimum passing score.\n   *\n   * @default 0.7\n   */\n  minScore?: number\n}\n\n/**\n * Rubric judge result returned by teacher-model or rubric logic.\n */\nexport interface RubricJudgeResult {\n  /**\n   * Optional judge model id.\n   */\n  judgeModel?: string\n  /**\n   * Judge explanation text.\n   */\n  reason: string\n  /**\n   * Normalized score in the `0..1` range.\n   */\n  score: number\n}\n\n/**\n * Options for structured-output assertions.\n */\nexport interface StructuredOutputAssertionOptions<TValue> {\n  /**\n   * Optional failure reason.\n   */\n  failureReason?: string\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Runtime validator for structured output.\n   */\n  validate: (value: unknown) => value is TValue\n}\n\n/**\n * Represents one tool call emitted by a model response.\n */\nexport interface ToolCall {\n  /**\n   * Tool arguments payload.\n   */\n  args: unknown\n  /**\n   * Tool name used by the call.\n   */\n  name: string\n}\n\n/**\n * Options for tool-call argument assertions.\n */\nexport interface ToolCallArgsAssertionOptions {\n  /**\n   * Stable assertion id.\n   */\n  id: string\n  /**\n   * Tool name to inspect.\n   */\n  toolName: string\n  /**\n   * Runtime validator for tool arguments.\n   */\n  validate: (args: unknown) => boolean\n}\n\n/**\n * Returns failing assertion outcomes in original order.\n */\nexport function collectFailedAssertions(outcomes: readonly AssertionOutcome[]): AssertionOutcome[] {\n  return outcomes.filter(outcome => !outcome.pass)\n}\n\n/**\n * Executes assertion list and returns all outcomes.\n *\n * Call stack:\n *\n * {@link evaluateAssertions}\n *   -> `assertion(context)`\n *     -> {@link AssertionOutcome}[]\n */\nexport async function evaluateAssertions(\n  assertions: readonly Assertion[],\n  context: Omit<AssertionContext, 'state'> & { state?: AssertionState },\n): Promise<AssertionOutcome[]> {\n  const state = context.state ?? new Map<string, unknown>()\n  const normalizedContext: AssertionContext = {\n    state,\n    structuredOutput: context.structuredOutput,\n    text: context.text,\n    toolCalls: context.toolCalls,\n  }\n\n  const outcomes: AssertionOutcome[] = []\n\n  for (const assertion of assertions) {\n    outcomes.push(await assertion(normalizedContext))\n  }\n\n  return outcomes\n}\n\n/**\n * Creates a custom assertion with fully user-defined logic.\n *\n * Example:\n * `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`\n */\nexport function expectCustom(options: CustomAssertionOptions): Assertion {\n  return async (context) => {\n    const result = await options.evaluate(context)\n\n    return createOutcome(options.id, options.scoreKind, result.pass, result.score, result.reason)\n  }\n}\n\n/**\n * Creates an assertion that forbids specific keywords.\n *\n * Example:\n * `expectMustExclude({ id: 'no-engine-dump', keywords: ['bestmove', 'ponder'] })`\n */\nexport function expectMustExclude(options: MustExcludeAssertionOptions): Assertion {\n  return async (context) => {\n    if (options.keywords.length === 0) {\n      return createOutcome(options.id, 'exact', true, 1, 'No excluded keywords configured.')\n    }\n\n    const caseSensitive = options.caseSensitive ?? false\n    const normalizedText = normalizeMatchText(context.text, caseSensitive)\n    const forbiddenMatches = options.keywords.filter((keyword) => {\n      const normalizedKeyword = normalizeMatchText(keyword, caseSensitive)\n      return normalizedText.includes(normalizedKeyword)\n    })\n\n    const pass = forbiddenMatches.length === 0\n    const score = pass ? 1 : 0\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      score,\n      pass\n        ? 'No forbidden keywords found.'\n        : `Forbidden keywords found: ${forbiddenMatches.join(', ')}`,\n    )\n  }\n}\n\n/**\n * Creates an assertion that requires specific keywords in model text.\n *\n * Example:\n * `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`\n */\nexport function expectMustInclude(options: MustIncludeAssertionOptions): Assertion {\n  return async (context) => {\n    if (options.keywords.length === 0) {\n      return createOutcome(options.id, 'exact', true, 1, 'No required keywords configured.')\n    }\n\n    const caseSensitive = options.caseSensitive ?? false\n    const normalizedText = normalizeMatchText(context.text, caseSensitive)\n    const matches = options.keywords.filter((keyword) => {\n      const normalizedKeyword = normalizeMatchText(keyword, caseSensitive)\n      return normalizedText.includes(normalizedKeyword)\n    })\n\n    const mode = options.mode ?? 'all'\n    const pass = mode === 'all'\n      ? matches.length === options.keywords.length\n      : matches.length > 0\n\n    const score = options.keywords.length === 0 ? 1 : matches.length / options.keywords.length\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      score,\n      pass\n        ? `Matched ${matches.length}/${options.keywords.length} required keywords.`\n        : `Matched ${matches.length}/${options.keywords.length} required keywords.`,\n    )\n  }\n}\n\n/**\n * Creates an inverse assertion.\n *\n * Example:\n * `expectNot(expectMustInclude({ id: 'contains-engine-word', keywords: ['bestmove'] }), { id: 'no-engine-word' })`\n */\nexport function expectNot(assertion: Assertion, options: { id: string }): Assertion {\n  return async (context) => {\n    const baseOutcome = await assertion(context)\n\n    return createOutcome(\n      options.id,\n      baseOutcome.scoreKind,\n      !baseOutcome.pass,\n      1 - baseOutcome.score,\n      `NOT(${baseOutcome.id}): ${baseOutcome.reason}`,\n    )\n  }\n}\n\n/**\n * Creates an assertion based on a regular expression.\n *\n * Example:\n * `expectRegex({ id: 'starts-with-act', pattern: /^<\\|ACT:/ })`\n */\nexport function expectRegex(options: RegexAssertionOptions): Assertion {\n  return async (context) => {\n    const pass = options.pattern.test(context.text)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? 'Regex matched response text.' : `Regex did not match: ${options.pattern}`,\n    )\n  }\n}\n\n/**\n * Creates a rubric assertion driven by teacher-model style scoring.\n *\n * Example:\n * `expectRubric({ id: 'human-like-tone', judge: judgeFn, minScore: 0.8 })`\n */\nexport function expectRubric(options: RubricAssertionOptions): Assertion {\n  return async (context) => {\n    const result = await options.judge(context)\n    const minScore = options.minScore ?? 0.7\n    const normalizedScore = clampScore(result.score)\n    const pass = normalizedScore >= minScore\n\n    return createOutcome(\n      options.id,\n      'judge',\n      pass,\n      normalizedScore,\n      `${result.reason}${result.judgeModel ? ` (judge: ${result.judgeModel})` : ''}`,\n    )\n  }\n}\n\n/**\n * Creates an assertion for structured model output.\n *\n * Example:\n * `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`\n */\nexport function expectStructuredOutput<TValue>(options: StructuredOutputAssertionOptions<TValue>): Assertion {\n  return async (context) => {\n    const pass = options.validate(context.structuredOutput)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? 'Structured output matched validator.' : (options.failureReason ?? 'Structured output validation failed.'),\n    )\n  }\n}\n\n/**\n * Creates an assertion for validating tool-call arguments.\n *\n * Example:\n * `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`\n */\nexport function expectToolCallArgs(options: ToolCallArgsAssertionOptions): Assertion {\n  return async (context) => {\n    const targetCall = (context.toolCalls ?? []).find(call => call.name === options.toolName)\n\n    if (targetCall == null) {\n      return createOutcome(options.id, 'exact', false, 0, `Missing tool call: ${options.toolName}`)\n    }\n\n    const pass = options.validate(targetCall.args)\n\n    return createOutcome(\n      options.id,\n      'exact',\n      pass,\n      pass ? 1 : 0,\n      pass ? `Tool call args validated for ${options.toolName}.` : `Tool call args validation failed for ${options.toolName}.`,\n    )\n  }\n}\n\n/**\n * Normalizes text for matching.\n *\n * Before: `\"  Hello\\nWorld  \"`\n * After: `\"hello world\"`\n */\nexport function normalizeMatchText(value: string, caseSensitive: boolean): string {\n  const compactedWhitespace = value.trim().replaceAll(/\\s+/g, ' ')\n\n  if (caseSensitive) {\n    return compactedWhitespace\n  }\n\n  return compactedWhitespace.toLowerCase()\n}\n\n/**\n * Converts assertion outcomes to run-score tuples consumed by aggregation.\n */\nexport function toRunScores(outcomes: readonly AssertionOutcome[]): RunScore[] {\n  return outcomes.map(outcome => ({\n    kind: outcome.scoreKind,\n    score: outcome.score,\n  }))\n}\n\nfunction clampScore(score: number): number {\n  if (Number.isNaN(score)) {\n    return 0\n  }\n\n  if (score < 0) {\n    return 0\n  }\n\n  if (score > 1) {\n    return 1\n  }\n\n  return score\n}\n\nfunction createOutcome(\n  id: string,\n  scoreKind: RunScoreKind,\n  pass: boolean,\n  score: number,\n  reason: string,\n): AssertionOutcome {\n  return {\n    id,\n    pass,\n    reason,\n    score: clampScore(score),\n    scoreKind,\n  }\n}\n"],"mappings":";;;;AAyOA,SAAgB,wBAAwB,UAA2D;CACjG,OAAO,SAAS,QAAO,YAAW,CAAC,QAAQ,IAAI;AACjD;;;;;;;;;;AAWA,eAAsB,mBACpB,YACA,SAC6B;CAE7B,MAAM,oBAAsC;EAC1C,OAFY,QAAQ,yBAAS,IAAI,IAAqB;EAGtD,kBAAkB,QAAQ;EAC1B,MAAM,QAAQ;EACd,WAAW,QAAQ;CACrB;CAEA,MAAM,WAA+B,CAAC;CAEtC,KAAK,MAAM,aAAa,YACtB,SAAS,KAAK,MAAM,UAAU,iBAAiB,CAAC;CAGlD,OAAO;AACT;;;;;;;AAQA,SAAgB,aAAa,SAA4C;CACvE,OAAO,OAAO,YAAY;EACxB,MAAM,SAAS,MAAM,QAAQ,SAAS,OAAO;EAE7C,OAAO,cAAc,QAAQ,IAAI,QAAQ,WAAW,OAAO,MAAM,OAAO,OAAO,OAAO,MAAM;CAC9F;AACF;;;;;;;AAQA,SAAgB,kBAAkB,SAAiD;CACjF,OAAO,OAAO,YAAY;EACxB,IAAI,QAAQ,SAAS,WAAW,GAC9B,OAAO,cAAc,QAAQ,IAAI,SAAS,MAAM,GAAG,kCAAkC;EAGvF,MAAM,gBAAgB,QAAQ,iBAAiB;EAC/C,MAAM,iBAAiB,mBAAmB,QAAQ,MAAM,aAAa;EACrE,MAAM,mBAAmB,QAAQ,SAAS,QAAQ,YAAY;GAC5D,MAAM,oBAAoB,mBAAmB,SAAS,aAAa;GACnE,OAAO,eAAe,SAAS,iBAAiB;EAClD,CAAC;EAED,MAAM,OAAO,iBAAiB,WAAW;EACzC,MAAM,QAAQ,OAAO,IAAI;EAEzB,OAAO,cACL,QAAQ,IACR,SACA,MACA,OACA,OACI,iCACA,6BAA6B,iBAAiB,KAAK,IAAI,GAC7D;CACF;AACF;;;;;;;AAQA,SAAgB,kBAAkB,SAAiD;CACjF,OAAO,OAAO,YAAY;EACxB,IAAI,QAAQ,SAAS,WAAW,GAC9B,OAAO,cAAc,QAAQ,IAAI,SAAS,MAAM,GAAG,kCAAkC;EAGvF,MAAM,gBAAgB,QAAQ,iBAAiB;EAC/C,MAAM,iBAAiB,mBAAmB,QAAQ,MAAM,aAAa;EACrE,MAAM,UAAU,QAAQ,SAAS,QAAQ,YAAY;GACnD,MAAM,oBAAoB,mBAAmB,SAAS,aAAa;GACnE,OAAO,eAAe,SAAS,iBAAiB;EAClD,CAAC;EAGD,MAAM,QADO,QAAQ,QAAQ,WACP,QAClB,QAAQ,WAAW,QAAQ,SAAS,SACpC,QAAQ,SAAS;EAErB,MAAM,QAAQ,QAAQ,SAAS,WAAW,IAAI,IAAI,QAAQ,SAAS,QAAQ,SAAS;EAEpF,OAAO,cACL,QAAQ,IACR,SACA,MACA,OACA,OACI,WAAW,QAAQ,OAAO,GAAG,QAAQ,SAAS,OAAO,uBACrD,WAAW,QAAQ,OAAO,GAAG,QAAQ,SAAS,OAAO,oBAC3D;CACF;AACF;;;;;;;AAQA,SAAgB,UAAU,WAAsB,SAAoC;CAClF,OAAO,OAAO,YAAY;EACxB,MAAM,cAAc,MAAM,UAAU,OAAO;EAE3C,OAAO,cACL,QAAQ,IACR,YAAY,WACZ,CAAC,YAAY,MACb,IAAI,YAAY,OAChB,OAAO,YAAY,GAAG,KAAK,YAAY,QACzC;CACF;AACF;;;;;;;AAQA,SAAgB,YAAY,SAA2C;CACrE,OAAO,OAAO,YAAY;EACxB,MAAM,OAAO,QAAQ,QAAQ,KAAK,QAAQ,IAAI;EAE9C,OAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,iCAAiC,wBAAwB,QAAQ,SAC1E;CACF;AACF;;;;;;;AAQA,SAAgB,aAAa,SAA4C;CACvE,OAAO,OAAO,YAAY;EACxB,MAAM,SAAS,MAAM,QAAQ,MAAM,OAAO;EAC1C,MAAM,WAAW,QAAQ,YAAY;EACrC,MAAM,kBAAkB,WAAW,OAAO,KAAK;EAC/C,MAAM,OAAO,mBAAmB;EAEhC,OAAO,cACL,QAAQ,IACR,SACA,MACA,iBACA,GAAG,OAAO,SAAS,OAAO,aAAa,YAAY,OAAO,WAAW,KAAK,IAC5E;CACF;AACF;;;;;;;AAQA,SAAgB,uBAA+B,SAA8D;CAC3G,OAAO,OAAO,YAAY;EACxB,MAAM,OAAO,QAAQ,SAAS,QAAQ,gBAAgB;EAEtD,OAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,yCAA0C,QAAQ,iBAAiB,sCAC5E;CACF;AACF;;;;;;;AAQA,SAAgB,mBAAmB,SAAkD;CACnF,OAAO,OAAO,YAAY;EACxB,MAAM,cAAc,QAAQ,aAAa,CAAC,EAAA,CAAG,MAAK,SAAQ,KAAK,SAAS,QAAQ,QAAQ;EAExF,IAAI,cAAc,MAChB,OAAO,cAAc,QAAQ,IAAI,SAAS,OAAO,GAAG,sBAAsB,QAAQ,UAAU;EAG9F,MAAM,OAAO,QAAQ,SAAS,WAAW,IAAI;EAE7C,OAAO,cACL,QAAQ,IACR,SACA,MACA,OAAO,IAAI,GACX,OAAO,gCAAgC,QAAQ,SAAS,KAAK,wCAAwC,QAAQ,SAAS,EACxH;CACF;AACF;;;;;;;AAQA,SAAgB,mBAAmB,OAAe,eAAgC;CAChF,MAAM,sBAAsB,MAAM,KAAK,CAAC,CAAC,WAAW,QAAQ,GAAG;CAE/D,IAAI,eACF,OAAO;CAGT,OAAO,oBAAoB,YAAY;AACzC;;;;AAKA,SAAgB,YAAY,UAAmD;CAC7E,OAAO,SAAS,KAAI,aAAY;EAC9B,MAAM,QAAQ;EACd,OAAO,QAAQ;CACjB,EAAE;AACJ;AAEA,SAAS,WAAW,OAAuB;CACzC,IAAI,OAAO,MAAM,KAAK,GACpB,OAAO;CAGT,IAAI,QAAQ,GACV,OAAO;CAGT,IAAI,QAAQ,GACV,OAAO;CAGT,OAAO;AACT;AAEA,SAAS,cACP,IACA,WACA,MACA,OACA,QACkB;CAClB,OAAO;EACL;EACA;EACA;EACA,OAAO,WAAW,KAAK;EACvB;CACF;AACF"}

package/dist/core/inference-executors/index.d.mts

@@ -1,4 +1,4 @@
-import { a as requiredEnvFrom, i as envFrom, n as EnvValueType, r as RequiredEnvFromOptions, t as EnvFromOptions } from "../../env-bRH0K6fU.mjs";
+import { a as requiredEnvFrom, i as envFrom, n as EnvValueType, r as RequiredEnvFromOptions, t as EnvFromOptions } from "../../env-Br6jaWGL.mjs";
 import { createOpenAI } from "@xsai-ext/providers/create";
 
 //#region src/core/inference-executors/retry-policy.d.ts
@@ -11,14 +11,14 @@ import { createOpenAI } from "@xsai-ext/providers/create";
  * attempt -> run request -> non-retriable failure -> throw
  */
 interface RetryPolicy {
-  /**
-   * Maximum number of total attempts, including the first try.
-   */
-  maxAttempts: number;
   /**
    * Returns the wait time for a retry attempt.
    */
   delayMs: (attempt: number) => number;
+  /**
+   * Maximum number of total attempts, including the first try.
+   */
+  maxAttempts: number;
   /**
    * Determines whether an error can be retried safely.
    */
@@ -40,18 +40,18 @@ interface RetryPolicy {
  * - `delayMs` to return a non-negative wait time in milliseconds
  */
 interface RetryPolicyOptions {
-  /**
-   * Maximum total attempts, including the first request.
-   *
-   * @default 3
-   */
-  maxAttempts?: number;
   /**
    * Computes the delay for a retry attempt.
    *
    * The attempt number starts at `1` for the first retry.
    */
   delayMs?: (attempt: number) => number;
+  /**
+   * Maximum total attempts, including the first request.
+   *
+   * @default 3
+   */
+  maxAttempts?: number;
   /**
    * Overrides the retry classifier.
    */
@@ -61,6 +61,7 @@ interface RetryPolicyOptions {
    */
   sleep?: (milliseconds: number) => Promise<void>;
 }
+declare function createRetryPolicy(options?: RetryPolicyOptions): RetryPolicy;
 /**
  * Returns true when a provider failure is temporary and a retry is reasonable.
  *
@@ -71,7 +72,6 @@ interface RetryPolicyOptions {
  * - provider errors to expose a status code, name, or message when possible
  */
 declare function isRetriableProviderError(error: unknown): boolean;
-declare function createRetryPolicy(options?: RetryPolicyOptions): RetryPolicy;
 /**
  * Runs an operation with bounded retries.
  *
@@ -118,24 +118,10 @@ interface ProviderAdapter<TProvider> {
 declare function createProviderAdapter<TProvider>(provider: TProvider, options?: RetryPolicyOptions): ProviderAdapter<TProvider>;
 //#endregion
 //#region src/core/inference-executors/remote-providers/openai/index.d.ts
-/**
- * Represents the OpenAI provider instance returned by xsai.
- */
-type OpenAIProvider = ReturnType<typeof createOpenAI>;
-/**
- * Represents the OpenAI adapter used by vieval.
- */
-type OpenAIProviderAdapter = ProviderAdapter<OpenAIProvider>;
 /**
  * Configures env key names and source for OpenAI provider setup.
  */
 interface OpenAIEnvSourceOptions {
-  /**
-   * Environment object used for variable lookup.
-   *
-   * @default process.env
-   */
-  env?: NodeJS.ProcessEnv;
   /**
    * Env key name for API key.
    *
@@ -148,6 +134,12 @@ interface OpenAIEnvSourceOptions {
    * @default 'OPENAI_BASE_URL'
    */
   baseURL?: string;
+  /**
+   * Environment object used for variable lookup.
+   *
+   * @default process.env
+   */
+  env?: NodeJS.ProcessEnv;
   /**
    * Env key name for model.
    *
@@ -185,6 +177,14 @@ interface OpenAIFromEnvResult {
   baseURL?: string;
   model: string;
 }
+/**
+ * Represents the OpenAI provider instance returned by xsai.
+ */
+type OpenAIProvider = ReturnType<typeof createOpenAI>;
+/**
+ * Represents the OpenAI adapter used by vieval.
+ */
+type OpenAIProviderAdapter = ProviderAdapter<OpenAIProvider>;
 /**
  * Minimal response shape returned by text-generation calls.
  */
@@ -194,18 +194,8 @@ interface OpenAITextGenerationResult {
    *
    * Some OpenAI-compatible implementations may return `null`.
    */
-  text?: string | null;
+  text?: null | string;
 }
-/**
- * Normalizes provider text output to a safe string.
- *
- * Before: `{ text: null }`
- * After: `''`
- *
- * Before: `{ text: 'hello' }`
- * After: `'hello'`
- */
-declare function normalizeOpenAITextOutput(result: OpenAITextGenerationResult): string;
 /**
  * Creates an OpenAI provider adapter using environment variables with defaults.
  *
@@ -224,6 +214,16 @@ declare function createOpenAIFromEnv(source?: OpenAIEnvSourceOptions, defaults?:
  * - `retryOptions` to follow the same invariants as `createRetryPolicy`
  */
 declare function createOpenAIProviderAdapter(apiKey: string, baseURL?: string, retryOptions?: RetryPolicyOptions): OpenAIProviderAdapter;
+/**
+ * Normalizes provider text output to a safe string.
+ *
+ * Before: `{ text: null }`
+ * After: `''`
+ *
+ * Before: `{ text: 'hello' }`
+ * After: `'hello'`
+ */
+declare function normalizeOpenAITextOutput(result: OpenAITextGenerationResult): string;
 //#endregion
 export { EnvFromOptions, EnvValueType, OpenAIEnvSourceOptions, OpenAIFromEnvDefaultOptions, OpenAIFromEnvResult, OpenAIProvider, OpenAIProviderAdapter, OpenAITextGenerationResult, ProviderAdapter, RequiredEnvFromOptions, RetryPolicy, RetryPolicyOptions, createOpenAIFromEnv, createOpenAIProviderAdapter, createProviderAdapter, createRetryPolicy, envFrom, isRetriableProviderError, normalizeOpenAITextOutput, requiredEnvFrom, runWithRetry };
 //# sourceMappingURL=index.d.mts.map

package/dist/core/inference-executors/index.mjs

@@ -1,4 +1,4 @@
-import { n as requiredEnvFrom, t as envFrom } from "../../env-BVYeJhGA.mjs";
+import { n as requiredEnvFrom, t as envFrom } from "../../env-egxaJtNn.mjs";
 import process from "node:process";
 import { errorMessageFrom, errorNameFrom, sleep } from "@moeru/std";
 import { createOpenAI } from "@xsai-ext/providers/create";
@@ -12,7 +12,7 @@ const retryableStatusCodes = /* @__PURE__ */ new Set([
 	503,
 	504
 ]);
-const retryableErrorNames = /* @__PURE__ */ new Set(["TimeoutError", "FetchError"]);
+const retryableErrorNames = /* @__PURE__ */ new Set(["FetchError", "TimeoutError"]);
 const retryableMessagePatterns = [
 	/rate limit/i,
 	/rate-limited/i,
@@ -29,16 +29,14 @@ const retryableMessagePatterns = [
 	/timed out/i,
 	/timeout/i
 ];
-function getStatusCode(error) {
-	if (error == null || typeof error !== "object") return;
-	const maybeStatusCode = error.statusCode;
-	if (typeof maybeStatusCode === "number") return maybeStatusCode;
-	const maybeStatus = error.status;
-	if (typeof maybeStatus === "number") return maybeStatus;
-	const response = error.response;
-	if (response == null || typeof response !== "object") return;
-	const responseStatus = response.status;
-	return typeof responseStatus === "number" ? responseStatus : void 0;
+function createRetryPolicy(options = {}) {
+	const maxAttempts = assertValidMaxAttempts(options.maxAttempts ?? 3);
+	return {
+		delayMs: options.delayMs ?? defaultDelayMs,
+		maxAttempts,
+		shouldRetry: options.shouldRetry ?? isRetriableProviderError,
+		sleep: options.sleep ?? sleep
+	};
 }
 /**
 * Returns true when a provider failure is temporary and a retry is reasonable.
@@ -58,34 +56,6 @@ function isRetriableProviderError(error) {
 	if (errorMessage == null) return false;
 	return retryableMessagePatterns.some((pattern) => pattern.test(errorMessage));
 }
-function defaultDelayMs(attempt) {
-	return 500 * 2 ** (attempt - 1);
-}
-/**
-* Creates a retry policy for provider work.
-*
-* Use when:
-* - you need a reusable retry runner for eval-time provider calls
-* - you want to keep retry behavior deterministic in tests
-*
-* Expects:
-* - callers to treat `maxAttempts` as total attempts, not retries
-*
-* Throws:
-* - `RangeError` when `maxAttempts` is not a finite integer greater than or equal to `1`
-*/
-function assertValidMaxAttempts(value) {
-	if (!Number.isFinite(value) || !Number.isInteger(value) || value < 1) throw new RangeError("maxAttempts must be a finite integer greater than or equal to 1.");
-	return value;
-}
-function createRetryPolicy(options = {}) {
-	return {
-		maxAttempts: assertValidMaxAttempts(options.maxAttempts ?? 3),
-		delayMs: options.delayMs ?? defaultDelayMs,
-		shouldRetry: options.shouldRetry ?? isRetriableProviderError,
-		sleep: options.sleep ?? sleep
-	};
-}
 /**
 * Runs an operation with bounded retries.
 *
@@ -106,6 +76,37 @@ async function runWithRetry(operation, policy = createRetryPolicy()) {
 	}
 	throw new Error("Retry loop exited without returning a value.");
 }
+/**
+* Creates a retry policy for provider work.
+*
+* Use when:
+* - you need a reusable retry runner for eval-time provider calls
+* - you want to keep retry behavior deterministic in tests
+*
+* Expects:
+* - callers to treat `maxAttempts` as total attempts, not retries
+*
+* Throws:
+* - `RangeError` when `maxAttempts` is not a finite integer greater than or equal to `1`
+*/
+function assertValidMaxAttempts(value) {
+	if (!Number.isFinite(value) || !Number.isInteger(value) || value < 1) throw new RangeError("maxAttempts must be a finite integer greater than or equal to 1.");
+	return value;
+}
+function defaultDelayMs(attempt) {
+	return 500 * 2 ** (attempt - 1);
+}
+function getStatusCode(error) {
+	if (error == null || typeof error !== "object") return;
+	const maybeStatusCode = error.statusCode;
+	if (typeof maybeStatusCode === "number") return maybeStatusCode;
+	const maybeStatus = error.status;
+	if (typeof maybeStatus === "number") return maybeStatus;
+	const response = error.response;
+	if (response == null || typeof response !== "object") return;
+	const responseStatus = response.status;
+	return typeof responseStatus === "number" ? responseStatus : void 0;
+}
 //#endregion
 //#region src/core/inference-executors/adapters.ts
 /**
@@ -128,18 +129,6 @@ function createProviderAdapter(provider, options = {}) {
 //#endregion
 //#region src/core/inference-executors/remote-providers/openai/index.ts
 /**
-* Normalizes provider text output to a safe string.
-*
-* Before: `{ text: null }`
-* After: `''`
-*
-* Before: `{ text: 'hello' }`
-* After: `'hello'`
-*/
-function normalizeOpenAITextOutput(result) {
-	return typeof result.text === "string" ? result.text : "";
-}
-/**
 * Creates an OpenAI provider adapter using environment variables with defaults.
 *
 * Example:
@@ -188,6 +177,18 @@ function createOpenAIFromEnv(source = {}, defaults = {}) {
 function createOpenAIProviderAdapter(apiKey, baseURL, retryOptions = {}) {
 	return createProviderAdapter(createOpenAI(apiKey, baseURL), retryOptions);
 }
+/**
+* Normalizes provider text output to a safe string.
+*
+* Before: `{ text: null }`
+* After: `''`
+*
+* Before: `{ text: 'hello' }`
+* After: `'hello'`
+*/
+function normalizeOpenAITextOutput(result) {
+	return typeof result.text === "string" ? result.text : "";
+}
 //#endregion
 export { createOpenAIFromEnv, createOpenAIProviderAdapter, createProviderAdapter, createRetryPolicy, envFrom, isRetriableProviderError, normalizeOpenAITextOutput, requiredEnvFrom, runWithRetry };