vieval 0.0.1

package/dist/runner-4ZsOveoY.mjs

@@ -0,0 +1,480 @@
+import { t as resolveModelByName } from "./models-D_MsBtYw.mjs";
+import { createRequire } from "node:module";
+import { basename, dirname, relative } from "node:path";
+import { fileURLToPath } from "node:url";
+import { errorMessageFrom } from "@moeru/std";
+//#region src/core/runner/aggregate.ts
+function cloneScheduledTaskMatrix(matrix) {
+	return {
+		eval: { ...matrix.eval },
+		meta: { ...matrix.meta },
+		run: { ...matrix.run }
+	};
+}
+function assertKnownScoreKind(kind) {
+	if (kind === "exact" || kind === "judge") return kind;
+	throw new TypeError(`Unknown eval score kind "${kind}".`);
+}
+function average(scores) {
+	if (scores.length === 0) return null;
+	return scores.reduce((sum, score) => sum + score, 0) / scores.length;
+}
+function createHybridAverage(exactAverage, judgeAverage) {
+	if (exactAverage != null && judgeAverage != null) return (exactAverage + judgeAverage) / 2;
+	if (exactAverage != null) return exactAverage;
+	if (judgeAverage != null) return judgeAverage;
+	return null;
+}
+function collectScoreBuckets(scores) {
+	const buckets = {
+		exact: [],
+		judge: []
+	};
+	for (const score of scores) {
+		if (assertKnownScoreKind(score.kind) === "exact") {
+			buckets.exact.push(score.score);
+			continue;
+		}
+		buckets.judge.push(score.score);
+	}
+	return buckets;
+}
+function createRunSummary(result) {
+	const buckets = collectScoreBuckets(result.scores);
+	const exactAverage = average(buckets.exact);
+	const judgeAverage = average(buckets.judge);
+	return {
+		entryId: result.entryId,
+		exactAverage,
+		hybridAverage: createHybridAverage(exactAverage, judgeAverage),
+		id: result.id,
+		judgeAverage,
+		matrix: cloneScheduledTaskMatrix(result.matrix),
+		inferenceExecutorId: result.inferenceExecutorId
+	};
+}
+function createProviderSummary(inferenceExecutorId, results) {
+	const exactScores = [];
+	const judgeScores = [];
+	for (const result of results) {
+		const buckets = collectScoreBuckets(result.scores);
+		exactScores.push(...buckets.exact);
+		judgeScores.push(...buckets.judge);
+	}
+	const exactAverage = average(exactScores);
+	const judgeAverage = average(judgeScores);
+	return {
+		exactAverage,
+		hybridAverage: createHybridAverage(exactAverage, judgeAverage),
+		judgeAverage,
+		inferenceExecutorId,
+		runCount: results.length
+	};
+}
+/**
+* Aggregates exact-match and judge-based scores into hybrid runner summaries.
+*
+* Call stack:
+*
+* {@link runScheduledTasks}
+*   -> {@link aggregateRunResults}
+*     -> {@link createRunSummary}
+*     -> {@link createProviderSummary}
+*       -> `report output`
+*
+* Use when:
+* - a runner batch mixes deterministic exact checks with judge-based grading
+* - inferenceExecutor comparison should preserve both score families and one hybrid view
+*
+* Expects:
+* - each score to be normalized to the `0..1` range before aggregation
+* - `scores.kind` to use only `'exact'` or `'judge'`
+*/
+function aggregateRunResults(results) {
+	const runs = results.map(createRunSummary);
+	const inferenceExecutors = Array.from(new Set(results.map((result) => result.inferenceExecutorId))).map((inferenceExecutorId) => {
+		return createProviderSummary(inferenceExecutorId, results.filter((result) => result.inferenceExecutorId === inferenceExecutorId));
+	}).sort((left, right) => left.inferenceExecutorId.localeCompare(right.inferenceExecutorId));
+	const overall = createProviderSummary("overall", results);
+	return {
+		overall: {
+			exactAverage: overall.exactAverage,
+			hybridAverage: overall.hybridAverage,
+			judgeAverage: overall.judgeAverage,
+			runCount: overall.runCount
+		},
+		inferenceExecutors,
+		runs
+	};
+}
+//#endregion
+//#region src/core/runner/collect.ts
+const evalFileSuffix = ".eval.ts";
+const absolutePathPattern = /^(?:[A-Z]:\/|\/|\\\\)/i;
+function normalizePath(value) {
+	return value.replaceAll("\\", "/");
+}
+/**
+* Converts a file path into a project-relative path when possible.
+*
+* Before: `/repo/plugins/airi-plugin-game-chess/src/agent/evals/chess-commentary.eval.ts`
+* After: `plugins/airi-plugin-game-chess/src/agent/evals/chess-commentary.eval.ts`
+*
+* Before: `D:/repo/plugins/airi-plugin-game-chess/src/agent/evals/chess-commentary.eval.ts`
+* After: `D:/repo/plugins/airi-plugin-game-chess/src/agent/evals/chess-commentary.eval.ts`
+*/
+function asProjectRelativePath(filePath, context) {
+	const normalizedFilePath = normalizePath(filePath);
+	const normalizedProjectRootDirectory = normalizePath(context.projectRootDirectory);
+	const filePathWindowsDrive = normalizedFilePath.match(/^[A-Z]:\//i)?.[0];
+	const projectRootWindowsDrive = normalizedProjectRootDirectory.match(/^[A-Z]:\//i)?.[0];
+	if (filePathWindowsDrive != null && projectRootWindowsDrive == null) return normalizedFilePath;
+	if (filePathWindowsDrive != null && projectRootWindowsDrive != null && filePathWindowsDrive.toLowerCase() !== projectRootWindowsDrive.toLowerCase()) return normalizedFilePath;
+	const projectRootDirectory = context.projectRootDirectory;
+	const relativeFilePath = normalizePath(relative(projectRootDirectory, filePath));
+	if (!absolutePathPattern.test(relativeFilePath)) {
+		if (relativeFilePath === "..") return normalizePath(filePath);
+		if (!relativeFilePath.startsWith("../")) return relativeFilePath;
+	}
+	return normalizePath(filePath);
+}
+function resolveModuleFilePath(moduleHref) {
+	if (!moduleHref.startsWith("file:")) return null;
+	try {
+		return fileURLToPath(moduleHref);
+	} catch {
+		return null;
+	}
+}
+function createCollectedEvalEntry(moduleHref, moduleDefinition, context) {
+	const filePath = resolveModuleFilePath(moduleHref);
+	if (!filePath) return null;
+	const relativeFilePath = asProjectRelativePath(filePath, context);
+	if (!relativeFilePath.endsWith(evalFileSuffix)) return null;
+	const entryName = basename(relativeFilePath, evalFileSuffix);
+	if (entryName.length === 0) return null;
+	const relativeDirectory = dirname(relativeFilePath);
+	const directory = relativeDirectory === "." ? "" : relativeDirectory;
+	return {
+		...moduleDefinition.default,
+		directory,
+		filePath,
+		id: directory.length === 0 ? entryName : `${directory}/${entryName}`,
+		name: entryName
+	};
+}
+/**
+* Collects loaded vieval modules into sorted runner entries with stable ids.
+*
+* Call stack:
+*
+* `import.meta.glob(...)`
+*   -> {@link collectEvalEntries}
+*     -> {@link createCollectedEvalEntry}
+*       -> {@link CollectedEvalEntry}[]
+*
+* Use when:
+* - the runner has already loaded candidate eval modules
+* - downstream scheduling needs stable entry ids and directory metadata
+*/
+function collectEvalEntries(modules, context) {
+	return Object.entries(modules).flatMap(([moduleHref, moduleDefinition]) => {
+		const entry = createCollectedEvalEntry(moduleHref, moduleDefinition, context);
+		if (!entry) return [];
+		return [entry];
+	}).sort((left, right) => left.id.localeCompare(right.id));
+}
+//#endregion
+//#region src/core/runner/run.ts
+function createDefaultExecutionContext(task) {
+	return { model(options) {
+		const requestedModelName = typeof options === "string" ? options : options?.name;
+		if (requestedModelName != null) throw new Error(`No model registry configured. Requested model: ${requestedModelName}`);
+		throw new Error(`No model registry configured for task inferenceExecutor id "${task.inferenceExecutor.id}".`);
+	} };
+}
+/**
+* Error thrown when a scheduled run fails before producing a normalized result.
+*/
+var RunnerExecutionError = class extends Error {
+	/**
+	* Stable task id that failed.
+	*/
+	taskId;
+	constructor(taskId, cause) {
+		const message = errorMessageFrom(cause) ?? "Unknown runner execution failure.";
+		super(`Runner task "${taskId}" failed: ${message}`);
+		this.name = "RunnerExecutionError";
+		this.taskId = taskId;
+		this.cause = cause;
+	}
+};
+function createRunnerExecutionError(taskId, cause) {
+	if (cause instanceof RunnerExecutionError && cause.taskId === taskId) return cause;
+	return new RunnerExecutionError(taskId, cause);
+}
+/**
+* Executes runner tasks sequentially and aggregates the normalized results.
+*
+* Call stack:
+*
+* {@link createRunnerSchedule}
+*   -> {@link runScheduledTasks}
+*     -> `executor(task)`
+*       -> {@link aggregateRunResults}
+*
+* Use when:
+* - the caller already expanded the runner matrix
+* - task execution should stay deterministic and easy to debug
+*
+* Expects:
+* - `executor` to return normalized `0..1` scores
+* - callers to handle concurrency outside this helper when needed
+* - `onTaskStart` / `onTaskEnd` hooks to be synchronous lifecycle observers
+*
+* Throws:
+* - `RunnerExecutionError` when task setup, hooks, or the executor throws
+*/
+async function runScheduledTasks(tasks, executor, options = {}) {
+	if (tasks.length === 0) return aggregateRunResults([]);
+	const results = [];
+	for (const task of tasks) {
+		let executionContext;
+		try {
+			executionContext = options.createExecutionContext?.(task) ?? createDefaultExecutionContext(task);
+		} catch (error) {
+			throw createRunnerExecutionError(task.id, error);
+		}
+		try {
+			options.onTaskStart?.(task);
+		} catch (error) {
+			throw createRunnerExecutionError(task.id, error);
+		}
+		try {
+			results.push(await executor(task, executionContext));
+		} catch (error) {
+			try {
+				options.onTaskEnd?.(task, "failed");
+			} catch {}
+			throw createRunnerExecutionError(task.id, error);
+		}
+		try {
+			options.onTaskEnd?.(task, "passed");
+		} catch (error) {
+			throw createRunnerExecutionError(task.id, error);
+		}
+	}
+	return aggregateRunResults(results);
+}
+//#endregion
+//#region src/core/runner/runtime-context.ts
+const require = createRequire(import.meta.url);
+/**
+* Creates a side-effect-free runtime context for runner path normalization.
+*
+* Call stack:
+*
+* {@link createRunnerRuntimeContext}
+*   -> `findWorkspaceDir(cwd)`
+*     -> `resolve projectRootDirectory`
+*       -> `{ projectRootDirectory }`
+*
+* Use when:
+* - initializing runner infrastructure before collecting eval modules
+* - tests need deterministic root resolution behavior
+*/
+async function createRunnerRuntimeContext(options = {}) {
+	const cwd = options.cwd ?? dirname(fileURLToPath(import.meta.url));
+	const fallbackProjectRootDirectory = options.fallbackProjectRootDirectory ?? fileURLToPath(new URL("../../../", import.meta.url));
+	const { findWorkspaceDir } = require("@pnpm/find-workspace-dir");
+	return { projectRootDirectory: await findWorkspaceDir(cwd) ?? fallbackProjectRootDirectory };
+}
+//#endregion
+//#region src/core/runner/schedule.ts
+const matrixLayerKeys = new Set([
+	"disable",
+	"extend",
+	"override"
+]);
+const ambiguousMatrixDefinitionErrorMessage = "Ambiguous matrix definition: cannot mix reserved layer keys (disable, extend, override) with matrix axis keys.";
+function encodeTaskIdSegment(value) {
+	return encodeURIComponent(value);
+}
+function stringifyMatrixValue(value) {
+	return String(value);
+}
+function cloneMatrixSelection(matrix) {
+	return { ...matrix };
+}
+function createScheduledTaskMatrix(runMatrix, evalMatrix) {
+	return {
+		eval: cloneMatrixSelection(evalMatrix),
+		meta: {
+			evalRowId: createStableRowId(evalMatrix),
+			runRowId: createStableRowId(runMatrix)
+		},
+		run: cloneMatrixSelection(runMatrix)
+	};
+}
+function isMatrixLayer(matrix) {
+	const matrixKeys = Object.keys(matrix);
+	return matrixKeys.length > 0 && matrixKeys.every((key) => matrixLayerKeys.has(key));
+}
+function assertNonAmbiguousMatrixDefinition(matrix) {
+	const matrixKeys = Object.keys(matrix);
+	const hasReservedKeys = matrixKeys.some((key) => matrixLayerKeys.has(key));
+	const hasAxisKeys = matrixKeys.some((key) => !matrixLayerKeys.has(key));
+	if (hasReservedKeys && hasAxisKeys) throw new TypeError(ambiguousMatrixDefinitionErrorMessage);
+}
+function normalizeLayerInputToAxes(matrix) {
+	if (matrix == null) return;
+	assertNonAmbiguousMatrixDefinition(matrix);
+	if (isMatrixLayer(matrix)) return matrix;
+	return { extend: matrix };
+}
+function dedupeAxisValues(values) {
+	return Array.from(new Set(values.map(stringifyMatrixValue)));
+}
+function applyAxisValues(axes, definition, mode) {
+	if (definition == null) return;
+	for (const [axis, values] of Object.entries(definition)) {
+		const nextValues = dedupeAxisValues(values);
+		if (mode === "extend") {
+			const existingValues = axes.get(axis) ?? [];
+			axes.set(axis, Array.from(new Set([...existingValues, ...nextValues])));
+			continue;
+		}
+		axes.set(axis, nextValues);
+	}
+}
+function applyLayer(baseAxes, layer) {
+	const nextAxes = new Map(Array.from(baseAxes.entries()).map(([axis, values]) => [axis, [...values]]));
+	for (const axis of layer?.disable ?? []) nextAxes.delete(axis);
+	applyAxisValues(nextAxes, layer?.extend, "extend");
+	applyAxisValues(nextAxes, layer?.override, "override");
+	return nextAxes;
+}
+function expandAxesToRows(axes) {
+	if (axes.size === 0) return [{}];
+	const dimensions = Array.from(axes.entries());
+	let selections = [{}];
+	for (const [axis, values] of dimensions) {
+		if (values.length === 0) return [];
+		const nextSelections = [];
+		for (const selection of selections) for (const value of values) nextSelections.push({
+			...selection,
+			[axis]: value
+		});
+		selections = nextSelections;
+	}
+	return selections;
+}
+function createStableRowId(matrix) {
+	const segments = Object.entries(matrix).sort(([leftAxis], [rightAxis]) => leftAxis.localeCompare(rightAxis)).map(([axis, value]) => `${encodeTaskIdSegment(axis)}=${encodeTaskIdSegment(value)}`);
+	if (segments.length === 0) return "default";
+	return segments.join("&");
+}
+function createTaskId(entryId, inferenceExecutorId, runRowId, evalRowId) {
+	return [
+		encodeTaskIdSegment(entryId),
+		encodeTaskIdSegment(inferenceExecutorId),
+		`run=${encodeTaskIdSegment(runRowId)}`,
+		`eval=${encodeTaskIdSegment(evalRowId)}`
+	].join("::");
+}
+function createResolvedRunAxes(entry, runMatrix) {
+	let resolvedAxes = /* @__PURE__ */ new Map();
+	for (const layerInput of [
+		runMatrix,
+		entry.matrix?.runMatrix,
+		entry.task?.matrix?.runMatrix
+	]) resolvedAxes = applyLayer(resolvedAxes, normalizeLayerInputToAxes(layerInput));
+	return resolvedAxes;
+}
+function createResolvedEvalAxes(entry, evalMatrix) {
+	let resolvedAxes = /* @__PURE__ */ new Map();
+	for (const layerInput of [
+		evalMatrix,
+		entry.matrix?.evalMatrix,
+		entry.task?.matrix?.evalMatrix
+	]) resolvedAxes = applyLayer(resolvedAxes, normalizeLayerInputToAxes(layerInput));
+	return resolvedAxes;
+}
+/**
+* Expands collected entries into a stable runner schedule.
+*
+* Call stack:
+*
+* {@link collectEvalEntries} (`../runner`)
+*   -> {@link createRunnerSchedule}
+*     -> {@link expandAxesToRows}
+*       -> {@link ScheduledTask}[]
+*
+* Use when:
+* - the runner already knows which eval entries are available
+* - each entry must run against multiple inferenceExecutors or matrix variants
+*
+* Expects:
+* - `entries` and `inferenceExecutors` to be provided in the desired execution order
+* - matrix axes to use insertion order when generating combinations
+*/
+function createRunnerSchedule(options) {
+	if (options.entries.length === 0) return [];
+	if (options.inferenceExecutors.length === 0) return [];
+	const tasks = [];
+	for (const entry of options.entries) {
+		const runSelections = expandAxesToRows(createResolvedRunAxes(entry, options.runMatrix));
+		const evalSelections = expandAxesToRows(createResolvedEvalAxes(entry, options.evalMatrix));
+		if (runSelections.length === 0 || evalSelections.length === 0) continue;
+		for (const inferenceExecutor of options.inferenceExecutors) for (const runMatrix of runSelections) for (const evalMatrix of evalSelections) {
+			const isolatedMatrix = createScheduledTaskMatrix(runMatrix, evalMatrix);
+			tasks.push({
+				entry,
+				id: createTaskId(entry.id, inferenceExecutor.id, isolatedMatrix.meta.runRowId, isolatedMatrix.meta.evalRowId),
+				matrix: isolatedMatrix,
+				inferenceExecutor
+			});
+		}
+	}
+	return tasks;
+}
+//#endregion
+//#region src/core/runner/task-context.ts
+function resolveDefaultTaskModel(models, task) {
+	const runMatrixModelName = task.matrix.run.model;
+	if (runMatrixModelName != null) {
+		const matrixSelectedModel = resolveModelByName(models, runMatrixModelName);
+		if (matrixSelectedModel != null) return matrixSelectedModel;
+		throw new Error(`Unknown configured model "${runMatrixModelName}" from task.matrix.run.model.`);
+	}
+	const matched = resolveModelByName(models, task.inferenceExecutor.id);
+	if (matched != null) return matched;
+	if (models.length > 0) {
+		const firstModel = models[0];
+		if (firstModel != null) return firstModel;
+	}
+	throw new Error(`No configured model found for inferenceExecutor id "${task.inferenceExecutor.id}".`);
+}
+/**
+* Creates task-scoped model resolver context for runner execution.
+*
+* Call stack:
+*
+* {@link runScheduledTasks}
+*   -> {@link createTaskExecutionContext}
+*     -> {@link resolveModelByName}
+*       -> `task.model()` / `task.model({ name })`
+*/
+function createTaskExecutionContext(options) {
+	return { model(selection) {
+		if (selection == null) return resolveDefaultTaskModel(options.models, options.task);
+		const name = typeof selection === "string" ? selection : selection.name;
+		const namedModel = resolveModelByName(options.models, name);
+		if (namedModel == null) throw new Error(`Unknown configured model "${name}".`);
+		return namedModel;
+	} };
+}
+//#endregion
+export { runScheduledTasks as a, aggregateRunResults as c, RunnerExecutionError as i, createRunnerSchedule as n, asProjectRelativePath as o, createRunnerRuntimeContext as r, collectEvalEntries as s, createTaskExecutionContext as t };
+
+//# sourceMappingURL=runner-4ZsOveoY.mjs.map

package/dist/runner-4ZsOveoY.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner-4ZsOveoY.mjs","names":[],"sources":["../src/core/runner/aggregate.ts","../src/core/runner/collect.ts","../src/core/runner/run.ts","../src/core/runner/runtime-context.ts","../src/core/runner/schedule.ts","../src/core/runner/task-context.ts"],"sourcesContent":["import type { ScheduledTaskMatrix } from './schedule'\n\n/**\n * Identifies the scoring family for a single eval score.\n */\nexport type RunScoreKind = 'exact' | 'judge'\n\n/**\n * Represents one normalized score emitted by a completed eval run.\n */\nexport interface RunScore {\n  /**\n   * Score family used for aggregation.\n   */\n  kind: RunScoreKind\n  /**\n   * Normalized score in the `0..1` range.\n   */\n  score: number\n}\n\n/**\n * Captures the output of one scheduled runner task.\n */\nexport interface RunResult {\n  /**\n   * Stable run id, usually copied from the scheduled task id.\n   */\n  id: string\n  /**\n   * Collected eval entry id.\n   */\n  entryId: string\n  /**\n   * Stable inferenceExecutor id.\n   */\n  inferenceExecutorId: string\n  /**\n   * Concrete matrix selection used by the run.\n   */\n  matrix: ScheduledTaskMatrix\n  /**\n   * Raw scores emitted by the eval.\n   */\n  scores: readonly RunScore[]\n}\n\n/**\n * Stores the per-run score averages after normalization.\n */\nexport interface AggregatedRunSummary {\n  /**\n   * Stable run id.\n   */\n  id: string\n  /**\n   * Collected eval entry id.\n   */\n  entryId: string\n  /**\n   * Stable inferenceExecutor id.\n   */\n  inferenceExecutorId: string\n  /**\n   * Concrete matrix selection used by the run.\n   */\n  matrix: ScheduledTaskMatrix\n  /**\n   * Mean of exact-match scores or `null` when absent.\n   */\n  exactAverage: number | null\n  /**\n   * Mean of judge-based scores or `null` when absent.\n   */\n  judgeAverage: number | null\n  /**\n   * Hybrid average. Uses both families when present, otherwise falls back to the\n   * single available family.\n   */\n  hybridAverage: number | null\n}\n\n/**\n * Stores inferenceExecutor-level score aggregates across multiple runs.\n */\nexport interface AggregatedProviderSummary {\n  /**\n   * Stable inferenceExecutor id.\n   */\n  inferenceExecutorId: string\n  /**\n   * Number of runs included in this inferenceExecutor bucket.\n   */\n  runCount: number\n  /**\n   * Mean of all exact-match scores or `null` when absent.\n   */\n  exactAverage: number | null\n  /**\n   * Mean of all judge-based scores or `null` when absent.\n   */\n  judgeAverage: number | null\n  /**\n   * Hybrid average derived from the inferenceExecutor exact and judge means.\n   */\n  hybridAverage: number | null\n}\n\n/**\n * Stores the final aggregation output for a batch of runner results.\n */\nexport interface AggregatedRunResults {\n  /**\n   * Per-run normalized score summaries.\n   */\n  runs: AggregatedRunSummary[]\n  /**\n   * Provider-level summaries sorted by inferenceExecutor id.\n   */\n  inferenceExecutors: AggregatedProviderSummary[]\n  /**\n   * Overall summary across every run.\n   */\n  overall: {\n    exactAverage: number | null\n    judgeAverage: number | null\n    hybridAverage: number | null\n    runCount: number\n  }\n}\n\ninterface ScoreBuckets {\n  exact: number[]\n  judge: number[]\n}\n\nfunction cloneScheduledTaskMatrix(matrix: ScheduledTaskMatrix): ScheduledTaskMatrix {\n  return {\n    eval: {\n      ...matrix.eval,\n    },\n    meta: {\n      ...matrix.meta,\n    },\n    run: {\n      ...matrix.run,\n    },\n  }\n}\n\nfunction assertKnownScoreKind(kind: string): RunScoreKind {\n  if (kind === 'exact' || kind === 'judge') {\n    return kind\n  }\n\n  throw new TypeError(`Unknown eval score kind \"${kind}\".`)\n}\n\nfunction average(scores: readonly number[]): number | null {\n  if (scores.length === 0) {\n    return null\n  }\n\n  const total = scores.reduce((sum, score) => sum + score, 0)\n  return total / scores.length\n}\n\nfunction createHybridAverage(exactAverage: number | null, judgeAverage: number | null): number | null {\n  if (exactAverage != null && judgeAverage != null) {\n    return (exactAverage + judgeAverage) / 2\n  }\n\n  if (exactAverage != null) {\n    return exactAverage\n  }\n\n  if (judgeAverage != null) {\n    return judgeAverage\n  }\n\n  return null\n}\n\nfunction collectScoreBuckets(scores: readonly RunScore[]): ScoreBuckets {\n  const buckets: ScoreBuckets = {\n    exact: [],\n    judge: [],\n  }\n\n  for (const score of scores) {\n    const kind = assertKnownScoreKind(score.kind)\n\n    if (kind === 'exact') {\n      buckets.exact.push(score.score)\n      continue\n    }\n\n    buckets.judge.push(score.score)\n  }\n\n  return buckets\n}\n\nfunction createRunSummary(result: RunResult): AggregatedRunSummary {\n  const buckets = collectScoreBuckets(result.scores)\n  const exactAverage = average(buckets.exact)\n  const judgeAverage = average(buckets.judge)\n\n  return {\n    entryId: result.entryId,\n    exactAverage,\n    hybridAverage: createHybridAverage(exactAverage, judgeAverage),\n    id: result.id,\n    judgeAverage,\n    matrix: cloneScheduledTaskMatrix(result.matrix),\n    inferenceExecutorId: result.inferenceExecutorId,\n  }\n}\n\nfunction createProviderSummary(inferenceExecutorId: string, results: readonly RunResult[]): AggregatedProviderSummary {\n  const exactScores: number[] = []\n  const judgeScores: number[] = []\n\n  for (const result of results) {\n    const buckets = collectScoreBuckets(result.scores)\n    exactScores.push(...buckets.exact)\n    judgeScores.push(...buckets.judge)\n  }\n\n  const exactAverage = average(exactScores)\n  const judgeAverage = average(judgeScores)\n\n  return {\n    exactAverage,\n    hybridAverage: createHybridAverage(exactAverage, judgeAverage),\n    judgeAverage,\n    inferenceExecutorId,\n    runCount: results.length,\n  }\n}\n\n/**\n * Aggregates exact-match and judge-based scores into hybrid runner summaries.\n *\n * Call stack:\n *\n * {@link runScheduledTasks}\n *   -> {@link aggregateRunResults}\n *     -> {@link createRunSummary}\n *     -> {@link createProviderSummary}\n *       -> `report output`\n *\n * Use when:\n * - a runner batch mixes deterministic exact checks with judge-based grading\n * - inferenceExecutor comparison should preserve both score families and one hybrid view\n *\n * Expects:\n * - each score to be normalized to the `0..1` range before aggregation\n * - `scores.kind` to use only `'exact'` or `'judge'`\n */\nexport function aggregateRunResults(results: readonly RunResult[]): AggregatedRunResults {\n  const runs = results.map(createRunSummary)\n\n  const inferenceExecutorIds = Array.from(new Set(results.map(result => result.inferenceExecutorId)))\n  const inferenceExecutors = inferenceExecutorIds\n    .map((inferenceExecutorId) => {\n      const providerResults = results.filter(result => result.inferenceExecutorId === inferenceExecutorId)\n      return createProviderSummary(inferenceExecutorId, providerResults)\n    })\n    .sort((left, right) => left.inferenceExecutorId.localeCompare(right.inferenceExecutorId))\n\n  const overall = createProviderSummary(\n    'overall',\n    results,\n  )\n\n  return {\n    overall: {\n      exactAverage: overall.exactAverage,\n      hybridAverage: overall.hybridAverage,\n      judgeAverage: overall.judgeAverage,\n      runCount: overall.runCount,\n    },\n    inferenceExecutors,\n    runs,\n  }\n}\n","import type { CollectedEvalEntry, EvalModule, EvalModuleMap } from '../../config'\nimport type { RunnerRuntimeContext } from './runtime-context'\n\nimport { basename, dirname, relative } from 'node:path'\nimport { fileURLToPath } from 'node:url'\n\nconst evalFileSuffix = '.eval.ts'\nconst absolutePathPattern = /^(?:[A-Z]:\\/|\\/|\\\\\\\\)/i\n\nfunction normalizePath(value: string): string {\n  return value.replaceAll('\\\\', '/')\n}\n\n/**\n * Converts a file path into a project-relative path when possible.\n *\n * Before: `/repo/plugins/airi-plugin-game-chess/src/agent/evals/chess-commentary.eval.ts`\n * After: `plugins/airi-plugin-game-chess/src/agent/evals/chess-commentary.eval.ts`\n *\n * Before: `D:/repo/plugins/airi-plugin-game-chess/src/agent/evals/chess-commentary.eval.ts`\n * After: `D:/repo/plugins/airi-plugin-game-chess/src/agent/evals/chess-commentary.eval.ts`\n */\nexport function asProjectRelativePath(filePath: string, context: RunnerRuntimeContext): string {\n  const normalizedFilePath = normalizePath(filePath)\n  const normalizedProjectRootDirectory = normalizePath(context.projectRootDirectory)\n  const filePathWindowsDrive = normalizedFilePath.match(/^[A-Z]:\\//i)?.[0]\n  const projectRootWindowsDrive = normalizedProjectRootDirectory.match(/^[A-Z]:\\//i)?.[0]\n\n  if (filePathWindowsDrive != null && projectRootWindowsDrive == null) {\n    return normalizedFilePath\n  }\n\n  if (\n    filePathWindowsDrive != null\n    && projectRootWindowsDrive != null\n    && filePathWindowsDrive.toLowerCase() !== projectRootWindowsDrive.toLowerCase()\n  ) {\n    return normalizedFilePath\n  }\n\n  const projectRootDirectory = context.projectRootDirectory\n  const relativeFilePath = normalizePath(relative(projectRootDirectory, filePath))\n\n  if (!absolutePathPattern.test(relativeFilePath)) {\n    if (relativeFilePath === '..') {\n      return normalizePath(filePath)\n    }\n\n    if (!relativeFilePath.startsWith('../')) {\n      return relativeFilePath\n    }\n  }\n\n  return normalizePath(filePath)\n}\n\nfunction resolveModuleFilePath(moduleHref: string): string | null {\n  if (!moduleHref.startsWith('file:')) {\n    return null\n  }\n\n  try {\n    return fileURLToPath(moduleHref)\n  }\n  catch {\n    return null\n  }\n}\n\nfunction createCollectedEvalEntry(\n  moduleHref: string,\n  moduleDefinition: EvalModule,\n  context: RunnerRuntimeContext,\n): CollectedEvalEntry | null {\n  const filePath = resolveModuleFilePath(moduleHref)\n\n  if (!filePath) {\n    return null\n  }\n\n  const relativeFilePath = asProjectRelativePath(filePath, context)\n\n  if (!relativeFilePath.endsWith(evalFileSuffix)) {\n    return null\n  }\n\n  const entryName = basename(relativeFilePath, evalFileSuffix)\n\n  if (entryName.length === 0) {\n    return null\n  }\n\n  const relativeDirectory = dirname(relativeFilePath)\n  const directory = relativeDirectory === '.' ? '' : relativeDirectory\n\n  return {\n    ...moduleDefinition.default,\n    directory,\n    filePath,\n    id: directory.length === 0 ? entryName : `${directory}/${entryName}`,\n    name: entryName,\n  }\n}\n\n/**\n * Collects loaded vieval modules into sorted runner entries with stable ids.\n *\n * Call stack:\n *\n * `import.meta.glob(...)`\n *   -> {@link collectEvalEntries}\n *     -> {@link createCollectedEvalEntry}\n *       -> {@link CollectedEvalEntry}[]\n *\n * Use when:\n * - the runner has already loaded candidate eval modules\n * - downstream scheduling needs stable entry ids and directory metadata\n */\nexport function collectEvalEntries(\n  modules: EvalModuleMap,\n  context: RunnerRuntimeContext,\n): CollectedEvalEntry[] {\n  return Object.entries(modules)\n    .flatMap(([moduleHref, moduleDefinition]) => {\n      const entry = createCollectedEvalEntry(moduleHref, moduleDefinition, context)\n\n      if (!entry) {\n        return []\n      }\n\n      return [entry]\n    })\n    .sort((left, right) => left.id.localeCompare(right.id))\n}\n","import type { AggregatedRunResults, RunResult } from './aggregate'\nimport type { ScheduledTask } from './schedule'\nimport type { TaskExecutionContext } from './task-context'\n\nimport { errorMessageFrom } from '@moeru/std'\n\nimport { aggregateRunResults } from './aggregate'\n\n/**\n * Executes one scheduled runner task and returns a normalized run result.\n *\n * Use when:\n * - a scheduler already selected the task and execution context\n * - the caller wants a typed executor contract for runner workers\n *\n * Expects:\n * - the task context to be ready for model resolution and task-scoped work\n *\n * Returns:\n * - a normalized run result with score entries ready for aggregation\n */\nexport type ScheduledTaskExecutor = (\n  task: ScheduledTask,\n  context: TaskExecutionContext,\n) => Promise<RunResult>\n\n/**\n * Terminal task state reported by runner lifecycle hooks.\n *\n * Use when:\n * - reporting the outcome of one scheduled task to lifecycle observers\n *\n * Expects:\n * - hooks treat the value as final for the completed task\n */\nexport type RunnerTaskState = 'passed' | 'failed'\n\n/**\n * Optional runner execution hooks used while processing scheduled tasks.\n *\n * Use when:\n * - callers want lifecycle visibility around sequential task execution\n * - task execution should remain deterministic while still observable\n *\n * Expects:\n * - hook functions are synchronous lifecycle observers\n */\nexport interface RunScheduledTasksOptions {\n  /**\n   * Creates per-task execution context.\n   *\n   * Use when:\n   * - executor code needs per-task model resolution or other task-scoped data\n   */\n  createExecutionContext?: (task: ScheduledTask) => TaskExecutionContext\n  /**\n   * Runs before the executor starts handling a task.\n   *\n   * Use when:\n   * - callers want to observe task activation before execution begins\n   *\n   * Expects:\n   * - thrown errors abort the task before executor work starts\n   */\n  onTaskStart?: (task: ScheduledTask) => void\n  /**\n   * Runs after the executor settles for a task.\n   *\n   * Use when:\n   * - callers want to observe successful and failed task completion\n   *\n   * Expects:\n   * - thrown errors abort successful runs\n   * - failed-task observers do not override the executor error for the task\n   */\n  onTaskEnd?: (task: ScheduledTask, state: RunnerTaskState) => void\n}\n\nfunction createDefaultExecutionContext(task: ScheduledTask): TaskExecutionContext {\n  return {\n    model(options) {\n      const requestedModelName = typeof options === 'string' ? options : options?.name\n      if (requestedModelName != null) {\n        throw new Error(`No model registry configured. Requested model: ${requestedModelName}`)\n      }\n\n      throw new Error(`No model registry configured for task inferenceExecutor id \"${task.inferenceExecutor.id}\".`)\n    },\n  }\n}\n\n/**\n * Error thrown when a scheduled run fails before producing a normalized result.\n */\nexport class RunnerExecutionError extends Error {\n  /**\n   * Stable task id that failed.\n   */\n  taskId: string\n\n  constructor(taskId: string, cause: unknown) {\n    const message = errorMessageFrom(cause) ?? 'Unknown runner execution failure.'\n    super(`Runner task \"${taskId}\" failed: ${message}`)\n    this.name = 'RunnerExecutionError'\n    this.taskId = taskId\n    this.cause = cause\n  }\n}\n\nfunction createRunnerExecutionError(taskId: string, cause: unknown): RunnerExecutionError {\n  if (cause instanceof RunnerExecutionError && cause.taskId === taskId) {\n    return cause\n  }\n\n  return new RunnerExecutionError(taskId, cause)\n}\n\n/**\n * Executes runner tasks sequentially and aggregates the normalized results.\n *\n * Call stack:\n *\n * {@link createRunnerSchedule}\n *   -> {@link runScheduledTasks}\n *     -> `executor(task)`\n *       -> {@link aggregateRunResults}\n *\n * Use when:\n * - the caller already expanded the runner matrix\n * - task execution should stay deterministic and easy to debug\n *\n * Expects:\n * - `executor` to return normalized `0..1` scores\n * - callers to handle concurrency outside this helper when needed\n * - `onTaskStart` / `onTaskEnd` hooks to be synchronous lifecycle observers\n *\n * Throws:\n * - `RunnerExecutionError` when task setup, hooks, or the executor throws\n */\nexport async function runScheduledTasks(\n  tasks: readonly ScheduledTask[],\n  executor: ScheduledTaskExecutor,\n  options: RunScheduledTasksOptions = {},\n): Promise<AggregatedRunResults> {\n  if (tasks.length === 0) {\n    return aggregateRunResults([])\n  }\n\n  const results: RunResult[] = []\n\n  for (const task of tasks) {\n    let executionContext: TaskExecutionContext\n\n    try {\n      executionContext = options.createExecutionContext?.(task) ?? createDefaultExecutionContext(task)\n    }\n    catch (error) {\n      throw createRunnerExecutionError(task.id, error)\n    }\n\n    try {\n      options.onTaskStart?.(task)\n    }\n    catch (error) {\n      throw createRunnerExecutionError(task.id, error)\n    }\n\n    try {\n      results.push(await executor(task, executionContext))\n    }\n    catch (error) {\n      try {\n        options.onTaskEnd?.(task, 'failed')\n      }\n      catch {\n        // Failed-task observers must not mask the task execution failure.\n      }\n      throw createRunnerExecutionError(task.id, error)\n    }\n\n    try {\n      options.onTaskEnd?.(task, 'passed')\n    }\n    catch (error) {\n      throw createRunnerExecutionError(task.id, error)\n    }\n  }\n\n  return aggregateRunResults(results)\n}\n","import { createRequire } from 'node:module'\nimport { dirname } from 'node:path'\nimport { fileURLToPath } from 'node:url'\n\nconst require = createRequire(import.meta.url)\n\n/**\n * Shared runtime context used by the vieval runner.\n *\n * Use when:\n * - runner services need stable path resolution without module-level side effects\n * - call sites want deterministic control over workspace root detection\n */\nexport interface RunnerRuntimeContext {\n  /**\n   * Absolute project root directory used for path normalization.\n   */\n  projectRootDirectory: string\n}\n\n/**\n * Options used to construct the runner runtime context.\n */\nexport interface CreateVievalRunnerRuntimeContextOptions {\n  /**\n   * Directory used to search for the nearest pnpm workspace.\n   *\n   * @default directory of this module file\n   */\n  cwd?: string\n  /**\n   * Absolute fallback directory when a pnpm workspace root is not found.\n   *\n   * @default package root directory (`packages/vieval`)\n   */\n  fallbackProjectRootDirectory?: string\n}\n\n/**\n * Creates a side-effect-free runtime context for runner path normalization.\n *\n * Call stack:\n *\n * {@link createRunnerRuntimeContext}\n *   -> `findWorkspaceDir(cwd)`\n *     -> `resolve projectRootDirectory`\n *       -> `{ projectRootDirectory }`\n *\n * Use when:\n * - initializing runner infrastructure before collecting eval modules\n * - tests need deterministic root resolution behavior\n */\nexport async function createRunnerRuntimeContext(\n  options: CreateVievalRunnerRuntimeContextOptions = {},\n): Promise<RunnerRuntimeContext> {\n  const cwd = options.cwd ?? dirname(fileURLToPath(import.meta.url))\n  const fallbackProjectRootDirectory = options.fallbackProjectRootDirectory\n    ?? fileURLToPath(new URL('../../../', import.meta.url))\n\n  // NOTICE:\n  // We use dynamic `require` here because `@pnpm/find-workspace-dir` is CommonJS.\n  // Keeping this load inside the factory avoids module-level initialization side effects.\n  const { findWorkspaceDir } = require('@pnpm/find-workspace-dir') as {\n    findWorkspaceDir: (currentWorkingDirectory: string) => Promise<string | undefined>\n  }\n\n  // NOTICE:\n  // Workspace discovery is required to keep collected eval ids stable when this\n  // package is moved inside different monorepo layouts.\n  const workspaceDirectory = await findWorkspaceDir(cwd)\n\n  return {\n    projectRootDirectory: workspaceDirectory ?? fallbackProjectRootDirectory,\n  }\n}\n","import type { CollectedEvalEntry, MatrixDefinition, MatrixLayer, MatrixValue } from '../../config'\n\n/**\n * Describes the inferenceExecutor target for a scheduled eval run.\n */\nexport interface InferenceExecutor {\n  /**\n   * Stable inferenceExecutor identifier such as `openai:gpt-4.1-mini`.\n   */\n  id: string\n}\n\n/**\n * Stores the selected value for each matrix axis.\n */\nexport type RunnerMatrixSelection = Record<string, string>\n\n/**\n * Stores stable row ids for one resolved scheduled task matrix.\n */\nexport interface ScheduledTaskMatrixMeta {\n  /**\n   * Stable row id for the resolved run matrix selection.\n   */\n  runRowId: string\n  /**\n   * Stable row id for the resolved eval matrix selection.\n   */\n  evalRowId: string\n}\n\n/**\n * Stores the structured matrix payload for one scheduled task.\n */\nexport interface ScheduledTaskMatrix {\n  /**\n   * Runtime matrix selection visible to task code.\n   */\n  run: RunnerMatrixSelection\n  /**\n   * Eval-time matrix selection visible to task code.\n   */\n  eval: RunnerMatrixSelection\n  /**\n   * Stable row ids for both scopes.\n   */\n  meta: ScheduledTaskMatrixMeta\n}\n\n/**\n * Maps matrix axis names to the values that should be expanded.\n */\nexport type RunnerMatrixDefinition = MatrixDefinition\n\n/**\n * Accepts either flat axis definitions or one layered matrix object.\n */\nexport type RunnerMatrixInput = RunnerMatrixDefinition | MatrixLayer\n\nconst matrixLayerKeys = new Set(['disable', 'extend', 'override'])\nconst ambiguousMatrixDefinitionErrorMessage = 'Ambiguous matrix definition: cannot mix reserved layer keys (disable, extend, override) with matrix axis keys.'\n\n/**\n * Represents one fully expanded runner task.\n */\nexport interface ScheduledTask {\n  /**\n   * Stable task id derived from the entry, inferenceExecutor, and matrix selection.\n   */\n  id: string\n  /**\n   * The collected eval entry to execute.\n   */\n  entry: CollectedEvalEntry\n  /**\n   * The inferenceExecutor selected for this task.\n   */\n  inferenceExecutor: InferenceExecutor\n  /**\n   * The concrete scoped matrix selection for this task.\n   */\n  matrix: ScheduledTaskMatrix\n}\n\n/**\n * Configures how the runner should expand its execution matrix.\n */\nexport interface CreateRunnerScheduleOptions {\n  /**\n   * Collected eval entries that should be scheduled.\n   */\n  entries: readonly CollectedEvalEntry[]\n  /**\n   * Providers that should run each entry.\n   */\n  inferenceExecutors: readonly InferenceExecutor[]\n  /**\n   * Optional run-time matrix axes expanded as a cartesian product.\n   */\n  runMatrix?: RunnerMatrixInput\n  /**\n   * Optional eval-time matrix axes expanded as a cartesian product.\n   */\n  evalMatrix?: RunnerMatrixInput\n}\n\nfunction encodeTaskIdSegment(value: string): string {\n  return encodeURIComponent(value)\n}\n\nfunction stringifyMatrixValue(value: MatrixValue): string {\n  return String(value)\n}\n\nfunction cloneMatrixSelection(matrix: RunnerMatrixSelection): RunnerMatrixSelection {\n  return { ...matrix }\n}\n\nfunction createScheduledTaskMatrix(\n  runMatrix: RunnerMatrixSelection,\n  evalMatrix: RunnerMatrixSelection,\n): ScheduledTaskMatrix {\n  return {\n    eval: cloneMatrixSelection(evalMatrix),\n    meta: {\n      evalRowId: createStableRowId(evalMatrix),\n      runRowId: createStableRowId(runMatrix),\n    },\n    run: cloneMatrixSelection(runMatrix),\n  }\n}\n\nfunction isMatrixLayer(matrix: RunnerMatrixInput): matrix is MatrixLayer {\n  const matrixKeys = Object.keys(matrix)\n  return (\n    matrixKeys.length > 0\n    && matrixKeys.every(key => matrixLayerKeys.has(key))\n  )\n}\n\nfunction assertNonAmbiguousMatrixDefinition(matrix: RunnerMatrixInput): void {\n  const matrixKeys = Object.keys(matrix)\n  const hasReservedKeys = matrixKeys.some(key => matrixLayerKeys.has(key))\n  const hasAxisKeys = matrixKeys.some(key => !matrixLayerKeys.has(key))\n\n  if (hasReservedKeys && hasAxisKeys) {\n    throw new TypeError(ambiguousMatrixDefinitionErrorMessage)\n  }\n}\n\nfunction normalizeLayerInputToAxes(matrix: RunnerMatrixInput | undefined): MatrixLayer | undefined {\n  if (matrix == null) {\n    return undefined\n  }\n\n  assertNonAmbiguousMatrixDefinition(matrix)\n\n  if (isMatrixLayer(matrix)) {\n    return matrix\n  }\n\n  return {\n    extend: matrix,\n  }\n}\n\nfunction dedupeAxisValues(values: readonly MatrixValue[]): string[] {\n  return Array.from(new Set(values.map(stringifyMatrixValue)))\n}\n\nfunction applyAxisValues(\n  axes: Map<string, string[]>,\n  definition: RunnerMatrixDefinition | undefined,\n  mode: 'extend' | 'override',\n): void {\n  if (definition == null) {\n    return\n  }\n\n  for (const [axis, values] of Object.entries(definition)) {\n    const nextValues = dedupeAxisValues(values)\n\n    if (mode === 'extend') {\n      const existingValues = axes.get(axis) ?? []\n      axes.set(axis, Array.from(new Set([...existingValues, ...nextValues])))\n      continue\n    }\n\n    axes.set(axis, nextValues)\n  }\n}\n\nfunction applyLayer(\n  baseAxes: ReadonlyMap<string, string[]>,\n  layer: MatrixLayer | undefined,\n): Map<string, string[]> {\n  const nextAxes = new Map<string, string[]>(\n    Array.from(baseAxes.entries()).map(([axis, values]) => [axis, [...values]]),\n  )\n\n  for (const axis of layer?.disable ?? []) {\n    nextAxes.delete(axis)\n  }\n\n  applyAxisValues(nextAxes, layer?.extend, 'extend')\n  applyAxisValues(nextAxes, layer?.override, 'override')\n\n  return nextAxes\n}\n\nfunction expandAxesToRows(axes: ReadonlyMap<string, readonly string[]>): RunnerMatrixSelection[] {\n  if (axes.size === 0) {\n    return [{}]\n  }\n\n  const dimensions = Array.from(axes.entries())\n\n  let selections: RunnerMatrixSelection[] = [{}]\n\n  for (const [axis, values] of dimensions) {\n    if (values.length === 0) {\n      return []\n    }\n\n    const nextSelections: RunnerMatrixSelection[] = []\n\n    for (const selection of selections) {\n      for (const value of values) {\n        nextSelections.push({\n          ...selection,\n          [axis]: value,\n        })\n      }\n    }\n\n    selections = nextSelections\n  }\n\n  return selections\n}\n\nfunction createStableRowId(matrix: RunnerMatrixSelection): string {\n  const segments = Object.entries(matrix)\n    .sort(([leftAxis], [rightAxis]) => leftAxis.localeCompare(rightAxis))\n    .map(([axis, value]) => `${encodeTaskIdSegment(axis)}=${encodeTaskIdSegment(value)}`)\n\n  if (segments.length === 0) {\n    return 'default'\n  }\n\n  return segments.join('&')\n}\n\nfunction createTaskId(entryId: string, inferenceExecutorId: string, runRowId: string, evalRowId: string): string {\n  const encodedEntryId = encodeTaskIdSegment(entryId)\n  const encodedProviderId = encodeTaskIdSegment(inferenceExecutorId)\n\n  return [\n    encodedEntryId,\n    encodedProviderId,\n    `run=${encodeTaskIdSegment(runRowId)}`,\n    `eval=${encodeTaskIdSegment(evalRowId)}`,\n  ].join('::')\n}\n\nfunction createResolvedRunAxes(\n  entry: CollectedEvalEntry,\n  runMatrix: RunnerMatrixInput | undefined,\n): Map<string, string[]> {\n  let resolvedAxes = new Map<string, string[]>()\n\n  for (const layerInput of [\n    runMatrix,\n    entry.matrix?.runMatrix,\n    entry.task?.matrix?.runMatrix,\n  ]) {\n    resolvedAxes = applyLayer(resolvedAxes, normalizeLayerInputToAxes(layerInput))\n  }\n\n  return resolvedAxes\n}\n\nfunction createResolvedEvalAxes(\n  entry: CollectedEvalEntry,\n  evalMatrix: RunnerMatrixInput | undefined,\n): Map<string, string[]> {\n  let resolvedAxes = new Map<string, string[]>()\n\n  for (const layerInput of [\n    evalMatrix,\n    entry.matrix?.evalMatrix,\n    entry.task?.matrix?.evalMatrix,\n  ]) {\n    resolvedAxes = applyLayer(resolvedAxes, normalizeLayerInputToAxes(layerInput))\n  }\n\n  return resolvedAxes\n}\n\n/**\n * Expands collected entries into a stable runner schedule.\n *\n * Call stack:\n *\n * {@link collectEvalEntries} (`../runner`)\n *   -> {@link createRunnerSchedule}\n *     -> {@link expandAxesToRows}\n *       -> {@link ScheduledTask}[]\n *\n * Use when:\n * - the runner already knows which eval entries are available\n * - each entry must run against multiple inferenceExecutors or matrix variants\n *\n * Expects:\n * - `entries` and `inferenceExecutors` to be provided in the desired execution order\n * - matrix axes to use insertion order when generating combinations\n */\nexport function createRunnerSchedule(options: CreateRunnerScheduleOptions): ScheduledTask[] {\n  if (options.entries.length === 0) {\n    return []\n  }\n\n  if (options.inferenceExecutors.length === 0) {\n    return []\n  }\n\n  const tasks: ScheduledTask[] = []\n\n  for (const entry of options.entries) {\n    const runSelections = expandAxesToRows(createResolvedRunAxes(entry, options.runMatrix))\n    const evalSelections = expandAxesToRows(createResolvedEvalAxes(entry, options.evalMatrix))\n\n    if (runSelections.length === 0 || evalSelections.length === 0) {\n      continue\n    }\n\n    for (const inferenceExecutor of options.inferenceExecutors) {\n      for (const runMatrix of runSelections) {\n        for (const evalMatrix of evalSelections) {\n          const isolatedMatrix = createScheduledTaskMatrix(runMatrix, evalMatrix)\n\n          tasks.push({\n            entry,\n            id: createTaskId(\n              entry.id,\n              inferenceExecutor.id,\n              isolatedMatrix.meta.runRowId,\n              isolatedMatrix.meta.evalRowId,\n            ),\n            matrix: isolatedMatrix,\n            inferenceExecutor,\n          })\n        }\n      }\n    }\n  }\n\n  return tasks\n}\n","import type { ModelDefinition } from '../../config/models'\nimport type { ScheduledTask } from './schedule'\n\nimport { resolveModelByName } from '../../config/models'\n\n/**\n * Options for selecting a model from the execution context.\n */\nexport interface TaskModelSelectionOptions {\n  /**\n   * Model id or alias name.\n   */\n  name: string\n}\n\n/**\n * Task-scoped execution context exposed to runner executors.\n */\nexport interface TaskExecutionContext {\n  /**\n   * Resolves model configuration for the current task.\n   *\n   * Use when:\n   * - no arguments are provided to use the model selected by run matrix/inferenceExecutor\n   * - `name` is provided to resolve a specific model id or alias\n   */\n  model: (\n    selection?: string | TaskModelSelectionOptions,\n  ) => ModelDefinition\n}\n\n/**\n * Inputs used to build task execution context.\n */\nexport interface CreateTaskExecutionContextOptions {\n  models: readonly ModelDefinition[]\n  task: ScheduledTask\n}\n\nfunction resolveDefaultTaskModel(\n  models: readonly ModelDefinition[],\n  task: ScheduledTask,\n): ModelDefinition {\n  const runMatrixModelName = task.matrix.run.model\n  if (runMatrixModelName != null) {\n    const matrixSelectedModel = resolveModelByName(models, runMatrixModelName)\n    if (matrixSelectedModel != null) {\n      return matrixSelectedModel\n    }\n\n    throw new Error(`Unknown configured model \"${runMatrixModelName}\" from task.matrix.run.model.`)\n  }\n\n  const matched = resolveModelByName(models, task.inferenceExecutor.id)\n  if (matched != null) {\n    return matched\n  }\n\n  if (models.length > 0) {\n    const firstModel = models[0]\n    if (firstModel != null) {\n      return firstModel\n    }\n  }\n\n  throw new Error(`No configured model found for inferenceExecutor id \"${task.inferenceExecutor.id}\".`)\n}\n\n/**\n * Creates task-scoped model resolver context for runner execution.\n *\n * Call stack:\n *\n * {@link runScheduledTasks}\n *   -> {@link createTaskExecutionContext}\n *     -> {@link resolveModelByName}\n *       -> `task.model()` / `task.model({ name })`\n */\nexport function createTaskExecutionContext(options: CreateTaskExecutionContextOptions): TaskExecutionContext {\n  return {\n    model(selection) {\n      if (selection == null) {\n        return resolveDefaultTaskModel(options.models, options.task)\n      }\n\n      const name = typeof selection === 'string' ? selection : selection.name\n\n      const namedModel = resolveModelByName(options.models, name)\n      if (namedModel == null) {\n        throw new Error(`Unknown configured model \"${name}\".`)\n      }\n\n      return namedModel\n    },\n  }\n}\n"],"mappings":";;;;;;AAwIA,SAAS,yBAAyB,QAAkD;AAClF,QAAO;EACL,MAAM,EACJ,GAAG,OAAO,MACX;EACD,MAAM,EACJ,GAAG,OAAO,MACX;EACD,KAAK,EACH,GAAG,OAAO,KACX;EACF;;AAGH,SAAS,qBAAqB,MAA4B;AACxD,KAAI,SAAS,WAAW,SAAS,QAC/B,QAAO;AAGT,OAAM,IAAI,UAAU,4BAA4B,KAAK,IAAI;;AAG3D,SAAS,QAAQ,QAA0C;AACzD,KAAI,OAAO,WAAW,EACpB,QAAO;AAIT,QADc,OAAO,QAAQ,KAAK,UAAU,MAAM,OAAO,EAAE,GAC5C,OAAO;;AAGxB,SAAS,oBAAoB,cAA6B,cAA4C;AACpG,KAAI,gBAAgB,QAAQ,gBAAgB,KAC1C,SAAQ,eAAe,gBAAgB;AAGzC,KAAI,gBAAgB,KAClB,QAAO;AAGT,KAAI,gBAAgB,KAClB,QAAO;AAGT,QAAO;;AAGT,SAAS,oBAAoB,QAA2C;CACtE,MAAM,UAAwB;EAC5B,OAAO,EAAE;EACT,OAAO,EAAE;EACV;AAED,MAAK,MAAM,SAAS,QAAQ;AAG1B,MAFa,qBAAqB,MAAM,KAAK,KAEhC,SAAS;AACpB,WAAQ,MAAM,KAAK,MAAM,MAAM;AAC/B;;AAGF,UAAQ,MAAM,KAAK,MAAM,MAAM;;AAGjC,QAAO;;AAGT,SAAS,iBAAiB,QAAyC;CACjE,MAAM,UAAU,oBAAoB,OAAO,OAAO;CAClD,MAAM,eAAe,QAAQ,QAAQ,MAAM;CAC3C,MAAM,eAAe,QAAQ,QAAQ,MAAM;AAE3C,QAAO;EACL,SAAS,OAAO;EAChB;EACA,eAAe,oBAAoB,cAAc,aAAa;EAC9D,IAAI,OAAO;EACX;EACA,QAAQ,yBAAyB,OAAO,OAAO;EAC/C,qBAAqB,OAAO;EAC7B;;AAGH,SAAS,sBAAsB,qBAA6B,SAA0D;CACpH,MAAM,cAAwB,EAAE;CAChC,MAAM,cAAwB,EAAE;AAEhC,MAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,UAAU,oBAAoB,OAAO,OAAO;AAClD,cAAY,KAAK,GAAG,QAAQ,MAAM;AAClC,cAAY,KAAK,GAAG,QAAQ,MAAM;;CAGpC,MAAM,eAAe,QAAQ,YAAY;CACzC,MAAM,eAAe,QAAQ,YAAY;AAEzC,QAAO;EACL;EACA,eAAe,oBAAoB,cAAc,aAAa;EAC9D;EACA;EACA,UAAU,QAAQ;EACnB;;;;;;;;;;;;;;;;;;;;;AAsBH,SAAgB,oBAAoB,SAAqD;CACvF,MAAM,OAAO,QAAQ,IAAI,iBAAiB;CAG1C,MAAM,qBADuB,MAAM,KAAK,IAAI,IAAI,QAAQ,KAAI,WAAU,OAAO,oBAAoB,CAAC,CAAC,CAEhG,KAAK,wBAAwB;AAE5B,SAAO,sBAAsB,qBADL,QAAQ,QAAO,WAAU,OAAO,wBAAwB,oBAAoB,CAClC;GAClE,CACD,MAAM,MAAM,UAAU,KAAK,oBAAoB,cAAc,MAAM,oBAAoB,CAAC;CAE3F,MAAM,UAAU,sBACd,WACA,QACD;AAED,QAAO;EACL,SAAS;GACP,cAAc,QAAQ;GACtB,eAAe,QAAQ;GACvB,cAAc,QAAQ;GACtB,UAAU,QAAQ;GACnB;EACD;EACA;EACD;;;;ACvRH,MAAM,iBAAiB;AACvB,MAAM,sBAAsB;AAE5B,SAAS,cAAc,OAAuB;AAC5C,QAAO,MAAM,WAAW,MAAM,IAAI;;;;;;;;;;;AAYpC,SAAgB,sBAAsB,UAAkB,SAAuC;CAC7F,MAAM,qBAAqB,cAAc,SAAS;CAClD,MAAM,iCAAiC,cAAc,QAAQ,qBAAqB;CAClF,MAAM,uBAAuB,mBAAmB,MAAM,aAAa,GAAG;CACtE,MAAM,0BAA0B,+BAA+B,MAAM,aAAa,GAAG;AAErF,KAAI,wBAAwB,QAAQ,2BAA2B,KAC7D,QAAO;AAGT,KACE,wBAAwB,QACrB,2BAA2B,QAC3B,qBAAqB,aAAa,KAAK,wBAAwB,aAAa,CAE/E,QAAO;CAGT,MAAM,uBAAuB,QAAQ;CACrC,MAAM,mBAAmB,cAAc,SAAS,sBAAsB,SAAS,CAAC;AAEhF,KAAI,CAAC,oBAAoB,KAAK,iBAAiB,EAAE;AAC/C,MAAI,qBAAqB,KACvB,QAAO,cAAc,SAAS;AAGhC,MAAI,CAAC,iBAAiB,WAAW,MAAM,CACrC,QAAO;;AAIX,QAAO,cAAc,SAAS;;AAGhC,SAAS,sBAAsB,YAAmC;AAChE,KAAI,CAAC,WAAW,WAAW,QAAQ,CACjC,QAAO;AAGT,KAAI;AACF,SAAO,cAAc,WAAW;SAE5B;AACJ,SAAO;;;AAIX,SAAS,yBACP,YACA,kBACA,SAC2B;CAC3B,MAAM,WAAW,sBAAsB,WAAW;AAElD,KAAI,CAAC,SACH,QAAO;CAGT,MAAM,mBAAmB,sBAAsB,UAAU,QAAQ;AAEjE,KAAI,CAAC,iBAAiB,SAAS,eAAe,CAC5C,QAAO;CAGT,MAAM,YAAY,SAAS,kBAAkB,eAAe;AAE5D,KAAI,UAAU,WAAW,EACvB,QAAO;CAGT,MAAM,oBAAoB,QAAQ,iBAAiB;CACnD,MAAM,YAAY,sBAAsB,MAAM,KAAK;AAEnD,QAAO;EACL,GAAG,iBAAiB;EACpB;EACA;EACA,IAAI,UAAU,WAAW,IAAI,YAAY,GAAG,UAAU,GAAG;EACzD,MAAM;EACP;;;;;;;;;;;;;;;;AAiBH,SAAgB,mBACd,SACA,SACsB;AACtB,QAAO,OAAO,QAAQ,QAAQ,CAC3B,SAAS,CAAC,YAAY,sBAAsB;EAC3C,MAAM,QAAQ,yBAAyB,YAAY,kBAAkB,QAAQ;AAE7E,MAAI,CAAC,MACH,QAAO,EAAE;AAGX,SAAO,CAAC,MAAM;GACd,CACD,MAAM,MAAM,UAAU,KAAK,GAAG,cAAc,MAAM,GAAG,CAAC;;;;ACtD3D,SAAS,8BAA8B,MAA2C;AAChF,QAAO,EACL,MAAM,SAAS;EACb,MAAM,qBAAqB,OAAO,YAAY,WAAW,UAAU,SAAS;AAC5E,MAAI,sBAAsB,KACxB,OAAM,IAAI,MAAM,kDAAkD,qBAAqB;AAGzF,QAAM,IAAI,MAAM,+DAA+D,KAAK,kBAAkB,GAAG,IAAI;IAEhH;;;;;AAMH,IAAa,uBAAb,cAA0C,MAAM;;;;CAI9C;CAEA,YAAY,QAAgB,OAAgB;EAC1C,MAAM,UAAU,iBAAiB,MAAM,IAAI;AAC3C,QAAM,gBAAgB,OAAO,YAAY,UAAU;AACnD,OAAK,OAAO;AACZ,OAAK,SAAS;AACd,OAAK,QAAQ;;;AAIjB,SAAS,2BAA2B,QAAgB,OAAsC;AACxF,KAAI,iBAAiB,wBAAwB,MAAM,WAAW,OAC5D,QAAO;AAGT,QAAO,IAAI,qBAAqB,QAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;AAyBhD,eAAsB,kBACpB,OACA,UACA,UAAoC,EAAE,EACP;AAC/B,KAAI,MAAM,WAAW,EACnB,QAAO,oBAAoB,EAAE,CAAC;CAGhC,MAAM,UAAuB,EAAE;AAE/B,MAAK,MAAM,QAAQ,OAAO;EACxB,IAAI;AAEJ,MAAI;AACF,sBAAmB,QAAQ,yBAAyB,KAAK,IAAI,8BAA8B,KAAK;WAE3F,OAAO;AACZ,SAAM,2BAA2B,KAAK,IAAI,MAAM;;AAGlD,MAAI;AACF,WAAQ,cAAc,KAAK;WAEtB,OAAO;AACZ,SAAM,2BAA2B,KAAK,IAAI,MAAM;;AAGlD,MAAI;AACF,WAAQ,KAAK,MAAM,SAAS,MAAM,iBAAiB,CAAC;WAE/C,OAAO;AACZ,OAAI;AACF,YAAQ,YAAY,MAAM,SAAS;WAE/B;AAGN,SAAM,2BAA2B,KAAK,IAAI,MAAM;;AAGlD,MAAI;AACF,WAAQ,YAAY,MAAM,SAAS;WAE9B,OAAO;AACZ,SAAM,2BAA2B,KAAK,IAAI,MAAM;;;AAIpD,QAAO,oBAAoB,QAAQ;;;;ACxLrC,MAAM,UAAU,cAAc,OAAO,KAAK,IAAI;;;;;;;;;;;;;;;AAgD9C,eAAsB,2BACpB,UAAmD,EAAE,EACtB;CAC/B,MAAM,MAAM,QAAQ,OAAO,QAAQ,cAAc,OAAO,KAAK,IAAI,CAAC;CAClE,MAAM,+BAA+B,QAAQ,gCACxC,cAAc,IAAI,IAAI,aAAa,OAAO,KAAK,IAAI,CAAC;CAKzD,MAAM,EAAE,qBAAqB,QAAQ,2BAA2B;AAShE,QAAO,EACL,sBAHyB,MAAM,iBAAiB,IAAI,IAGR,8BAC7C;;;;ACdH,MAAM,kBAAkB,IAAI,IAAI;CAAC;CAAW;CAAU;CAAW,CAAC;AAClE,MAAM,wCAAwC;AA8C9C,SAAS,oBAAoB,OAAuB;AAClD,QAAO,mBAAmB,MAAM;;AAGlC,SAAS,qBAAqB,OAA4B;AACxD,QAAO,OAAO,MAAM;;AAGtB,SAAS,qBAAqB,QAAsD;AAClF,QAAO,EAAE,GAAG,QAAQ;;AAGtB,SAAS,0BACP,WACA,YACqB;AACrB,QAAO;EACL,MAAM,qBAAqB,WAAW;EACtC,MAAM;GACJ,WAAW,kBAAkB,WAAW;GACxC,UAAU,kBAAkB,UAAU;GACvC;EACD,KAAK,qBAAqB,UAAU;EACrC;;AAGH,SAAS,cAAc,QAAkD;CACvE,MAAM,aAAa,OAAO,KAAK,OAAO;AACtC,QACE,WAAW,SAAS,KACjB,WAAW,OAAM,QAAO,gBAAgB,IAAI,IAAI,CAAC;;AAIxD,SAAS,mCAAmC,QAAiC;CAC3E,MAAM,aAAa,OAAO,KAAK,OAAO;CACtC,MAAM,kBAAkB,WAAW,MAAK,QAAO,gBAAgB,IAAI,IAAI,CAAC;CACxE,MAAM,cAAc,WAAW,MAAK,QAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC;AAErE,KAAI,mBAAmB,YACrB,OAAM,IAAI,UAAU,sCAAsC;;AAI9D,SAAS,0BAA0B,QAAgE;AACjG,KAAI,UAAU,KACZ;AAGF,oCAAmC,OAAO;AAE1C,KAAI,cAAc,OAAO,CACvB,QAAO;AAGT,QAAO,EACL,QAAQ,QACT;;AAGH,SAAS,iBAAiB,QAA0C;AAClE,QAAO,MAAM,KAAK,IAAI,IAAI,OAAO,IAAI,qBAAqB,CAAC,CAAC;;AAG9D,SAAS,gBACP,MACA,YACA,MACM;AACN,KAAI,cAAc,KAChB;AAGF,MAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,WAAW,EAAE;EACvD,MAAM,aAAa,iBAAiB,OAAO;AAE3C,MAAI,SAAS,UAAU;GACrB,MAAM,iBAAiB,KAAK,IAAI,KAAK,IAAI,EAAE;AAC3C,QAAK,IAAI,MAAM,MAAM,KAAK,IAAI,IAAI,CAAC,GAAG,gBAAgB,GAAG,WAAW,CAAC,CAAC,CAAC;AACvE;;AAGF,OAAK,IAAI,MAAM,WAAW;;;AAI9B,SAAS,WACP,UACA,OACuB;CACvB,MAAM,WAAW,IAAI,IACnB,MAAM,KAAK,SAAS,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,YAAY,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAC5E;AAED,MAAK,MAAM,QAAQ,OAAO,WAAW,EAAE,CACrC,UAAS,OAAO,KAAK;AAGvB,iBAAgB,UAAU,OAAO,QAAQ,SAAS;AAClD,iBAAgB,UAAU,OAAO,UAAU,WAAW;AAEtD,QAAO;;AAGT,SAAS,iBAAiB,MAAuE;AAC/F,KAAI,KAAK,SAAS,EAChB,QAAO,CAAC,EAAE,CAAC;CAGb,MAAM,aAAa,MAAM,KAAK,KAAK,SAAS,CAAC;CAE7C,IAAI,aAAsC,CAAC,EAAE,CAAC;AAE9C,MAAK,MAAM,CAAC,MAAM,WAAW,YAAY;AACvC,MAAI,OAAO,WAAW,EACpB,QAAO,EAAE;EAGX,MAAM,iBAA0C,EAAE;AAElD,OAAK,MAAM,aAAa,WACtB,MAAK,MAAM,SAAS,OAClB,gBAAe,KAAK;GAClB,GAAG;IACF,OAAO;GACT,CAAC;AAIN,eAAa;;AAGf,QAAO;;AAGT,SAAS,kBAAkB,QAAuC;CAChE,MAAM,WAAW,OAAO,QAAQ,OAAO,CACpC,MAAM,CAAC,WAAW,CAAC,eAAe,SAAS,cAAc,UAAU,CAAC,CACpE,KAAK,CAAC,MAAM,WAAW,GAAG,oBAAoB,KAAK,CAAC,GAAG,oBAAoB,MAAM,GAAG;AAEvF,KAAI,SAAS,WAAW,EACtB,QAAO;AAGT,QAAO,SAAS,KAAK,IAAI;;AAG3B,SAAS,aAAa,SAAiB,qBAA6B,UAAkB,WAA2B;AAI/G,QAAO;EAHgB,oBAAoB,QAAQ;EACzB,oBAAoB,oBAAoB;EAKhE,OAAO,oBAAoB,SAAS;EACpC,QAAQ,oBAAoB,UAAU;EACvC,CAAC,KAAK,KAAK;;AAGd,SAAS,sBACP,OACA,WACuB;CACvB,IAAI,+BAAe,IAAI,KAAuB;AAE9C,MAAK,MAAM,cAAc;EACvB;EACA,MAAM,QAAQ;EACd,MAAM,MAAM,QAAQ;EACrB,CACC,gBAAe,WAAW,cAAc,0BAA0B,WAAW,CAAC;AAGhF,QAAO;;AAGT,SAAS,uBACP,OACA,YACuB;CACvB,IAAI,+BAAe,IAAI,KAAuB;AAE9C,MAAK,MAAM,cAAc;EACvB;EACA,MAAM,QAAQ;EACd,MAAM,MAAM,QAAQ;EACrB,CACC,gBAAe,WAAW,cAAc,0BAA0B,WAAW,CAAC;AAGhF,QAAO;;;;;;;;;;;;;;;;;;;;AAqBT,SAAgB,qBAAqB,SAAuD;AAC1F,KAAI,QAAQ,QAAQ,WAAW,EAC7B,QAAO,EAAE;AAGX,KAAI,QAAQ,mBAAmB,WAAW,EACxC,QAAO,EAAE;CAGX,MAAM,QAAyB,EAAE;AAEjC,MAAK,MAAM,SAAS,QAAQ,SAAS;EACnC,MAAM,gBAAgB,iBAAiB,sBAAsB,OAAO,QAAQ,UAAU,CAAC;EACvF,MAAM,iBAAiB,iBAAiB,uBAAuB,OAAO,QAAQ,WAAW,CAAC;AAE1F,MAAI,cAAc,WAAW,KAAK,eAAe,WAAW,EAC1D;AAGF,OAAK,MAAM,qBAAqB,QAAQ,mBACtC,MAAK,MAAM,aAAa,cACtB,MAAK,MAAM,cAAc,gBAAgB;GACvC,MAAM,iBAAiB,0BAA0B,WAAW,WAAW;AAEvE,SAAM,KAAK;IACT;IACA,IAAI,aACF,MAAM,IACN,kBAAkB,IAClB,eAAe,KAAK,UACpB,eAAe,KAAK,UACrB;IACD,QAAQ;IACR;IACD,CAAC;;;AAMV,QAAO;;;;AC9TT,SAAS,wBACP,QACA,MACiB;CACjB,MAAM,qBAAqB,KAAK,OAAO,IAAI;AAC3C,KAAI,sBAAsB,MAAM;EAC9B,MAAM,sBAAsB,mBAAmB,QAAQ,mBAAmB;AAC1E,MAAI,uBAAuB,KACzB,QAAO;AAGT,QAAM,IAAI,MAAM,6BAA6B,mBAAmB,+BAA+B;;CAGjG,MAAM,UAAU,mBAAmB,QAAQ,KAAK,kBAAkB,GAAG;AACrE,KAAI,WAAW,KACb,QAAO;AAGT,KAAI,OAAO,SAAS,GAAG;EACrB,MAAM,aAAa,OAAO;AAC1B,MAAI,cAAc,KAChB,QAAO;;AAIX,OAAM,IAAI,MAAM,uDAAuD,KAAK,kBAAkB,GAAG,IAAI;;;;;;;;;;;;AAavG,SAAgB,2BAA2B,SAAkE;AAC3G,QAAO,EACL,MAAM,WAAW;AACf,MAAI,aAAa,KACf,QAAO,wBAAwB,QAAQ,QAAQ,QAAQ,KAAK;EAG9D,MAAM,OAAO,OAAO,cAAc,WAAW,YAAY,UAAU;EAEnE,MAAM,aAAa,mBAAmB,QAAQ,QAAQ,KAAK;AAC3D,MAAI,cAAc,KAChB,OAAM,IAAI,MAAM,6BAA6B,KAAK,IAAI;AAGxD,SAAO;IAEV"}

package/dist/testing/expect-extensions.d.mts

@@ -0,0 +1,86 @@
+import { f as ToolCall } from "../index-oSXhM1zx.mjs";
+
+//#region src/testing/expect-extensions.d.ts
+/**
+ * Options for keyword-based matcher behavior.
+ */
+interface KeywordMatcherOptions {
+  /**
+   * Case-sensitive matching toggle.
+   *
+   * @default false
+   */
+  caseSensitive?: boolean;
+  /**
+   * Match mode.
+   *
+   * @default 'all'
+   */
+  mode?: 'all' | 'any';
+}
+/**
+ * Shape used by tool-call matchers.
+ */
+interface ToolCallContainer {
+  /**
+   * Tool calls to inspect.
+   */
+  toolCalls?: readonly ToolCall[];
+}
+/**
+ * Registers vieval custom matchers on Vitest `expect`.
+ *
+ * Call stack:
+ *
+ * {@link installVievalExpectMatchers}
+ *   -> `expect.extend(...)`
+ *     -> `expect(received).toMustInclude(...)`
+ *     -> `expect(received).toScoreRubricGreaterThan(...)`
+ *
+ * Use when:
+ * - eval suites need domain assertions while preserving native Vitest ergonomics
+ * - callers want native `.not` chaining with the same matchers
+ */
+declare function installVievalExpectMatchers(): void;
+declare module 'vitest' {
+  interface Assertion {
+    /**
+     * Asserts that text includes required keywords.
+     *
+     * Example:
+     * `expect('calm answer').toMustInclude(['calm'])`
+     */
+    toMustInclude: (keywords: string | readonly string[], options?: KeywordMatcherOptions) => void;
+    /**
+     * Asserts that text excludes forbidden keywords.
+     *
+     * Example:
+     * `expect('calm answer').toMustExclude(['bestmove'])`
+     */
+    toMustExclude: (keywords: string | readonly string[], options?: KeywordMatcherOptions) => void;
+    /**
+     * Asserts rubric score is greater than a threshold.
+     *
+     * Example:
+     * `expect({ score: 0.91 }).toScoreRubricGreaterThan(0.8)`
+     */
+    toScoreRubricGreaterThan: (threshold: number) => void;
+    /**
+     * Asserts structured output satisfies a validator.
+     *
+     * Example:
+     * `expect(value).toSatisfyStructuredOutput(isMyShape)`
+     */
+    toSatisfyStructuredOutput: <TValue>(validator: (value: unknown) => value is TValue) => void;
+    /**
+     * Asserts selected tool-call args satisfy validator.
+     *
+     * Example:
+     * `expect({ toolCalls }).toSatisfyToolCallArgs('builtIn_sparkCommand', isSparkArgs)`
+     */
+    toSatisfyToolCallArgs: (toolName: string, validator: (args: unknown) => boolean) => void;
+  }
+}
+//#endregion
+export { KeywordMatcherOptions, ToolCallContainer, installVievalExpectMatchers };
+//# sourceMappingURL=expect-extensions.d.mts.map

package/dist/testing/expect-extensions.mjs

@@ -0,0 +1,2 @@
+import { t as installVievalExpectMatchers } from "../expect-extensions-CwPtgTz8.mjs";
+export { installVievalExpectMatchers };