vieval 0.0.5 → 0.0.6

package/dist/index.mjs

@@ -1,5 +1,6 @@
-import { a as defineConfig, i as registerEvalDefinition, s as loadEnv } from "./registry-CwcMMjnZ.mjs";
-import { n as requiredEnvFrom } from "./env-BFSjny07.mjs";
+import { a as defineConfig, i as registerEvalDefinition, s as loadEnv } from "./registry-CcKZqDJY.mjs";
+import { t as createSchedulerQueue } from "./queue-DsZQkZO_.mjs";
+import { n as requiredEnvFrom } from "./env--94B0UtW.mjs";
 import { defineEval, defineTask } from "./config.mjs";
 import { expect } from "./expect.mjs";
 import { errorMessageFrom } from "@moeru/std";
@@ -17,6 +18,12 @@ function createTaskCaseReporterId(index, name) {
 function assertValidScore(score) {
 	if (!Number.isFinite(score) || score < 0 || score > 1) throw new Error(`Case score must be a finite number in range 0..1, got "${score}".`);
 }
+function assertNonNegativeInteger(value, label) {
+	if (!Number.isFinite(value) || !Number.isInteger(value) || value < 0) throw new Error(`Invalid ${label}: ${String(value)}`);
+}
+function assertPositiveInteger(value, label) {
+	if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) throw new Error(`Invalid ${label}: ${String(value)}`);
+}
 function emitCaseStart(hooks, payload) {
 	try {
 		hooks?.onCaseStart?.(payload);
@@ -27,9 +34,126 @@ function emitCaseEnd(hooks, payload) {
 		hooks?.onCaseEnd?.(payload);
 	} catch {}
 }
+function createCaseTimeoutError(timeout) {
+	const error = /* @__PURE__ */ new Error(`Case timed out after ${timeout}ms.`);
+	error.name = "TimeoutError";
+	return error;
+}
+function normalizeExecutionPolicy(policy, label) {
+	if (policy == null) return;
+	if (policy.autoAttempt != null) assertNonNegativeInteger(policy.autoAttempt, `${label} autoAttempt`);
+	if (policy.autoRetry != null) assertNonNegativeInteger(policy.autoRetry, `${label} autoRetry`);
+	if (policy.timeout != null) assertPositiveInteger(policy.timeout, `${label} timeout`);
+	const normalized = {
+		autoAttempt: policy.autoAttempt,
+		autoRetry: policy.autoRetry,
+		timeout: policy.timeout
+	};
+	return Object.values(normalized).some((value) => value != null) ? normalized : void 0;
+}
+function resolveCaseExecutionPolicy(taskCase, taskExecutionPolicy) {
+	return {
+		autoAttempt: taskCase.executionPolicy?.autoAttempt ?? taskExecutionPolicy?.autoAttempt ?? 0,
+		autoRetry: taskCase.executionPolicy?.autoRetry ?? taskExecutionPolicy?.autoRetry ?? 0,
+		timeout: taskCase.executionPolicy?.timeout ?? taskExecutionPolicy?.timeout
+	};
+}
+async function runCaseOnce(context, taskCase, index, timeout) {
+	const customScoresByKind = /* @__PURE__ */ new Map();
+	const abortController = new AbortController();
+	let timeoutHandle;
+	let timedOut = false;
+	let settled = false;
+	try {
+		const runPromise = Promise.resolve(taskCase.run({
+			...context,
+			matrix: {
+				...cloneCaseMatrix(context.task.matrix),
+				inputs: taskCase.input
+			},
+			metric(name, value) {
+				if (abortController.signal.aborted || settled) return;
+				context.reporterHooks?.onEvent?.({
+					caseId: createTaskCaseReporterId(index, taskCase.name),
+					data: {
+						name,
+						value
+					},
+					event: "task.case.metric"
+				});
+			},
+			score(score, kind = "exact") {
+				if (abortController.signal.aborted || settled) return;
+				assertValidScore(score);
+				customScoresByKind.set(kind, score);
+			},
+			signal: abortController.signal
+		}));
+		if (timeout != null) {
+			const timeoutPromise = new Promise((_, reject) => {
+				timeoutHandle = setTimeout(() => {
+					timedOut = true;
+					abortController.abort(createCaseTimeoutError(timeout));
+					reject(createCaseTimeoutError(timeout));
+				}, timeout);
+			});
+			await Promise.race([runPromise, timeoutPromise]);
+		} else await runPromise;
+		settled = true;
+		return {
+			scoresByKind: customScoresByKind,
+			state: "passed"
+		};
+	} catch (error) {
+		settled = true;
+		return {
+			errorMessage: errorMessageFrom(error) ?? (timedOut && timeout != null ? `Case timed out after ${timeout}ms.` : "Unknown case failure."),
+			scoresByKind: customScoresByKind,
+			state: timedOut ? "timeout" : "failed"
+		};
+	} finally {
+		if (timeoutHandle != null) clearTimeout(timeoutHandle);
+	}
+}
+async function executeRegisteredCase(context, taskCase, index, totalCases, taskExecutionPolicy) {
+	const resolvedPolicy = resolveCaseExecutionPolicy(taskCase, taskExecutionPolicy);
+	let lastOutcome;
+	for (let retryIndex = 0; retryIndex <= resolvedPolicy.autoRetry; retryIndex += 1) {
+		emitCaseStart(context.reporterHooks, {
+			...resolvedPolicy.autoRetry > 0 ? {
+				autoRetry: resolvedPolicy.autoRetry,
+				retryIndex
+			} : {},
+			index,
+			name: taskCase.name,
+			total: totalCases
+		});
+		lastOutcome = await runCaseOnce(context, taskCase, index, resolvedPolicy.timeout);
+		if (lastOutcome.state === "passed") return lastOutcome;
+	}
+	return lastOutcome ?? {
+		errorMessage: "Unknown case failure.",
+		scoresByKind: /* @__PURE__ */ new Map(),
+		state: "failed"
+	};
+}
+function collectCaseOutcomeScores(outcome, scoreBucketsByKind) {
+	if (outcome.state !== "passed") {
+		scoreBucketsByKind.exact.push(0);
+		return;
+	}
+	if (outcome.scoresByKind.size === 0) {
+		scoreBucketsByKind.exact.push(1);
+		return;
+	}
+	scoreBucketsByKind.exact.push(outcome.scoresByKind.get("exact") ?? 1);
+	const judgeScore = outcome.scoresByKind.get("judge");
+	if (judgeScore != null) scoreBucketsByKind.judge.push(judgeScore);
+}
 function createCaseBuilder(registeredCases) {
 	function registerCase(name, run, options) {
 		registeredCases.push({
+			executionPolicy: normalizeExecutionPolicy(options, "task case"),
 			input: options?.input,
 			name,
 			run
@@ -37,11 +161,15 @@ function createCaseBuilder(registeredCases) {
 	}
 	return {
 		caseOf: registerCase,
-		casesFromInputs(namePrefix, inputs, run) {
+		casesFromInputs(namePrefix, inputs, run, options) {
+			const queueKey = options?.concurrency == null ? void 0 : {};
 			inputs.forEach((input, index) => {
 				registeredCases.push({
+					concurrency: options?.concurrency,
+					executionPolicy: normalizeExecutionPolicy(options, "casesFromInputs"),
 					input,
 					name: `${namePrefix} #${index + 1}`,
+					queueKey,
 					run
 				});
 			});
@@ -64,6 +192,7 @@ function getActiveCases() {
 }
 function caseOf(name, run, options) {
 	getActiveCases().push({
+		executionPolicy: normalizeExecutionPolicy(options, "task case"),
 		input: options?.input,
 		name,
 		run
@@ -72,16 +201,40 @@ function caseOf(name, run, options) {
 /**
 * Registers multiple cases in the currently active task scope.
 */
-function casesFromInputs(namePrefix, inputs, run) {
+function casesFromInputs(namePrefix, inputs, run, options) {
+	const queueKey = options?.concurrency == null ? void 0 : {};
 	inputs.forEach((input, index) => {
 		getActiveCases().push({
+			concurrency: options?.concurrency,
+			executionPolicy: normalizeExecutionPolicy(options, "casesFromInputs"),
 			input,
 			name: `${namePrefix} #${index + 1}`,
+			queueKey,
 			run
 		});
 	});
 }
 /**
+* Resolves the effective case concurrency for one registered task case.
+*
+* Before:
+* - registered case override `2`, task default `4`
+* - registered case override `undefined`, task default `3`
+*
+* After:
+* - `2`
+* - `3`
+*/
+function resolveCaseConcurrency(taskCase, taskConcurrency, runtimeConcurrency) {
+	const concurrency = runtimeConcurrency?.case ?? taskCase.concurrency ?? taskConcurrency?.case;
+	if (concurrency == null) return;
+	if (!Number.isFinite(concurrency) || !Number.isInteger(concurrency) || concurrency <= 0) throw new Error(`Invalid task case concurrency: ${String(concurrency)}`);
+	return concurrency;
+}
+function resolveCaseQueueKey(taskCase, defaultQueueKey) {
+	return taskCase.queueKey ?? defaultQueueKey;
+}
+/**
 * Defines one eval task with task/case semantics similar to Vitest.
 *
 * Use when:
@@ -98,10 +251,14 @@ function describeTask(name, build, options = {}) {
 		}
 		build();
 	});
+	const description = options.description ?? name;
+	const taskExecutionPolicy = normalizeExecutionPolicy(options, "describeTask");
 	const definition = defineEval({
-		description: options.description ?? name,
+		description,
 		name,
 		task: defineTask({
+			concurrency: options.concurrency,
+			executionPolicy: taskExecutionPolicy,
 			id: name,
 			async run(context) {
 				if (registeredCases.length === 0) return { scores: [{
@@ -113,62 +270,66 @@ function describeTask(name, build, options = {}) {
 					exact: [],
 					judge: []
 				};
-				await Promise.all(registeredCases.map(async (taskCase, index) => {
-					emitCaseStart(context.reporterHooks, {
-						index,
-						name: taskCase.name,
-						total: totalCases
-					});
-					let state = "passed";
-					let errorMessage;
-					const caseId = createTaskCaseReporterId(index, taskCase.name);
-					const customScoresByKind = /* @__PURE__ */ new Map();
-					try {
-						await taskCase.run({
-							...context,
-							matrix: {
-								...cloneCaseMatrix(context.task.matrix),
-								inputs: taskCase.input
-							},
-							metric(name, value) {
-								context.reporterHooks?.onEvent?.({
-									caseId,
-									data: {
-										name,
-										value
-									},
-									event: "task.case.metric"
-								});
-							},
-							score(score, kind = "exact") {
-								assertValidScore(score);
-								customScoresByKind.set(kind, score);
-							}
-						});
-					} catch (error) {
-						state = "failed";
-						errorMessage = errorMessageFrom(error) ?? "Unknown case failure.";
-					} finally {
+				const defaultCaseQueueKey = {};
+				const caseQueues = /* @__PURE__ */ new Map();
+				const hasAutoAttempt = registeredCases.some((taskCase) => resolveCaseExecutionPolicy(taskCase, taskExecutionPolicy).autoAttempt > 0);
+				const runtimeTaskConcurrency = context.task.entry.task?.concurrency ?? options.concurrency;
+				if (!hasAutoAttempt) await Promise.all(registeredCases.map(async (taskCase, index) => {
+					const executeCase = async () => {
+						const outcome = await executeRegisteredCase(context, taskCase, index, totalCases, taskExecutionPolicy);
 						emitCaseEnd(context.reporterHooks, {
-							...errorMessage == null ? {} : { errorMessage },
+							...outcome.errorMessage == null ? {} : { errorMessage: outcome.errorMessage },
 							index,
-							state,
+							state: outcome.state,
 							name: taskCase.name,
 							total: totalCases
 						});
-					}
-					if (state === "failed") {
-						scoreBucketsByKind.exact.push(0);
-						return;
-					}
-					if (customScoresByKind.size === 0) {
-						scoreBucketsByKind.exact.push(1);
+						collectCaseOutcomeScores(outcome, scoreBucketsByKind);
+					};
+					const concurrency = resolveCaseConcurrency(taskCase, runtimeTaskConcurrency, context.runtimeConcurrency);
+					if (concurrency == null) {
+						await executeCase();
 						return;
 					}
-					scoreBucketsByKind.exact.push(customScoresByKind.get("exact") ?? 1);
-					const judgeScore = customScoresByKind.get("judge");
-					if (judgeScore != null) scoreBucketsByKind.judge.push(judgeScore);
+					const queueKey = resolveCaseQueueKey(taskCase, defaultCaseQueueKey);
+					const queue = caseQueues.get(queueKey) ?? createSchedulerQueue(concurrency);
+					caseQueues.set(queueKey, queue);
+					await queue.run(executeCase);
 				}));
+				else {
+					let finalOutcomes = [];
+					let attemptIndex = 0;
+					for (;;) {
+						finalOutcomes = await Promise.all(registeredCases.map(async (taskCase, index) => {
+							const executeCase = async () => await executeRegisteredCase(context, taskCase, index, totalCases, taskExecutionPolicy);
+							const concurrency = resolveCaseConcurrency(taskCase, runtimeTaskConcurrency, context.runtimeConcurrency);
+							if (concurrency == null) return await executeCase();
+							const queueKey = resolveCaseQueueKey(taskCase, defaultCaseQueueKey);
+							const queue = caseQueues.get(queueKey) ?? createSchedulerQueue(concurrency);
+							caseQueues.set(queueKey, queue);
+							return await queue.run(executeCase);
+						}));
+						if (!finalOutcomes.some((outcome, index) => {
+							if (outcome.state === "passed") return false;
+							const taskCase = registeredCases[index];
+							if (taskCase == null) return false;
+							return attemptIndex < resolveCaseExecutionPolicy(taskCase, taskExecutionPolicy).autoAttempt;
+						})) break;
+						attemptIndex += 1;
+					}
+					finalOutcomes.forEach((outcome, index) => {
+						const taskCase = registeredCases[index];
+						if (taskCase == null) return;
+						emitCaseEnd(context.reporterHooks, {
+							...outcome.errorMessage == null ? {} : { errorMessage: outcome.errorMessage },
+							index,
+							state: outcome.state,
+							name: taskCase.name,
+							total: totalCases
+						});
+						collectCaseOutcomeScores(outcome, scoreBucketsByKind);
+					});
+				}
 				return { scores: Object.keys(scoreBucketsByKind).filter((kind) => scoreBucketsByKind[kind].length > 0).map((kind) => {
 					const values = scoreBucketsByKind[kind];
 					return {

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","names":[],"sources":["../src/dsl/task.ts"],"sourcesContent":["import type { TaskRunContext, TaskRunOutput } from '../config'\nimport type { RunScoreKind } from '../core/runner'\n\nimport { errorMessageFrom } from '@moeru/std'\n\nimport { defineEval, defineTask } from '../config'\nimport { registerEvalDefinition } from './registry'\n\n/**\n * Runtime context provided to a task case callback.\n */\nexport interface CaseRunContext<TInput> extends TaskRunContext {\n  /**\n   * Case-scoped matrix payload.\n   */\n  matrix: TaskRunContext['task']['matrix'] & { inputs: TInput }\n  /**\n   * Overrides one case score family with a custom normalized value.\n   *\n   * Use when:\n   * - one case computes a benchmark-native score that should flow into run aggregation\n   *\n   * Expects:\n   * - `score` to stay in the `0..1` range\n   */\n  score: (score: number, kind?: RunScoreKind) => void\n  /**\n   * Emits one custom case metric into report events.\n   *\n   * Use when:\n   * - tasks need structured benchmark metadata beyond exact/judge score families\n   *\n   * Expects:\n   * - `name` to be a stable metric identifier\n   * - `value` to be JSON-serializable\n   */\n  metric: (name: string, value: boolean | number | string | null) => void\n}\n\n/**\n * Callback for one task case.\n */\nexport type CaseRunner<TInput> = (context: CaseRunContext<TInput>) => Promise<void> | void\n\ninterface RegisteredCase<TInput> {\n  input: TInput\n  name: string\n  run: CaseRunner<TInput>\n}\n\nfunction cloneCaseMatrix(matrix: TaskRunContext['task']['matrix']): TaskRunContext['task']['matrix'] {\n  return {\n    eval: {\n      ...matrix.eval,\n    },\n    meta: {\n      ...matrix.meta,\n    },\n    run: {\n      ...matrix.run,\n    },\n  }\n}\n\nfunction createTaskCaseReporterId(index: number, name: string): string {\n  return `${index}:${encodeURIComponent(name)}`\n}\n\nfunction assertValidScore(score: number): void {\n  if (!Number.isFinite(score) || score < 0 || score > 1) {\n    throw new Error(`Case score must be a finite number in range 0..1, got \"${score}\".`)\n  }\n}\n\nfunction emitCaseStart(\n  hooks: TaskRunContext['reporterHooks'] | undefined,\n  payload: {\n    index: number\n    name: string\n    total: number\n  },\n): void {\n  try {\n    hooks?.onCaseStart?.(payload)\n  }\n  catch {\n    // Reporter hooks must never affect task scoring.\n  }\n}\n\nfunction emitCaseEnd(\n  hooks: TaskRunContext['reporterHooks'] | undefined,\n  payload: {\n    index: number\n    state: 'passed' | 'failed'\n    name: string\n    total: number\n    errorMessage?: string\n  },\n): void {\n  try {\n    hooks?.onCaseEnd?.(payload)\n  }\n  catch {\n    // Reporter hooks must never affect task scoring.\n  }\n}\n\n/**\n * Builder callbacks passed into `describeTask`.\n */\nexport interface DescribeTaskBuilder {\n  /**\n   * Registers one explicit case.\n   */\n  caseOf: {\n    (name: string, run: CaseRunner<undefined>): void\n    <TInput>(name: string, run: CaseRunner<TInput>, options: { input: TInput }): void\n  }\n  /**\n   * Registers multiple cases from input list.\n   */\n  casesFromInputs: <TInput>(\n    namePrefix: string,\n    inputs: readonly TInput[],\n    run: CaseRunner<TInput>,\n  ) => void\n}\n\n/**\n * Options for `describeTask`.\n */\nexport interface DescribeTaskOptions {\n  /**\n   * Optional description override.\n   */\n  description?: string\n}\n\nfunction createCaseBuilder(registeredCases: RegisteredCase<unknown>[]): DescribeTaskBuilder {\n  function registerCase(name: string, run: CaseRunner<undefined>): void\n  function registerCase<TInput>(name: string, run: CaseRunner<TInput>, options: { input: TInput }): void\n  function registerCase<TInput>(\n    name: string,\n    run: CaseRunner<TInput> | CaseRunner<undefined>,\n    options?: { input: TInput },\n  ): void {\n    registeredCases.push({\n      input: options?.input,\n      name,\n      run: run as CaseRunner<unknown>,\n    })\n  }\n\n  return {\n    caseOf: registerCase,\n    casesFromInputs(namePrefix, inputs, run) {\n      inputs.forEach((input, index) => {\n        registeredCases.push({\n          input,\n          name: `${namePrefix} #${index + 1}`,\n          run: run as CaseRunner<unknown>,\n        })\n      })\n    },\n  }\n}\n\nlet activeCasesStack: RegisteredCase<unknown>[][] = []\n\nfunction withActiveCases<T>(cases: RegisteredCase<unknown>[], callback: () => T): T {\n  activeCasesStack = [...activeCasesStack, cases]\n\n  try {\n    return callback()\n  }\n  finally {\n    activeCasesStack = activeCasesStack.slice(0, -1)\n  }\n}\n\nfunction getActiveCases(): RegisteredCase<unknown>[] {\n  const active = activeCasesStack.at(-1)\n  if (active == null) {\n    throw new Error('caseOf/casesFromInputs must be called inside describeTask/describeEval.')\n  }\n\n  return active\n}\n\n/**\n * Registers one case in the currently active task scope.\n */\nexport function caseOf(\n  name: string,\n  run: CaseRunner<undefined>,\n): void\n\nexport function caseOf<TInput>(\n  name: string,\n  run: CaseRunner<TInput>,\n  options: { input: TInput },\n): void\n\nexport function caseOf<TInput>(\n  name: string,\n  run: CaseRunner<TInput> | CaseRunner<undefined>,\n  options?: { input: TInput },\n): void {\n  getActiveCases().push({\n    input: options?.input,\n    name,\n    run: run as CaseRunner<unknown>,\n  })\n}\n\n/**\n * Registers multiple cases in the currently active task scope.\n */\nexport function casesFromInputs<TInput>(\n  namePrefix: string,\n  inputs: readonly TInput[],\n  run: CaseRunner<TInput>,\n): void {\n  inputs.forEach((input, index) => {\n    getActiveCases().push({\n      input,\n      name: `${namePrefix} #${index + 1}`,\n      run: run as CaseRunner<unknown>,\n    })\n  })\n}\n\n/**\n * Defines one eval task with task/case semantics similar to Vitest.\n *\n * Use when:\n * - task behavior should be declared with `caseOf` and `casesFromInputs`\n * - business agent code should be imported and run from eval task files\n */\nexport function describeTask(\n  name: string,\n  build: ((builder: DescribeTaskBuilder) => void) | (() => void),\n  options: DescribeTaskOptions = {},\n) {\n  const registeredCases: RegisteredCase<unknown>[] = []\n  const builder = createCaseBuilder(registeredCases)\n  withActiveCases(registeredCases, () => {\n    if (build.length > 0) {\n      (build as (builder: DescribeTaskBuilder) => void)(builder)\n      return\n    }\n\n    ;(build as () => void)()\n  })\n\n  const description = options.description ?? name\n\n  const definition = defineEval({\n    description,\n    name,\n    task: defineTask({\n      id: name,\n      async run(context): Promise<TaskRunOutput> {\n        if (registeredCases.length === 0) {\n          return {\n            scores: [{ kind: 'exact', score: 1 }],\n          }\n        }\n\n        const totalCases = registeredCases.length\n\n        const scoreBucketsByKind: Record<RunScoreKind, number[]> = {\n          exact: [],\n          judge: [],\n        }\n\n        await Promise.all(\n          registeredCases.map(async (taskCase, index) => {\n            emitCaseStart(context.reporterHooks, {\n              index,\n              name: taskCase.name,\n              total: totalCases,\n            })\n\n            let state: 'passed' | 'failed' = 'passed'\n            let errorMessage: string | undefined\n            const caseId = createTaskCaseReporterId(index, taskCase.name)\n            const customScoresByKind = new Map<RunScoreKind, number>()\n\n            try {\n              await taskCase.run({\n                ...context,\n                matrix: {\n                  ...cloneCaseMatrix(context.task.matrix),\n                  inputs: taskCase.input,\n                },\n                metric(name, value) {\n                  context.reporterHooks?.onEvent?.({\n                    caseId,\n                    data: {\n                      name,\n                      value,\n                    },\n                    event: 'task.case.metric',\n                  })\n                },\n                score(score, kind = 'exact') {\n                  assertValidScore(score)\n                  customScoresByKind.set(kind, score)\n                },\n              })\n            }\n            catch (error) {\n              state = 'failed'\n              errorMessage = errorMessageFrom(error) ?? 'Unknown case failure.'\n            }\n            finally {\n              emitCaseEnd(context.reporterHooks, {\n                ...(errorMessage == null ? {} : { errorMessage }),\n                index,\n                state,\n                name: taskCase.name,\n                total: totalCases,\n              })\n            }\n\n            if (state === 'failed') {\n              scoreBucketsByKind.exact.push(0)\n              return\n            }\n\n            if (customScoresByKind.size === 0) {\n              scoreBucketsByKind.exact.push(1)\n              return\n            }\n\n            scoreBucketsByKind.exact.push(customScoresByKind.get('exact') ?? 1)\n            const judgeScore = customScoresByKind.get('judge')\n            if (judgeScore != null) {\n              scoreBucketsByKind.judge.push(judgeScore)\n            }\n          }),\n        )\n\n        const scores = (Object.keys(scoreBucketsByKind) as RunScoreKind[])\n          .filter(kind => scoreBucketsByKind[kind].length > 0)\n          .map((kind) => {\n            const values = scoreBucketsByKind[kind]\n            const total = values.reduce((sum, value) => sum + value, 0)\n            return {\n              kind,\n              score: total / values.length,\n            }\n          })\n\n        return {\n          scores,\n        }\n      },\n    }),\n  })\n\n  registerEvalDefinition(definition)\n\n  return definition\n}\n\n/**\n * Alias of `describeTask` for eval-centric naming.\n */\nexport const describeEval = describeTask\n"],"mappings":";;;;;;AAkDA,SAAS,gBAAgB,QAA4E;AACnG,QAAO;EACL,MAAM,EACJ,GAAG,OAAO,MACX;EACD,MAAM,EACJ,GAAG,OAAO,MACX;EACD,KAAK,EACH,GAAG,OAAO,KACX;EACF;;AAGH,SAAS,yBAAyB,OAAe,MAAsB;AACrE,QAAO,GAAG,MAAM,GAAG,mBAAmB,KAAK;;AAG7C,SAAS,iBAAiB,OAAqB;AAC7C,KAAI,CAAC,OAAO,SAAS,MAAM,IAAI,QAAQ,KAAK,QAAQ,EAClD,OAAM,IAAI,MAAM,0DAA0D,MAAM,IAAI;;AAIxF,SAAS,cACP,OACA,SAKM;AACN,KAAI;AACF,SAAO,cAAc,QAAQ;SAEzB;;AAKR,SAAS,YACP,OACA,SAOM;AACN,KAAI;AACF,SAAO,YAAY,QAAQ;SAEvB;;AAoCR,SAAS,kBAAkB,iBAAiE;CAG1F,SAAS,aACP,MACA,KACA,SACM;AACN,kBAAgB,KAAK;GACnB,OAAO,SAAS;GAChB;GACK;GACN,CAAC;;AAGJ,QAAO;EACL,QAAQ;EACR,gBAAgB,YAAY,QAAQ,KAAK;AACvC,UAAO,SAAS,OAAO,UAAU;AAC/B,oBAAgB,KAAK;KACnB;KACA,MAAM,GAAG,WAAW,IAAI,QAAQ;KAC3B;KACN,CAAC;KACF;;EAEL;;AAGH,IAAI,mBAAgD,EAAE;AAEtD,SAAS,gBAAmB,OAAkC,UAAsB;AAClF,oBAAmB,CAAC,GAAG,kBAAkB,MAAM;AAE/C,KAAI;AACF,SAAO,UAAU;WAEX;AACN,qBAAmB,iBAAiB,MAAM,GAAG,GAAG;;;AAIpD,SAAS,iBAA4C;CACnD,MAAM,SAAS,iBAAiB,GAAG,GAAG;AACtC,KAAI,UAAU,KACZ,OAAM,IAAI,MAAM,0EAA0E;AAG5F,QAAO;;AAiBT,SAAgB,OACd,MACA,KACA,SACM;AACN,iBAAgB,CAAC,KAAK;EACpB,OAAO,SAAS;EAChB;EACK;EACN,CAAC;;;;;AAMJ,SAAgB,gBACd,YACA,QACA,KACM;AACN,QAAO,SAAS,OAAO,UAAU;AAC/B,kBAAgB,CAAC,KAAK;GACpB;GACA,MAAM,GAAG,WAAW,IAAI,QAAQ;GAC3B;GACN,CAAC;GACF;;;;;;;;;AAUJ,SAAgB,aACd,MACA,OACA,UAA+B,EAAE,EACjC;CACA,MAAM,kBAA6C,EAAE;CACrD,MAAM,UAAU,kBAAkB,gBAAgB;AAClD,iBAAgB,uBAAuB;AACrC,MAAI,MAAM,SAAS,GAAG;AACnB,SAAiD,QAAQ;AAC1D;;AAGA,SAAsB;GACxB;CAIF,MAAM,aAAa,WAAW;EAC5B,aAHkB,QAAQ,eAAe;EAIzC;EACA,MAAM,WAAW;GACf,IAAI;GACJ,MAAM,IAAI,SAAiC;AACzC,QAAI,gBAAgB,WAAW,EAC7B,QAAO,EACL,QAAQ,CAAC;KAAE,MAAM;KAAS,OAAO;KAAG,CAAC,EACtC;IAGH,MAAM,aAAa,gBAAgB;IAEnC,MAAM,qBAAqD;KACzD,OAAO,EAAE;KACT,OAAO,EAAE;KACV;AAED,UAAM,QAAQ,IACZ,gBAAgB,IAAI,OAAO,UAAU,UAAU;AAC7C,mBAAc,QAAQ,eAAe;MACnC;MACA,MAAM,SAAS;MACf,OAAO;MACR,CAAC;KAEF,IAAI,QAA6B;KACjC,IAAI;KACJ,MAAM,SAAS,yBAAyB,OAAO,SAAS,KAAK;KAC7D,MAAM,qCAAqB,IAAI,KAA2B;AAE1D,SAAI;AACF,YAAM,SAAS,IAAI;OACjB,GAAG;OACH,QAAQ;QACN,GAAG,gBAAgB,QAAQ,KAAK,OAAO;QACvC,QAAQ,SAAS;QAClB;OACD,OAAO,MAAM,OAAO;AAClB,gBAAQ,eAAe,UAAU;SAC/B;SACA,MAAM;UACJ;UACA;UACD;SACD,OAAO;SACR,CAAC;;OAEJ,MAAM,OAAO,OAAO,SAAS;AAC3B,yBAAiB,MAAM;AACvB,2BAAmB,IAAI,MAAM,MAAM;;OAEtC,CAAC;cAEG,OAAO;AACZ,cAAQ;AACR,qBAAe,iBAAiB,MAAM,IAAI;eAEpC;AACN,kBAAY,QAAQ,eAAe;OACjC,GAAI,gBAAgB,OAAO,EAAE,GAAG,EAAE,cAAc;OAChD;OACA;OACA,MAAM,SAAS;OACf,OAAO;OACR,CAAC;;AAGJ,SAAI,UAAU,UAAU;AACtB,yBAAmB,MAAM,KAAK,EAAE;AAChC;;AAGF,SAAI,mBAAmB,SAAS,GAAG;AACjC,yBAAmB,MAAM,KAAK,EAAE;AAChC;;AAGF,wBAAmB,MAAM,KAAK,mBAAmB,IAAI,QAAQ,IAAI,EAAE;KACnE,MAAM,aAAa,mBAAmB,IAAI,QAAQ;AAClD,SAAI,cAAc,KAChB,oBAAmB,MAAM,KAAK,WAAW;MAE3C,CACH;AAaD,WAAO,EACL,QAZc,OAAO,KAAK,mBAAmB,CAC5C,QAAO,SAAQ,mBAAmB,MAAM,SAAS,EAAE,CACnD,KAAK,SAAS;KACb,MAAM,SAAS,mBAAmB;AAElC,YAAO;MACL;MACA,OAHY,OAAO,QAAQ,KAAK,UAAU,MAAM,OAAO,EAAE,GAG1C,OAAO;MACvB;MACD,EAIH;;GAEJ,CAAC;EACH,CAAC;AAEF,wBAAuB,WAAW;AAElC,QAAO;;;;;AAMT,MAAa,eAAe"}
+{"version":3,"file":"index.mjs","names":[],"sources":["../src/dsl/task.ts"],"sourcesContent":["import type { TaskConcurrencyConfig, TaskExecutionPolicy, TaskRunContext, TaskRunOutput } from '../config'\nimport type { RunScoreKind } from '../core/runner'\n\nimport { errorMessageFrom } from '@moeru/std'\n\nimport { defineEval, defineTask } from '../config'\nimport { createSchedulerQueue } from '../core/scheduler/queue'\nimport { registerEvalDefinition } from './registry'\n\n/**\n * Runtime context provided to a task case callback.\n */\nexport interface CaseRunContext<TInput> extends TaskRunContext {\n  /**\n   * Case-scoped matrix payload.\n   */\n  matrix: TaskRunContext['task']['matrix'] & { inputs: TInput }\n  /**\n   * Overrides one case score family with a custom normalized value.\n   *\n   * Use when:\n   * - one case computes a benchmark-native score that should flow into run aggregation\n   *\n   * Expects:\n   * - `score` to stay in the `0..1` range\n   */\n  score: (score: number, kind?: RunScoreKind) => void\n  /**\n   * Emits one custom case metric into report events.\n   *\n   * Use when:\n   * - tasks need structured benchmark metadata beyond exact/judge score families\n   *\n   * Expects:\n   * - `name` to be a stable metric identifier\n   * - `value` to be JSON-serializable\n   */\n  metric: (name: string, value: boolean | number | string | null) => void\n  /**\n   * Cooperative abort signal for the current case execution.\n   */\n  signal: AbortSignal\n}\n\n/**\n * Callback for one task case.\n */\nexport type CaseRunner<TInput> = (context: CaseRunContext<TInput>) => Promise<void> | void\n\ninterface RegisteredCase<TInput> {\n  concurrency?: number\n  executionPolicy?: TaskExecutionPolicy\n  input: TInput\n  name: string\n  queueKey?: object\n  run: CaseRunner<TInput>\n}\n\n/**\n * Per-group options for `casesFromInputs`.\n *\n * Use when:\n * - one generated case group should run with a lower case concurrency than the task default\n * - a task should keep a broader task-level cap while one expensive case family stays bounded\n *\n * Expects:\n * - `concurrency` to be a positive integer when provided\n *\n * Returns:\n * - one partial case-group execution descriptor\n */\nexport interface CasesFromInputsOptions extends TaskExecutionPolicy {\n  /**\n   * Case-level concurrency cap for cases registered by one `casesFromInputs(...)` call.\n   */\n  concurrency?: number\n}\n\n/**\n * Per-case registration options for `caseOf`.\n */\nexport interface CaseRegistrationOptions<TInput> extends TaskExecutionPolicy {\n  /**\n   * Optional case input payload.\n   */\n  input: TInput\n}\n\ninterface CaseExecutionOutcome {\n  errorMessage?: string\n  scoresByKind: Map<RunScoreKind, number>\n  state: 'failed' | 'passed' | 'timeout'\n}\n\nfunction cloneCaseMatrix(matrix: TaskRunContext['task']['matrix']): TaskRunContext['task']['matrix'] {\n  return {\n    eval: {\n      ...matrix.eval,\n    },\n    meta: {\n      ...matrix.meta,\n    },\n    run: {\n      ...matrix.run,\n    },\n  }\n}\n\nfunction createTaskCaseReporterId(index: number, name: string): string {\n  return `${index}:${encodeURIComponent(name)}`\n}\n\nfunction assertValidScore(score: number): void {\n  if (!Number.isFinite(score) || score < 0 || score > 1) {\n    throw new Error(`Case score must be a finite number in range 0..1, got \"${score}\".`)\n  }\n}\n\nfunction assertNonNegativeInteger(value: number, label: string): void {\n  if (!Number.isFinite(value) || !Number.isInteger(value) || value < 0) {\n    throw new Error(`Invalid ${label}: ${String(value)}`)\n  }\n}\n\nfunction assertPositiveInteger(value: number, label: string): void {\n  if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) {\n    throw new Error(`Invalid ${label}: ${String(value)}`)\n  }\n}\n\nfunction emitCaseStart(\n  hooks: TaskRunContext['reporterHooks'] | undefined,\n  payload: {\n    autoRetry?: number\n    index: number\n    name: string\n    retryIndex?: number\n    total: number\n  },\n): void {\n  try {\n    hooks?.onCaseStart?.(payload)\n  }\n  catch {\n    // Reporter hooks must never affect task scoring.\n  }\n}\n\nfunction emitCaseEnd(\n  hooks: TaskRunContext['reporterHooks'] | undefined,\n  payload: {\n    index: number\n    state: 'passed' | 'failed' | 'timeout'\n    name: string\n    total: number\n    errorMessage?: string\n  },\n): void {\n  try {\n    hooks?.onCaseEnd?.(payload)\n  }\n  catch {\n    // Reporter hooks must never affect task scoring.\n  }\n}\n\nfunction createCaseTimeoutError(timeout: number): Error {\n  const error = new Error(`Case timed out after ${timeout}ms.`)\n  error.name = 'TimeoutError'\n  return error\n}\n\nfunction normalizeExecutionPolicy(policy: TaskExecutionPolicy | undefined, label: string): TaskExecutionPolicy | undefined {\n  if (policy == null) {\n    return undefined\n  }\n\n  if (policy.autoAttempt != null) {\n    assertNonNegativeInteger(policy.autoAttempt, `${label} autoAttempt`)\n  }\n\n  if (policy.autoRetry != null) {\n    assertNonNegativeInteger(policy.autoRetry, `${label} autoRetry`)\n  }\n\n  if (policy.timeout != null) {\n    assertPositiveInteger(policy.timeout, `${label} timeout`)\n  }\n\n  const normalized = {\n    autoAttempt: policy.autoAttempt,\n    autoRetry: policy.autoRetry,\n    timeout: policy.timeout,\n  }\n\n  return Object.values(normalized).some(value => value != null)\n    ? normalized\n    : undefined\n}\n\nfunction resolveCaseExecutionPolicy(\n  taskCase: RegisteredCase<unknown>,\n  taskExecutionPolicy: TaskExecutionPolicy | undefined,\n): Required<Pick<TaskExecutionPolicy, 'autoAttempt' | 'autoRetry'>> & Pick<TaskExecutionPolicy, 'timeout'> {\n  return {\n    autoAttempt: taskCase.executionPolicy?.autoAttempt ?? taskExecutionPolicy?.autoAttempt ?? 0,\n    autoRetry: taskCase.executionPolicy?.autoRetry ?? taskExecutionPolicy?.autoRetry ?? 0,\n    timeout: taskCase.executionPolicy?.timeout ?? taskExecutionPolicy?.timeout,\n  }\n}\n\nasync function runCaseOnce(\n  context: TaskRunContext,\n  taskCase: RegisteredCase<unknown>,\n  index: number,\n  timeout: number | undefined,\n): Promise<CaseExecutionOutcome> {\n  const customScoresByKind = new Map<RunScoreKind, number>()\n  const abortController = new AbortController()\n  let timeoutHandle: ReturnType<typeof setTimeout> | undefined\n  let timedOut = false\n  let settled = false\n\n  try {\n    const runPromise = Promise.resolve(taskCase.run({\n      ...context,\n      matrix: {\n        ...cloneCaseMatrix(context.task.matrix),\n        inputs: taskCase.input,\n      },\n      metric(name, value) {\n        if (abortController.signal.aborted || settled) {\n          return\n        }\n\n        context.reporterHooks?.onEvent?.({\n          caseId: createTaskCaseReporterId(index, taskCase.name),\n          data: {\n            name,\n            value,\n          },\n          event: 'task.case.metric',\n        })\n      },\n      score(score, kind = 'exact') {\n        if (abortController.signal.aborted || settled) {\n          return\n        }\n\n        assertValidScore(score)\n        customScoresByKind.set(kind, score)\n      },\n      signal: abortController.signal,\n    }))\n\n    if (timeout != null) {\n      const timeoutPromise = new Promise<never>((_, reject) => {\n        timeoutHandle = setTimeout(() => {\n          timedOut = true\n          abortController.abort(createCaseTimeoutError(timeout))\n          reject(createCaseTimeoutError(timeout))\n        }, timeout)\n      })\n\n      await Promise.race([runPromise, timeoutPromise])\n    }\n    else {\n      await runPromise\n    }\n\n    settled = true\n    return {\n      scoresByKind: customScoresByKind,\n      state: 'passed',\n    }\n  }\n  catch (error) {\n    settled = true\n    return {\n      errorMessage: errorMessageFrom(error) ?? (timedOut && timeout != null ? `Case timed out after ${timeout}ms.` : 'Unknown case failure.'),\n      scoresByKind: customScoresByKind,\n      state: timedOut ? 'timeout' : 'failed',\n    }\n  }\n  finally {\n    if (timeoutHandle != null) {\n      clearTimeout(timeoutHandle)\n    }\n  }\n}\n\nasync function executeRegisteredCase(\n  context: TaskRunContext,\n  taskCase: RegisteredCase<unknown>,\n  index: number,\n  totalCases: number,\n  taskExecutionPolicy: TaskExecutionPolicy | undefined,\n): Promise<CaseExecutionOutcome> {\n  const resolvedPolicy = resolveCaseExecutionPolicy(taskCase, taskExecutionPolicy)\n  let lastOutcome: CaseExecutionOutcome | undefined\n\n  for (let retryIndex = 0; retryIndex <= resolvedPolicy.autoRetry; retryIndex += 1) {\n    emitCaseStart(context.reporterHooks, {\n      ...(resolvedPolicy.autoRetry > 0\n        ? {\n            autoRetry: resolvedPolicy.autoRetry,\n            retryIndex,\n          }\n        : {}),\n      index,\n      name: taskCase.name,\n      total: totalCases,\n    })\n    lastOutcome = await runCaseOnce(context, taskCase, index, resolvedPolicy.timeout)\n    if (lastOutcome.state === 'passed') {\n      return lastOutcome\n    }\n  }\n\n  return lastOutcome ?? {\n    errorMessage: 'Unknown case failure.',\n    scoresByKind: new Map(),\n    state: 'failed',\n  }\n}\n\nfunction collectCaseOutcomeScores(\n  outcome: CaseExecutionOutcome,\n  scoreBucketsByKind: Record<RunScoreKind, number[]>,\n): void {\n  if (outcome.state !== 'passed') {\n    scoreBucketsByKind.exact.push(0)\n    return\n  }\n\n  if (outcome.scoresByKind.size === 0) {\n    scoreBucketsByKind.exact.push(1)\n    return\n  }\n\n  scoreBucketsByKind.exact.push(outcome.scoresByKind.get('exact') ?? 1)\n  const judgeScore = outcome.scoresByKind.get('judge')\n  if (judgeScore != null) {\n    scoreBucketsByKind.judge.push(judgeScore)\n  }\n}\n\n/**\n * Builder callbacks passed into `describeTask`.\n */\nexport interface DescribeTaskBuilder {\n  /**\n   * Registers one explicit case.\n   */\n  caseOf: {\n    (name: string, run: CaseRunner<undefined>): void\n    <TInput>(name: string, run: CaseRunner<TInput>, options: CaseRegistrationOptions<TInput>): void\n  }\n  /**\n   * Registers multiple cases from input list.\n   */\n  casesFromInputs: <TInput>(\n    namePrefix: string,\n    inputs: readonly TInput[],\n    run: CaseRunner<TInput>,\n    options?: CasesFromInputsOptions,\n  ) => void\n}\n\n/**\n * Options for `describeTask`.\n */\nexport interface DescribeTaskOptions extends TaskExecutionPolicy {\n  /**\n   * Optional description override.\n   */\n  description?: string\n  /**\n   * Optional task-local concurrency overrides.\n   *\n   * Use when:\n   * - one task should cap attempt fan-out independently from the surrounding project\n   * - one task should cap case fan-out without changing global scheduling defaults\n   *\n   * Expects:\n   * - each provided value to be a positive integer\n   *\n   * @default inherited from project or CLI concurrency settings\n   */\n  concurrency?: TaskConcurrencyConfig\n}\n\nfunction createCaseBuilder(registeredCases: RegisteredCase<unknown>[]): DescribeTaskBuilder {\n  function registerCase(name: string, run: CaseRunner<undefined>): void\n  function registerCase<TInput>(name: string, run: CaseRunner<TInput>, options: CaseRegistrationOptions<TInput>): void\n  function registerCase<TInput>(\n    name: string,\n    run: CaseRunner<TInput> | CaseRunner<undefined>,\n    options?: CaseRegistrationOptions<TInput>,\n  ): void {\n    registeredCases.push({\n      executionPolicy: normalizeExecutionPolicy(options, 'task case'),\n      input: options?.input,\n      name,\n      run: run as CaseRunner<unknown>,\n    })\n  }\n\n  return {\n    caseOf: registerCase,\n    casesFromInputs(namePrefix, inputs, run, options) {\n      const queueKey = options?.concurrency == null ? undefined : {}\n\n      inputs.forEach((input, index) => {\n        registeredCases.push({\n          concurrency: options?.concurrency,\n          executionPolicy: normalizeExecutionPolicy(options, 'casesFromInputs'),\n          input,\n          name: `${namePrefix} #${index + 1}`,\n          queueKey,\n          run: run as CaseRunner<unknown>,\n        })\n      })\n    },\n  }\n}\n\nlet activeCasesStack: RegisteredCase<unknown>[][] = []\n\nfunction withActiveCases<T>(cases: RegisteredCase<unknown>[], callback: () => T): T {\n  activeCasesStack = [...activeCasesStack, cases]\n\n  try {\n    return callback()\n  }\n  finally {\n    activeCasesStack = activeCasesStack.slice(0, -1)\n  }\n}\n\nfunction getActiveCases(): RegisteredCase<unknown>[] {\n  const active = activeCasesStack.at(-1)\n  if (active == null) {\n    throw new Error('caseOf/casesFromInputs must be called inside describeTask/describeEval.')\n  }\n\n  return active\n}\n\n/**\n * Registers one case in the currently active task scope.\n */\nexport function caseOf(\n  name: string,\n  run: CaseRunner<undefined>,\n): void\n\nexport function caseOf<TInput>(\n  name: string,\n  run: CaseRunner<TInput>,\n  options: CaseRegistrationOptions<TInput>,\n): void\n\nexport function caseOf<TInput>(\n  name: string,\n  run: CaseRunner<TInput> | CaseRunner<undefined>,\n  options?: CaseRegistrationOptions<TInput>,\n): void {\n  getActiveCases().push({\n    executionPolicy: normalizeExecutionPolicy(options, 'task case'),\n    input: options?.input,\n    name,\n    run: run as CaseRunner<unknown>,\n  })\n}\n\n/**\n * Registers multiple cases in the currently active task scope.\n */\nexport function casesFromInputs<TInput>(\n  namePrefix: string,\n  inputs: readonly TInput[],\n  run: CaseRunner<TInput>,\n  options?: CasesFromInputsOptions,\n): void {\n  const queueKey = options?.concurrency == null ? undefined : {}\n\n  inputs.forEach((input, index) => {\n    getActiveCases().push({\n      concurrency: options?.concurrency,\n      executionPolicy: normalizeExecutionPolicy(options, 'casesFromInputs'),\n      input,\n      name: `${namePrefix} #${index + 1}`,\n      queueKey,\n      run: run as CaseRunner<unknown>,\n    })\n  })\n}\n\n/**\n * Resolves the effective case concurrency for one registered task case.\n *\n * Before:\n * - registered case override `2`, task default `4`\n * - registered case override `undefined`, task default `3`\n *\n * After:\n * - `2`\n * - `3`\n */\nfunction resolveCaseConcurrency(\n  taskCase: RegisteredCase<unknown>,\n  taskConcurrency: TaskConcurrencyConfig | undefined,\n  runtimeConcurrency: TaskConcurrencyConfig | undefined,\n): number | undefined {\n  const concurrency = runtimeConcurrency?.case ?? taskCase.concurrency ?? taskConcurrency?.case\n  if (concurrency == null) {\n    return undefined\n  }\n\n  if (!Number.isFinite(concurrency) || !Number.isInteger(concurrency) || concurrency <= 0) {\n    throw new Error(`Invalid task case concurrency: ${String(concurrency)}`)\n  }\n\n  return concurrency\n}\n\nfunction resolveCaseQueueKey(taskCase: RegisteredCase<unknown>, defaultQueueKey: object): object {\n  return taskCase.queueKey ?? defaultQueueKey\n}\n\n/**\n * Defines one eval task with task/case semantics similar to Vitest.\n *\n * Use when:\n * - task behavior should be declared with `caseOf` and `casesFromInputs`\n * - business agent code should be imported and run from eval task files\n */\nexport function describeTask(\n  name: string,\n  build: ((builder: DescribeTaskBuilder) => void) | (() => void),\n  options: DescribeTaskOptions = {},\n) {\n  const registeredCases: RegisteredCase<unknown>[] = []\n  const builder = createCaseBuilder(registeredCases)\n  withActiveCases(registeredCases, () => {\n    if (build.length > 0) {\n      (build as (builder: DescribeTaskBuilder) => void)(builder)\n      return\n    }\n\n    ;(build as () => void)()\n  })\n\n  const description = options.description ?? name\n  const taskExecutionPolicy = normalizeExecutionPolicy(options, 'describeTask')\n\n  const definition = defineEval({\n    description,\n    name,\n    task: defineTask({\n      concurrency: options.concurrency,\n      executionPolicy: taskExecutionPolicy,\n      id: name,\n      async run(context): Promise<TaskRunOutput> {\n        if (registeredCases.length === 0) {\n          return {\n            scores: [{ kind: 'exact', score: 1 }],\n          }\n        }\n\n        const totalCases = registeredCases.length\n        const scoreBucketsByKind: Record<RunScoreKind, number[]> = {\n          exact: [],\n          judge: [],\n        }\n        const defaultCaseQueueKey = {}\n        const caseQueues = new Map<object, ReturnType<typeof createSchedulerQueue>>()\n        const hasAutoAttempt = registeredCases.some(taskCase => resolveCaseExecutionPolicy(taskCase, taskExecutionPolicy).autoAttempt > 0)\n        const runtimeTaskConcurrency = context.task.entry.task?.concurrency ?? options.concurrency\n\n        if (!hasAutoAttempt) {\n          await Promise.all(\n            registeredCases.map(async (taskCase, index) => {\n              const executeCase = async () => {\n                const outcome = await executeRegisteredCase(context, taskCase, index, totalCases, taskExecutionPolicy)\n                emitCaseEnd(context.reporterHooks, {\n                  ...(outcome.errorMessage == null ? {} : { errorMessage: outcome.errorMessage }),\n                  index,\n                  state: outcome.state,\n                  name: taskCase.name,\n                  total: totalCases,\n                })\n                collectCaseOutcomeScores(outcome, scoreBucketsByKind)\n              }\n\n              const concurrency = resolveCaseConcurrency(taskCase, runtimeTaskConcurrency, context.runtimeConcurrency)\n              if (concurrency == null) {\n                await executeCase()\n                return\n              }\n\n              const queueKey = resolveCaseQueueKey(taskCase, defaultCaseQueueKey)\n              const queue = caseQueues.get(queueKey) ?? createSchedulerQueue(concurrency)\n              caseQueues.set(queueKey, queue)\n              await queue.run(executeCase)\n            }),\n          )\n        }\n        else {\n          let finalOutcomes: CaseExecutionOutcome[] = []\n          let attemptIndex = 0\n\n          for (;;) {\n            finalOutcomes = await Promise.all(\n              registeredCases.map(async (taskCase, index) => {\n                const executeCase = async () => await executeRegisteredCase(context, taskCase, index, totalCases, taskExecutionPolicy)\n                const concurrency = resolveCaseConcurrency(taskCase, runtimeTaskConcurrency, context.runtimeConcurrency)\n                if (concurrency == null) {\n                  return await executeCase()\n                }\n\n                const queueKey = resolveCaseQueueKey(taskCase, defaultCaseQueueKey)\n                const queue = caseQueues.get(queueKey) ?? createSchedulerQueue(concurrency)\n                caseQueues.set(queueKey, queue)\n                return await queue.run(executeCase)\n              }),\n            )\n\n            const shouldContinue = finalOutcomes.some((outcome, index) => {\n              if (outcome.state === 'passed') {\n                return false\n              }\n\n              const taskCase = registeredCases[index]\n              if (taskCase == null) {\n                return false\n              }\n\n              return attemptIndex < resolveCaseExecutionPolicy(taskCase, taskExecutionPolicy).autoAttempt\n            })\n\n            if (!shouldContinue) {\n              break\n            }\n\n            attemptIndex += 1\n          }\n\n          finalOutcomes.forEach((outcome, index) => {\n            const taskCase = registeredCases[index]\n            if (taskCase == null) {\n              return\n            }\n\n            emitCaseEnd(context.reporterHooks, {\n              ...(outcome.errorMessage == null ? {} : { errorMessage: outcome.errorMessage }),\n              index,\n              state: outcome.state,\n              name: taskCase.name,\n              total: totalCases,\n            })\n            collectCaseOutcomeScores(outcome, scoreBucketsByKind)\n          })\n        }\n\n        const scores = (Object.keys(scoreBucketsByKind) as RunScoreKind[])\n          .filter(kind => scoreBucketsByKind[kind].length > 0)\n          .map((kind) => {\n            const values = scoreBucketsByKind[kind]\n            const total = values.reduce((sum, value) => sum + value, 0)\n            return {\n              kind,\n              score: total / values.length,\n            }\n          })\n\n        return {\n          scores,\n        }\n      },\n    }),\n  })\n\n  registerEvalDefinition(definition)\n\n  return definition\n}\n\n/**\n * Alias of `describeTask` for eval-centric naming.\n */\nexport const describeEval = describeTask\n"],"mappings":";;;;;;;AA8FA,SAAS,gBAAgB,QAA4E;AACnG,QAAO;EACL,MAAM,EACJ,GAAG,OAAO,MACX;EACD,MAAM,EACJ,GAAG,OAAO,MACX;EACD,KAAK,EACH,GAAG,OAAO,KACX;EACF;;AAGH,SAAS,yBAAyB,OAAe,MAAsB;AACrE,QAAO,GAAG,MAAM,GAAG,mBAAmB,KAAK;;AAG7C,SAAS,iBAAiB,OAAqB;AAC7C,KAAI,CAAC,OAAO,SAAS,MAAM,IAAI,QAAQ,KAAK,QAAQ,EAClD,OAAM,IAAI,MAAM,0DAA0D,MAAM,IAAI;;AAIxF,SAAS,yBAAyB,OAAe,OAAqB;AACpE,KAAI,CAAC,OAAO,SAAS,MAAM,IAAI,CAAC,OAAO,UAAU,MAAM,IAAI,QAAQ,EACjE,OAAM,IAAI,MAAM,WAAW,MAAM,IAAI,OAAO,MAAM,GAAG;;AAIzD,SAAS,sBAAsB,OAAe,OAAqB;AACjE,KAAI,CAAC,OAAO,SAAS,MAAM,IAAI,CAAC,OAAO,UAAU,MAAM,IAAI,SAAS,EAClE,OAAM,IAAI,MAAM,WAAW,MAAM,IAAI,OAAO,MAAM,GAAG;;AAIzD,SAAS,cACP,OACA,SAOM;AACN,KAAI;AACF,SAAO,cAAc,QAAQ;SAEzB;;AAKR,SAAS,YACP,OACA,SAOM;AACN,KAAI;AACF,SAAO,YAAY,QAAQ;SAEvB;;AAKR,SAAS,uBAAuB,SAAwB;CACtD,MAAM,wBAAQ,IAAI,MAAM,wBAAwB,QAAQ,KAAK;AAC7D,OAAM,OAAO;AACb,QAAO;;AAGT,SAAS,yBAAyB,QAAyC,OAAgD;AACzH,KAAI,UAAU,KACZ;AAGF,KAAI,OAAO,eAAe,KACxB,0BAAyB,OAAO,aAAa,GAAG,MAAM,cAAc;AAGtE,KAAI,OAAO,aAAa,KACtB,0BAAyB,OAAO,WAAW,GAAG,MAAM,YAAY;AAGlE,KAAI,OAAO,WAAW,KACpB,uBAAsB,OAAO,SAAS,GAAG,MAAM,UAAU;CAG3D,MAAM,aAAa;EACjB,aAAa,OAAO;EACpB,WAAW,OAAO;EAClB,SAAS,OAAO;EACjB;AAED,QAAO,OAAO,OAAO,WAAW,CAAC,MAAK,UAAS,SAAS,KAAK,GACzD,aACA,KAAA;;AAGN,SAAS,2BACP,UACA,qBACyG;AACzG,QAAO;EACL,aAAa,SAAS,iBAAiB,eAAe,qBAAqB,eAAe;EAC1F,WAAW,SAAS,iBAAiB,aAAa,qBAAqB,aAAa;EACpF,SAAS,SAAS,iBAAiB,WAAW,qBAAqB;EACpE;;AAGH,eAAe,YACb,SACA,UACA,OACA,SAC+B;CAC/B,MAAM,qCAAqB,IAAI,KAA2B;CAC1D,MAAM,kBAAkB,IAAI,iBAAiB;CAC7C,IAAI;CACJ,IAAI,WAAW;CACf,IAAI,UAAU;AAEd,KAAI;EACF,MAAM,aAAa,QAAQ,QAAQ,SAAS,IAAI;GAC9C,GAAG;GACH,QAAQ;IACN,GAAG,gBAAgB,QAAQ,KAAK,OAAO;IACvC,QAAQ,SAAS;IAClB;GACD,OAAO,MAAM,OAAO;AAClB,QAAI,gBAAgB,OAAO,WAAW,QACpC;AAGF,YAAQ,eAAe,UAAU;KAC/B,QAAQ,yBAAyB,OAAO,SAAS,KAAK;KACtD,MAAM;MACJ;MACA;MACD;KACD,OAAO;KACR,CAAC;;GAEJ,MAAM,OAAO,OAAO,SAAS;AAC3B,QAAI,gBAAgB,OAAO,WAAW,QACpC;AAGF,qBAAiB,MAAM;AACvB,uBAAmB,IAAI,MAAM,MAAM;;GAErC,QAAQ,gBAAgB;GACzB,CAAC,CAAC;AAEH,MAAI,WAAW,MAAM;GACnB,MAAM,iBAAiB,IAAI,SAAgB,GAAG,WAAW;AACvD,oBAAgB,iBAAiB;AAC/B,gBAAW;AACX,qBAAgB,MAAM,uBAAuB,QAAQ,CAAC;AACtD,YAAO,uBAAuB,QAAQ,CAAC;OACtC,QAAQ;KACX;AAEF,SAAM,QAAQ,KAAK,CAAC,YAAY,eAAe,CAAC;QAGhD,OAAM;AAGR,YAAU;AACV,SAAO;GACL,cAAc;GACd,OAAO;GACR;UAEI,OAAO;AACZ,YAAU;AACV,SAAO;GACL,cAAc,iBAAiB,MAAM,KAAK,YAAY,WAAW,OAAO,wBAAwB,QAAQ,OAAO;GAC/G,cAAc;GACd,OAAO,WAAW,YAAY;GAC/B;WAEK;AACN,MAAI,iBAAiB,KACnB,cAAa,cAAc;;;AAKjC,eAAe,sBACb,SACA,UACA,OACA,YACA,qBAC+B;CAC/B,MAAM,iBAAiB,2BAA2B,UAAU,oBAAoB;CAChF,IAAI;AAEJ,MAAK,IAAI,aAAa,GAAG,cAAc,eAAe,WAAW,cAAc,GAAG;AAChF,gBAAc,QAAQ,eAAe;GACnC,GAAI,eAAe,YAAY,IAC3B;IACE,WAAW,eAAe;IAC1B;IACD,GACD,EAAE;GACN;GACA,MAAM,SAAS;GACf,OAAO;GACR,CAAC;AACF,gBAAc,MAAM,YAAY,SAAS,UAAU,OAAO,eAAe,QAAQ;AACjF,MAAI,YAAY,UAAU,SACxB,QAAO;;AAIX,QAAO,eAAe;EACpB,cAAc;EACd,8BAAc,IAAI,KAAK;EACvB,OAAO;EACR;;AAGH,SAAS,yBACP,SACA,oBACM;AACN,KAAI,QAAQ,UAAU,UAAU;AAC9B,qBAAmB,MAAM,KAAK,EAAE;AAChC;;AAGF,KAAI,QAAQ,aAAa,SAAS,GAAG;AACnC,qBAAmB,MAAM,KAAK,EAAE;AAChC;;AAGF,oBAAmB,MAAM,KAAK,QAAQ,aAAa,IAAI,QAAQ,IAAI,EAAE;CACrE,MAAM,aAAa,QAAQ,aAAa,IAAI,QAAQ;AACpD,KAAI,cAAc,KAChB,oBAAmB,MAAM,KAAK,WAAW;;AAiD7C,SAAS,kBAAkB,iBAAiE;CAG1F,SAAS,aACP,MACA,KACA,SACM;AACN,kBAAgB,KAAK;GACnB,iBAAiB,yBAAyB,SAAS,YAAY;GAC/D,OAAO,SAAS;GAChB;GACK;GACN,CAAC;;AAGJ,QAAO;EACL,QAAQ;EACR,gBAAgB,YAAY,QAAQ,KAAK,SAAS;GAChD,MAAM,WAAW,SAAS,eAAe,OAAO,KAAA,IAAY,EAAE;AAE9D,UAAO,SAAS,OAAO,UAAU;AAC/B,oBAAgB,KAAK;KACnB,aAAa,SAAS;KACtB,iBAAiB,yBAAyB,SAAS,kBAAkB;KACrE;KACA,MAAM,GAAG,WAAW,IAAI,QAAQ;KAChC;KACK;KACN,CAAC;KACF;;EAEL;;AAGH,IAAI,mBAAgD,EAAE;AAEtD,SAAS,gBAAmB,OAAkC,UAAsB;AAClF,oBAAmB,CAAC,GAAG,kBAAkB,MAAM;AAE/C,KAAI;AACF,SAAO,UAAU;WAEX;AACN,qBAAmB,iBAAiB,MAAM,GAAG,GAAG;;;AAIpD,SAAS,iBAA4C;CACnD,MAAM,SAAS,iBAAiB,GAAG,GAAG;AACtC,KAAI,UAAU,KACZ,OAAM,IAAI,MAAM,0EAA0E;AAG5F,QAAO;;AAiBT,SAAgB,OACd,MACA,KACA,SACM;AACN,iBAAgB,CAAC,KAAK;EACpB,iBAAiB,yBAAyB,SAAS,YAAY;EAC/D,OAAO,SAAS;EAChB;EACK;EACN,CAAC;;;;;AAMJ,SAAgB,gBACd,YACA,QACA,KACA,SACM;CACN,MAAM,WAAW,SAAS,eAAe,OAAO,KAAA,IAAY,EAAE;AAE9D,QAAO,SAAS,OAAO,UAAU;AAC/B,kBAAgB,CAAC,KAAK;GACpB,aAAa,SAAS;GACtB,iBAAiB,yBAAyB,SAAS,kBAAkB;GACrE;GACA,MAAM,GAAG,WAAW,IAAI,QAAQ;GAChC;GACK;GACN,CAAC;GACF;;;;;;;;;;;;;AAcJ,SAAS,uBACP,UACA,iBACA,oBACoB;CACpB,MAAM,cAAc,oBAAoB,QAAQ,SAAS,eAAe,iBAAiB;AACzF,KAAI,eAAe,KACjB;AAGF,KAAI,CAAC,OAAO,SAAS,YAAY,IAAI,CAAC,OAAO,UAAU,YAAY,IAAI,eAAe,EACpF,OAAM,IAAI,MAAM,kCAAkC,OAAO,YAAY,GAAG;AAG1E,QAAO;;AAGT,SAAS,oBAAoB,UAAmC,iBAAiC;AAC/F,QAAO,SAAS,YAAY;;;;;;;;;AAU9B,SAAgB,aACd,MACA,OACA,UAA+B,EAAE,EACjC;CACA,MAAM,kBAA6C,EAAE;CACrD,MAAM,UAAU,kBAAkB,gBAAgB;AAClD,iBAAgB,uBAAuB;AACrC,MAAI,MAAM,SAAS,GAAG;AACnB,SAAiD,QAAQ;AAC1D;;AAGA,SAAsB;GACxB;CAEF,MAAM,cAAc,QAAQ,eAAe;CAC3C,MAAM,sBAAsB,yBAAyB,SAAS,eAAe;CAE7E,MAAM,aAAa,WAAW;EAC5B;EACA;EACA,MAAM,WAAW;GACf,aAAa,QAAQ;GACrB,iBAAiB;GACjB,IAAI;GACJ,MAAM,IAAI,SAAiC;AACzC,QAAI,gBAAgB,WAAW,EAC7B,QAAO,EACL,QAAQ,CAAC;KAAE,MAAM;KAAS,OAAO;KAAG,CAAC,EACtC;IAGH,MAAM,aAAa,gBAAgB;IACnC,MAAM,qBAAqD;KACzD,OAAO,EAAE;KACT,OAAO,EAAE;KACV;IACD,MAAM,sBAAsB,EAAE;IAC9B,MAAM,6BAAa,IAAI,KAAsD;IAC7E,MAAM,iBAAiB,gBAAgB,MAAK,aAAY,2BAA2B,UAAU,oBAAoB,CAAC,cAAc,EAAE;IAClI,MAAM,yBAAyB,QAAQ,KAAK,MAAM,MAAM,eAAe,QAAQ;AAE/E,QAAI,CAAC,eACH,OAAM,QAAQ,IACZ,gBAAgB,IAAI,OAAO,UAAU,UAAU;KAC7C,MAAM,cAAc,YAAY;MAC9B,MAAM,UAAU,MAAM,sBAAsB,SAAS,UAAU,OAAO,YAAY,oBAAoB;AACtG,kBAAY,QAAQ,eAAe;OACjC,GAAI,QAAQ,gBAAgB,OAAO,EAAE,GAAG,EAAE,cAAc,QAAQ,cAAc;OAC9E;OACA,OAAO,QAAQ;OACf,MAAM,SAAS;OACf,OAAO;OACR,CAAC;AACF,+BAAyB,SAAS,mBAAmB;;KAGvD,MAAM,cAAc,uBAAuB,UAAU,wBAAwB,QAAQ,mBAAmB;AACxG,SAAI,eAAe,MAAM;AACvB,YAAM,aAAa;AACnB;;KAGF,MAAM,WAAW,oBAAoB,UAAU,oBAAoB;KACnE,MAAM,QAAQ,WAAW,IAAI,SAAS,IAAI,qBAAqB,YAAY;AAC3E,gBAAW,IAAI,UAAU,MAAM;AAC/B,WAAM,MAAM,IAAI,YAAY;MAC5B,CACH;SAEE;KACH,IAAI,gBAAwC,EAAE;KAC9C,IAAI,eAAe;AAEnB,cAAS;AACP,sBAAgB,MAAM,QAAQ,IAC5B,gBAAgB,IAAI,OAAO,UAAU,UAAU;OAC7C,MAAM,cAAc,YAAY,MAAM,sBAAsB,SAAS,UAAU,OAAO,YAAY,oBAAoB;OACtH,MAAM,cAAc,uBAAuB,UAAU,wBAAwB,QAAQ,mBAAmB;AACxG,WAAI,eAAe,KACjB,QAAO,MAAM,aAAa;OAG5B,MAAM,WAAW,oBAAoB,UAAU,oBAAoB;OACnE,MAAM,QAAQ,WAAW,IAAI,SAAS,IAAI,qBAAqB,YAAY;AAC3E,kBAAW,IAAI,UAAU,MAAM;AAC/B,cAAO,MAAM,MAAM,IAAI,YAAY;QACnC,CACH;AAeD,UAAI,CAbmB,cAAc,MAAM,SAAS,UAAU;AAC5D,WAAI,QAAQ,UAAU,SACpB,QAAO;OAGT,MAAM,WAAW,gBAAgB;AACjC,WAAI,YAAY,KACd,QAAO;AAGT,cAAO,eAAe,2BAA2B,UAAU,oBAAoB,CAAC;QAChF,CAGA;AAGF,sBAAgB;;AAGlB,mBAAc,SAAS,SAAS,UAAU;MACxC,MAAM,WAAW,gBAAgB;AACjC,UAAI,YAAY,KACd;AAGF,kBAAY,QAAQ,eAAe;OACjC,GAAI,QAAQ,gBAAgB,OAAO,EAAE,GAAG,EAAE,cAAc,QAAQ,cAAc;OAC9E;OACA,OAAO,QAAQ;OACf,MAAM,SAAS;OACf,OAAO;OACR,CAAC;AACF,+BAAyB,SAAS,mBAAmB;OACrD;;AAcJ,WAAO,EACL,QAZc,OAAO,KAAK,mBAAmB,CAC5C,QAAO,SAAQ,mBAAmB,MAAM,SAAS,EAAE,CACnD,KAAK,SAAS;KACb,MAAM,SAAS,mBAAmB;AAElC,YAAO;MACL;MACA,OAHY,OAAO,QAAQ,KAAK,UAAU,MAAM,OAAO,EAAE,GAG1C,OAAO;MACvB;MACD,EAIH;;GAEJ,CAAC;EACH,CAAC;AAEF,wBAAuB,WAAW;AAElC,QAAO;;;;;AAMT,MAAa,eAAe"}

package/dist/{models-D_MsBtYw.mjs → models-DIGdOUpJ.mjs}

@@ -11,4 +11,4 @@ function resolveModelByName(models, name) {
 //#endregion
 export { resolveModelByName as t };
 
-//# sourceMappingURL=models-D_MsBtYw.mjs.map
+//# sourceMappingURL=models-DIGdOUpJ.mjs.map

package/dist/models-DIGdOUpJ.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"models-DIGdOUpJ.mjs","names":[],"sources":["../src/config/models.ts"],"sourcesContent":["import type { TaskExecutionPolicy } from './types'\n\n/**\n * Canonical model definition consumed by vieval runtime and config.\n *\n * Use when:\n * - declaring models in `vieval.config.*`\n * - resolving task runtime models by id, alias, or concrete model name\n *\n * Expects:\n * - `id` to be stable and unique within one config\n * - `inferenceExecutorId` to match scheduler/executor identifiers\n *\n * Returns:\n * - one normalized model registration record\n */\nexport interface ModelDefinition {\n  /**\n   * Stable model id.\n   */\n  id: string\n  /**\n   * Inference-executor id used for matching and reporting.\n   */\n  inferenceExecutorId: string\n  /**\n   * Executor reference passed through config.\n   *\n   * `vieval` core treats this as opaque runtime metadata. Builder plugins can\n   * narrow this field with plugin-specific executor input types.\n   */\n  inferenceExecutor: unknown\n  /**\n   * Concrete model name passed to the inference executor.\n   */\n  model: string\n  /**\n   * Alias names that can resolve this model.\n   */\n  aliases: string[]\n  /**\n   * Optional execution policy hints attached to this model.\n   */\n  executionPolicy?: TaskExecutionPolicy\n  /**\n   * Optional model-level call parameters.\n   */\n  parameters?: Record<string, unknown>\n}\n\n/**\n * Resolves one model by id, model name, or alias in registration order.\n *\n * Returns:\n * - the first matching model, or `undefined` when no match exists\n */\nexport function resolveModelByName(\n  models: readonly ModelDefinition[],\n  name: string,\n): ModelDefinition | undefined {\n  return models.find(model => model.id === name || model.model === name || model.aliases.includes(name))\n}\n"],"mappings":";;;;;;;AAwDA,SAAgB,mBACd,QACA,MAC6B;AAC7B,QAAO,OAAO,MAAK,UAAS,MAAM,OAAO,QAAQ,MAAM,UAAU,QAAQ,MAAM,QAAQ,SAAS,KAAK,CAAC"}

package/dist/plugins/chat-models/index.d.mts

@@ -1,4 +1,4 @@
-import { M as ModelDefinition, l as MatrixDefinition, t as ConfigHookPlugin, x as TaskRunContext } from "../../index-OEdqjQSe.mjs";
+import { C as TaskRunContext, P as ModelDefinition, b as TaskExecutionPolicy, l as MatrixDefinition, t as ConfigHookPlugin } from "../../index-DBZKkpBe.mjs";
 
 //#region src/plugins/chat-models/runtime-config.d.ts
 /**
@@ -382,6 +382,26 @@ interface ChatModelFromBaseOptions {
    * Alias names used by `resolveModelByName`.
    */
   aliases?: string[];
+  /**
+   * Optional execution policy hints attached to this model.
+   */
+  executionPolicy?: TaskExecutionPolicy;
+  /**
+   * Additional retries allowed within the current attempt.
+   *
+   * @default 0
+   */
+  autoRetry?: number;
+  /**
+   * Additional full task attempts allowed after the current attempt settles.
+   *
+   * @default 0
+   */
+  autoAttempt?: number;
+  /**
+   * Timeout in milliseconds for model-backed work.
+   */
+  timeout?: number;
   /**
    * Optional model-level call parameters.
    */

package/dist/plugins/chat-models/index.mjs

@@ -1,4 +1,4 @@
-import { n as requiredEnvFrom, t as envFrom } from "../../env-BFSjny07.mjs";
+import { n as requiredEnvFrom, t as envFrom } from "../../env--94B0UtW.mjs";
 import process from "node:process";
 import { errorMessageFrom } from "@moeru/std";
 //#region src/plugins/chat-models/runtime-config.ts
@@ -288,6 +288,31 @@ function emitChatModelErrorTelemetry(context, options) {
 }
 //#endregion
 //#region src/plugins/chat-models/index.ts
+function normalizeExecutionPolicy(policy) {
+	if (policy == null) return;
+	const normalized = {
+		autoAttempt: policy.autoAttempt,
+		autoRetry: policy.autoRetry,
+		timeout: policy.timeout
+	};
+	return Object.values(normalized).some((value) => value != null) ? normalized : void 0;
+}
+function hasJudgeAlias(model) {
+	return [
+		...model.aliases ?? [],
+		...model.id == null ? [] : [model.id],
+		model.model
+	].some((value) => value.toLowerCase().includes("judge"));
+}
+function resolveModelExecutionPolicy(options) {
+	const explicitPolicy = normalizeExecutionPolicy({
+		autoAttempt: options.autoAttempt ?? options.executionPolicy?.autoAttempt,
+		autoRetry: options.autoRetry ?? options.executionPolicy?.autoRetry,
+		timeout: options.timeout ?? options.executionPolicy?.timeout
+	});
+	if (explicitPolicy != null && Object.keys(explicitPolicy).length > 0) return explicitPolicy;
+	if (hasJudgeAlias(options)) return { autoRetry: 3 };
+}
 function normalizeInferenceExecutorId(inferenceExecutor, inferenceExecutorId) {
 	if (typeof inferenceExecutor === "string") return inferenceExecutor;
 	return inferenceExecutorId ?? "custom";
@@ -424,6 +449,7 @@ function chatModelFrom(options) {
 	} : void 0;
 	return {
 		aliases: options.aliases ?? [],
+		executionPolicy: resolveModelExecutionPolicy(options),
 		id: options.id ?? createDefaultModelId(inferenceExecutorId, options.model),
 		inferenceExecutor: fallbackInferenceExecutor,
 		inferenceExecutorId,