vieval 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +290 -0
  2. package/dist/assertions-DcAjfVDA.mjs +183 -0
  3. package/dist/assertions-DcAjfVDA.mjs.map +1 -0
  4. package/dist/cli/index.d.mts +11 -0
  5. package/dist/cli/index.mjs +1434 -0
  6. package/dist/cli/index.mjs.map +1 -0
  7. package/dist/config-D2fe1SnT.mjs +17 -0
  8. package/dist/config-D2fe1SnT.mjs.map +1 -0
  9. package/dist/config.d.mts +3 -0
  10. package/dist/config.mjs +3 -0
  11. package/dist/core/assertions/index.d.mts +2 -0
  12. package/dist/core/assertions/index.mjs +2 -0
  13. package/dist/core/inference-executors/index.d.mts +273 -0
  14. package/dist/core/inference-executors/index.mjs +225 -0
  15. package/dist/core/inference-executors/index.mjs.map +1 -0
  16. package/dist/core/processors/results/index.d.mts +96 -0
  17. package/dist/core/processors/results/index.mjs +64 -0
  18. package/dist/core/processors/results/index.mjs.map +1 -0
  19. package/dist/core/runner/index.d.mts +2 -0
  20. package/dist/core/runner/index.mjs +2 -0
  21. package/dist/expect-0jPJ7Zio.d.mts +2318 -0
  22. package/dist/expect-extensions-CwPtgTz8.mjs +13471 -0
  23. package/dist/expect-extensions-CwPtgTz8.mjs.map +1 -0
  24. package/dist/expect-i9WZWGrA.mjs +17 -0
  25. package/dist/expect-i9WZWGrA.mjs.map +1 -0
  26. package/dist/expect.d.mts +2 -0
  27. package/dist/expect.mjs +2 -0
  28. package/dist/index-DP7jsORl.d.mts +947 -0
  29. package/dist/index-oSXhM1zx.d.mts +314 -0
  30. package/dist/index.d.mts +92 -0
  31. package/dist/index.mjs +150 -0
  32. package/dist/index.mjs.map +1 -0
  33. package/dist/magic-string.es-CH1jwzMg.mjs +1013 -0
  34. package/dist/magic-string.es-CH1jwzMg.mjs.map +1 -0
  35. package/dist/models-D_MsBtYw.mjs +14 -0
  36. package/dist/models-D_MsBtYw.mjs.map +1 -0
  37. package/dist/plugin-DVaRZY2x.d.mts +84 -0
  38. package/dist/plugins/chat-models/index.d.mts +90 -0
  39. package/dist/plugins/chat-models/index.mjs +48 -0
  40. package/dist/plugins/chat-models/index.mjs.map +1 -0
  41. package/dist/registry-ChOjjdEC.mjs +245 -0
  42. package/dist/registry-ChOjjdEC.mjs.map +1 -0
  43. package/dist/runner-4ZsOveoY.mjs +480 -0
  44. package/dist/runner-4ZsOveoY.mjs.map +1 -0
  45. package/dist/testing/expect-extensions.d.mts +86 -0
  46. package/dist/testing/expect-extensions.mjs +2 -0
  47. package/package.json +88 -0
@@ -0,0 +1,245 @@
1
+ import { createRequire } from "node:module";
2
+ import process from "node:process";
3
+ import { access, readFile } from "node:fs/promises";
4
+ import { dirname, extname, isAbsolute, join, resolve } from "node:path";
5
+ import { pathToFileURL } from "node:url";
6
+ import { errorMessageFrom } from "@moeru/std";
7
+ import { createDefineConfig, loadConfig } from "c12";
8
+ import { loadEnv } from "vite";
9
+ //#region src/cli/config.ts
10
+ const matrixLayerKeys = new Set([
11
+ "disable",
12
+ "extend",
13
+ "override"
14
+ ]);
15
+ const ambiguousMatrixDefinitionErrorMessage = "Ambiguous matrix definition: cannot mix reserved layer keys (disable, extend, override) with matrix axis keys.";
16
+ const require = createRequire(import.meta.url);
17
+ /**
18
+ * Helper used by `vieval.config.*` for better type inference.
19
+ */
20
+ const defineConfig = createDefineConfig();
21
+ /**
22
+ * Loads `.env*` files using Vite's env resolution behavior.
23
+ *
24
+ * Use when:
25
+ * - `vieval.config.*` should mirror Vitest/Vite env loading semantics
26
+ * - config wants to populate top-level `env` via file-based values
27
+ *
28
+ * Expects:
29
+ * - `mode` to match the env file suffix (`.env.<mode>`)
30
+ * - `envDir` to point at the directory containing `.env` files
31
+ *
32
+ * Returns:
33
+ * - Key/value map compatible with `CliConfig['env']`
34
+ */
35
+ function loadEnv$1(mode, envDir, prefixes = "") {
36
+ return loadEnv(mode, envDir, prefixes);
37
+ }
38
+ async function applyVievalPlugins(config) {
39
+ let currentConfig = config;
40
+ const plugins = currentConfig.plugins ?? [];
41
+ for (const plugin of plugins) {
42
+ if (plugin.configVieval == null) continue;
43
+ const nextConfig = await plugin.configVieval(currentConfig);
44
+ if (nextConfig != null) currentConfig = {
45
+ ...currentConfig,
46
+ ...nextConfig
47
+ };
48
+ }
49
+ for (const plugin of plugins) await plugin.configVievalResolved?.(currentConfig);
50
+ return currentConfig;
51
+ }
52
+ async function isReadableFile(filePath) {
53
+ try {
54
+ await access(filePath);
55
+ return true;
56
+ } catch {
57
+ return false;
58
+ }
59
+ }
60
+ function isConfigFileExtensionUsingRequire(extension) {
61
+ return extension === ".cjs" || extension === ".cts";
62
+ }
63
+ function isConfigFileExtensionUsingJsonParse(extension) {
64
+ return extension === ".json";
65
+ }
66
+ async function importVievalConfigModule(filePath) {
67
+ const extension = extname(filePath);
68
+ if (isConfigFileExtensionUsingJsonParse(extension)) {
69
+ const raw = await readFile(filePath, "utf-8");
70
+ return JSON.parse(raw);
71
+ }
72
+ if (isConfigFileExtensionUsingRequire(extension)) return require(filePath);
73
+ return import(pathToFileURL(filePath).href);
74
+ }
75
+ function resolveConfigExport(moduleValue) {
76
+ if (moduleValue == null) return null;
77
+ if (typeof moduleValue !== "object") return moduleValue;
78
+ if ("default" in moduleValue) return moduleValue.default;
79
+ return moduleValue;
80
+ }
81
+ async function findNearestConfigFile(startDirectory) {
82
+ const supportedFileNames = [
83
+ "vieval.config.ts",
84
+ "vieval.config.mts",
85
+ "vieval.config.cts",
86
+ "vieval.config.js",
87
+ "vieval.config.mjs",
88
+ "vieval.config.cjs",
89
+ "vieval.config.json"
90
+ ];
91
+ let currentDirectory = resolve(startDirectory);
92
+ while (true) {
93
+ for (const fileName of supportedFileNames) {
94
+ const candidatePath = join(currentDirectory, fileName);
95
+ if (await isReadableFile(candidatePath)) return candidatePath;
96
+ }
97
+ const parentDirectory = dirname(currentDirectory);
98
+ if (parentDirectory === currentDirectory) return null;
99
+ currentDirectory = parentDirectory;
100
+ }
101
+ }
102
+ async function resolveVievalConfig(cwd, explicitConfigFilePath) {
103
+ const resolvedConfigFilePath = explicitConfigFilePath == null ? await findNearestConfigFile(cwd) : isAbsolute(explicitConfigFilePath) ? explicitConfigFilePath : resolve(cwd, explicitConfigFilePath);
104
+ if (explicitConfigFilePath != null && resolvedConfigFilePath != null && !await isReadableFile(resolvedConfigFilePath)) throw new Error(`Config file does not exist or is not readable: ${resolvedConfigFilePath}`);
105
+ if (resolvedConfigFilePath == null) return {
106
+ config: null,
107
+ configFilePath: null
108
+ };
109
+ return {
110
+ config: (await loadConfig({
111
+ configFile: resolvedConfigFilePath,
112
+ cwd,
113
+ dotenv: false,
114
+ envName: false,
115
+ extend: false,
116
+ import: importVievalConfigModule,
117
+ packageJson: false,
118
+ rcFile: false,
119
+ resolveModule: resolveConfigExport
120
+ })).config,
121
+ configFilePath: resolvedConfigFilePath
122
+ };
123
+ }
124
+ function isLayerMatrixDefinition(matrix) {
125
+ const matrixKeys = Object.keys(matrix);
126
+ return matrixKeys.length > 0 && matrixKeys.every((key) => matrixLayerKeys.has(key));
127
+ }
128
+ function assertNonAmbiguousMatrixDefinition(matrix) {
129
+ const matrixKeys = Object.keys(matrix);
130
+ const hasReservedKeys = matrixKeys.some((key) => matrixLayerKeys.has(key));
131
+ const hasAxisKeys = matrixKeys.some((key) => !matrixLayerKeys.has(key));
132
+ if (hasReservedKeys && hasAxisKeys) throw new TypeError(ambiguousMatrixDefinitionErrorMessage);
133
+ }
134
+ function normalizeMatrixLayerInput(matrix) {
135
+ if (matrix == null) return;
136
+ assertNonAmbiguousMatrixDefinition(matrix);
137
+ if (isLayerMatrixDefinition(matrix)) return matrix;
138
+ return { extend: matrix };
139
+ }
140
+ function normalizeProjectConfig(project, cwd, inheritedModels) {
141
+ const include = project.include ?? [
142
+ "**/*.eval.ts",
143
+ "**/*.eval.mts",
144
+ "**/*.eval.cts",
145
+ "**/*.eval.js",
146
+ "**/*.eval.mjs",
147
+ "**/*.eval.cjs"
148
+ ];
149
+ const exclude = project.exclude ?? [
150
+ "**/node_modules/**",
151
+ "**/dist/**",
152
+ "**/.git/**"
153
+ ];
154
+ const models = project.models ?? [...inheritedModels];
155
+ const inferenceExecutors = project.inferenceExecutors ?? (models.length > 0 ? models.map((model) => ({ id: model.id })) : [{ id: "default" }]);
156
+ const root = project.root == null ? cwd : isAbsolute(project.root) ? project.root : resolve(cwd, project.root);
157
+ return {
158
+ exclude,
159
+ executor: project.executor,
160
+ include,
161
+ evalMatrix: normalizeMatrixLayerInput(project.evalMatrix),
162
+ models,
163
+ name: project.name,
164
+ inferenceExecutors,
165
+ runMatrix: normalizeMatrixLayerInput(project.runMatrix),
166
+ root
167
+ };
168
+ }
169
+ function normalizeConfig(config, cwd) {
170
+ const projects = config?.projects ?? [{ name: "default" }];
171
+ const inheritedModels = config?.models ?? [];
172
+ return projects.map((project) => normalizeProjectConfig(project, cwd, inheritedModels));
173
+ }
174
+ /**
175
+ * Loads nearest `vieval.config.*` and returns normalized project definitions.
176
+ *
177
+ * Call stack:
178
+ *
179
+ * {@link loadVievalCliConfig}
180
+ * -> {@link resolveVievalConfig}
181
+ * -> {@link normalizeConfig}
182
+ * -> {@link NormalizedCliProjectConfig}[]
183
+ *
184
+ * Use when:
185
+ * - CLI orchestration needs project includes/excludes similar to Vitest
186
+ * - callers want config auto-discovery without manual imports in eval files
187
+ */
188
+ async function loadVievalCliConfig(options = {}) {
189
+ const cwd = options.cwd ?? process.cwd();
190
+ try {
191
+ const loadedConfig = await resolveVievalConfig(cwd, options.configFilePath);
192
+ if (loadedConfig.configFilePath == null || loadedConfig.config == null) return {
193
+ configFilePath: null,
194
+ env: {},
195
+ projects: normalizeConfig(null, cwd)
196
+ };
197
+ const config = await applyVievalPlugins(loadedConfig.config);
198
+ return {
199
+ configFilePath: loadedConfig.configFilePath,
200
+ env: config.env ?? {},
201
+ projects: normalizeConfig(config, dirname(loadedConfig.configFilePath))
202
+ };
203
+ } catch (error) {
204
+ const errorMessage = errorMessageFrom(error) ?? "Unknown config loading error.";
205
+ const configFilePath = options.configFilePath == null ? "vieval.config" : isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath);
206
+ throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
207
+ }
208
+ }
209
+ //#endregion
210
+ //#region src/dsl/registry.ts
211
+ const registeredDefinitionsByModule = /* @__PURE__ */ new Map();
212
+ let activeModuleHref = null;
213
+ /**
214
+ * Starts module-scoped eval registration collection.
215
+ */
216
+ function beginModuleRegistration(moduleHref) {
217
+ activeModuleHref = moduleHref;
218
+ }
219
+ /**
220
+ * Ends module-scoped eval registration collection.
221
+ */
222
+ function endModuleRegistration() {
223
+ activeModuleHref = null;
224
+ }
225
+ /**
226
+ * Registers one eval definition against the currently active module.
227
+ */
228
+ function registerEvalDefinition(definition) {
229
+ if (activeModuleHref == null) return;
230
+ const existing = registeredDefinitionsByModule.get(activeModuleHref) ?? [];
231
+ existing.push(definition);
232
+ registeredDefinitionsByModule.set(activeModuleHref, existing);
233
+ }
234
+ /**
235
+ * Consumes registered definitions for one module and clears stored state.
236
+ */
237
+ function consumeModuleRegistrations(moduleHref) {
238
+ const definitions = registeredDefinitionsByModule.get(moduleHref) ?? [];
239
+ registeredDefinitionsByModule.delete(moduleHref);
240
+ return definitions;
241
+ }
242
+ //#endregion
243
+ export { defineConfig as a, registerEvalDefinition as i, consumeModuleRegistrations as n, loadEnv$1 as o, endModuleRegistration as r, loadVievalCliConfig as s, beginModuleRegistration as t };
244
+
245
+ //# sourceMappingURL=registry-ChOjjdEC.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"registry-ChOjjdEC.mjs","names":["loadEnv","loadViteEnv"],"sources":["../src/cli/config.ts","../src/dsl/registry.ts"],"sourcesContent":["import type { ConfigHookPlugin, MatrixDefinition, MatrixLayer, TaskRunContext } from '../config'\nimport type { ModelDefinition } from '../config/models'\nimport type { RunResult, TaskExecutionContext } from '../core/runner'\nimport type { InferenceExecutor, ScheduledTask } from '../core/runner/schedule'\n\nimport process from 'node:process'\n\nimport { access, readFile } from 'node:fs/promises'\nimport { createRequire } from 'node:module'\nimport { dirname, extname, isAbsolute, join, resolve } from 'node:path'\nimport { pathToFileURL } from 'node:url'\n\nimport { errorMessageFrom } from '@moeru/std'\nimport { createDefineConfig, loadConfig } from 'c12'\nimport { loadEnv as loadViteEnv } from 'vite'\n\nconst matrixLayerKeys = new Set(['disable', 'extend', 'override'])\nconst ambiguousMatrixDefinitionErrorMessage = 'Ambiguous matrix definition: cannot mix reserved layer keys (disable, extend, override) with matrix axis keys.'\nconst require = createRequire(import.meta.url)\n\n/**\n * CLI plugin shape bound to the full CLI config object.\n */\nexport type CliConfigPlugin = ConfigHookPlugin<CliConfig>\n\n/**\n * Defines one project block for `vieval run`.\n */\nexport interface CliProjectConfig {\n /**\n * Project label used in summary output.\n */\n name: string\n /**\n * Project root used for include/exclude glob matching.\n *\n * @default process cwd\n */\n root?: string\n /**\n * Glob patterns for eval file discovery.\n *\n * @default Common eval file globs for TypeScript and JavaScript module formats.\n */\n include?: string[]\n /**\n * Glob patterns excluded from discovery.\n *\n * @default Common exclusion globs for dependencies, build output, and VCS directories.\n */\n exclude?: string[]\n /**\n * Providers expanded by scheduler.\n *\n * @default [{ id: 'default' }]\n */\n inferenceExecutors?: InferenceExecutor[]\n /**\n * Model definitions available to project runtime execution.\n *\n * Inference executors control schedule fan-out, while models provide\n * runtime lookup metadata for `context.model(...)` during task execution.\n *\n * @default inherited from top-level config models\n */\n models?: ModelDefinition[]\n /**\n * Optional run-time matrix dimensions.\n */\n runMatrix?: MatrixDefinition | MatrixLayer\n /**\n * Optional eval-time matrix dimensions.\n */\n evalMatrix?: MatrixDefinition | MatrixLayer\n /**\n * Optional task executor.\n *\n * Use when this project should execute live inferenceExecutor requests.\n * If omitted, `vieval run` performs collection + scheduling only.\n */\n executor?: (task: ScheduledTask, context: CliProjectExecutorContext) => Promise<RunResult>\n /**\n * Optional project-local plugins.\n */\n plugins?: CliConfigPlugin[]\n}\n\n/**\n * Execution context exposed to project-level `executor` implementations.\n *\n * Use when:\n * - a project executor needs the task-scoped model resolver plus case reporter hooks\n * - custom scheduling logic wants the same hook shape as `TaskRunContext`\n *\n * Expects:\n * - `model` resolves configured models for the current task\n * - `reporterHooks` follows `TaskRunContext['reporterHooks']`\n */\nexport interface CliProjectExecutorContext extends TaskExecutionContext {\n reporterHooks?: TaskRunContext['reporterHooks']\n}\n\n/**\n * Top-level CLI config loaded from `vieval.config.*`.\n */\nexport interface CliConfig {\n /**\n * Project list expanded by `vieval run`.\n *\n * @default [{ name: 'default' }]\n */\n projects?: CliProjectConfig[]\n /**\n * Global model definitions inherited by projects.\n *\n * @default []\n */\n models?: ModelDefinition[]\n /**\n * Global config plugins.\n *\n * @default []\n */\n plugins?: CliConfigPlugin[]\n /**\n * Environment variables injected into `process.env` during `vieval run`.\n *\n * Use when:\n * - eval tasks depend on runtime env values (for example inferenceExecutor API keys)\n * - config wants deterministic env values without shell-level exports\n *\n * @default {}\n */\n env?: NodeJS.ProcessEnv\n}\n\n/**\n * Normalized CLI project used by runtime orchestration.\n */\nexport interface NormalizedCliProjectConfig {\n exclude: string[]\n executor?: (task: ScheduledTask, context: CliProjectExecutorContext) => Promise<RunResult>\n include: string[]\n runMatrix?: MatrixLayer\n evalMatrix?: MatrixLayer\n models: ModelDefinition[]\n name: string\n inferenceExecutors: InferenceExecutor[]\n root: string\n}\n\n/**\n * Result of loading and normalizing a config file.\n */\nexport interface LoadedCliConfig {\n configFilePath: string | null\n env: NodeJS.ProcessEnv\n projects: NormalizedCliProjectConfig[]\n}\n\n/**\n * Runtime options for config loading.\n */\nexport interface LoadVievalCliConfigOptions {\n /**\n * Starting directory for config lookup.\n *\n * @default process.cwd()\n */\n cwd?: string\n /**\n * Explicit config file path.\n */\n configFilePath?: string\n}\n\n/**\n * Helper used by `vieval.config.*` for better type inference.\n */\nexport const defineConfig = createDefineConfig<CliConfig>()\n\n/**\n * Loads `.env*` files using Vite's env resolution behavior.\n *\n * Use when:\n * - `vieval.config.*` should mirror Vitest/Vite env loading semantics\n * - config wants to populate top-level `env` via file-based values\n *\n * Expects:\n * - `mode` to match the env file suffix (`.env.<mode>`)\n * - `envDir` to point at the directory containing `.env` files\n *\n * Returns:\n * - Key/value map compatible with `CliConfig['env']`\n */\nexport function loadEnv(mode: string, envDir: string, prefixes: string | string[] = ''): NodeJS.ProcessEnv {\n return loadViteEnv(mode, envDir, prefixes)\n}\n\nasync function applyVievalPlugins(config: CliConfig): Promise<CliConfig> {\n let currentConfig: CliConfig = config\n const plugins = currentConfig.plugins ?? []\n\n for (const plugin of plugins) {\n if (plugin.configVieval == null) {\n continue\n }\n\n const nextConfig = await plugin.configVieval(currentConfig)\n if (nextConfig != null) {\n currentConfig = {\n ...currentConfig,\n ...nextConfig,\n }\n }\n }\n\n for (const plugin of plugins) {\n await plugin.configVievalResolved?.(currentConfig)\n }\n\n return currentConfig\n}\n\nasync function isReadableFile(filePath: string): Promise<boolean> {\n try {\n await access(filePath)\n return true\n }\n catch {\n return false\n }\n}\n\nfunction isConfigFileExtensionUsingRequire(extension: string): boolean {\n return extension === '.cjs' || extension === '.cts'\n}\n\nfunction isConfigFileExtensionUsingJsonParse(extension: string): boolean {\n return extension === '.json'\n}\n\nasync function importVievalConfigModule(filePath: string): Promise<unknown> {\n const extension = extname(filePath)\n\n if (isConfigFileExtensionUsingJsonParse(extension)) {\n const raw = await readFile(filePath, 'utf-8')\n return JSON.parse(raw) as unknown\n }\n\n if (isConfigFileExtensionUsingRequire(extension)) {\n return require(filePath) as unknown\n }\n\n return import(pathToFileURL(filePath).href)\n}\n\nfunction resolveConfigExport(moduleValue: unknown): unknown {\n if (moduleValue == null) {\n return null\n }\n\n if (typeof moduleValue !== 'object') {\n return moduleValue\n }\n\n if ('default' in moduleValue) {\n return (moduleValue as { default: unknown }).default\n }\n\n return moduleValue\n}\n\nasync function findNearestConfigFile(startDirectory: string): Promise<string | null> {\n const supportedFileNames = [\n 'vieval.config.ts',\n 'vieval.config.mts',\n 'vieval.config.cts',\n 'vieval.config.js',\n 'vieval.config.mjs',\n 'vieval.config.cjs',\n 'vieval.config.json',\n ]\n\n let currentDirectory = resolve(startDirectory)\n\n while (true) {\n for (const fileName of supportedFileNames) {\n const candidatePath = join(currentDirectory, fileName)\n if (await isReadableFile(candidatePath)) {\n return candidatePath\n }\n }\n\n const parentDirectory = dirname(currentDirectory)\n if (parentDirectory === currentDirectory) {\n return null\n }\n currentDirectory = parentDirectory\n }\n}\n\nasync function resolveVievalConfig(\n cwd: string,\n explicitConfigFilePath: string | undefined,\n): Promise<{\n config: CliConfig | null\n configFilePath: string | null\n}> {\n const resolvedConfigFilePath = explicitConfigFilePath == null\n ? await findNearestConfigFile(cwd)\n : (isAbsolute(explicitConfigFilePath) ? explicitConfigFilePath : resolve(cwd, explicitConfigFilePath))\n\n if (explicitConfigFilePath != null && resolvedConfigFilePath != null && !await isReadableFile(resolvedConfigFilePath)) {\n throw new Error(`Config file does not exist or is not readable: ${resolvedConfigFilePath}`)\n }\n\n if (resolvedConfigFilePath == null) {\n return {\n config: null,\n configFilePath: null,\n }\n }\n\n const loaded = await loadConfig<CliConfig>({\n configFile: resolvedConfigFilePath,\n cwd,\n dotenv: false,\n envName: false,\n extend: false,\n import: importVievalConfigModule,\n packageJson: false,\n rcFile: false,\n resolveModule: resolveConfigExport,\n })\n return {\n config: loaded.config,\n configFilePath: resolvedConfigFilePath,\n }\n}\n\nfunction isLayerMatrixDefinition(matrix: MatrixDefinition | MatrixLayer): matrix is MatrixLayer {\n const matrixKeys = Object.keys(matrix)\n return (\n matrixKeys.length > 0\n && matrixKeys.every(key => matrixLayerKeys.has(key))\n )\n}\n\nfunction assertNonAmbiguousMatrixDefinition(matrix: MatrixDefinition | MatrixLayer): void {\n const matrixKeys = Object.keys(matrix)\n const hasReservedKeys = matrixKeys.some(key => matrixLayerKeys.has(key))\n const hasAxisKeys = matrixKeys.some(key => !matrixLayerKeys.has(key))\n\n if (hasReservedKeys && hasAxisKeys) {\n throw new TypeError(ambiguousMatrixDefinitionErrorMessage)\n }\n}\n\nfunction normalizeMatrixLayerInput(matrix: MatrixDefinition | MatrixLayer | undefined): MatrixLayer | undefined {\n if (matrix == null) {\n return undefined\n }\n\n assertNonAmbiguousMatrixDefinition(matrix)\n\n if (isLayerMatrixDefinition(matrix)) {\n return matrix\n }\n\n return {\n extend: matrix,\n }\n}\n\nfunction normalizeProjectConfig(\n project: CliProjectConfig,\n cwd: string,\n inheritedModels: readonly ModelDefinition[],\n): NormalizedCliProjectConfig {\n const include = project.include ?? [\n '**/*.eval.ts',\n '**/*.eval.mts',\n '**/*.eval.cts',\n '**/*.eval.js',\n '**/*.eval.mjs',\n '**/*.eval.cjs',\n ]\n const exclude = project.exclude ?? [\n '**/node_modules/**',\n '**/dist/**',\n '**/.git/**',\n ]\n const models = project.models ?? [...inheritedModels]\n const inferenceExecutors = project.inferenceExecutors ?? (\n models.length > 0\n ? models.map(model => ({ id: model.id }))\n : [{ id: 'default' }]\n )\n const root = project.root == null\n ? cwd\n : (isAbsolute(project.root) ? project.root : resolve(cwd, project.root))\n\n return {\n exclude,\n executor: project.executor,\n include,\n evalMatrix: normalizeMatrixLayerInput(project.evalMatrix),\n models,\n name: project.name,\n inferenceExecutors,\n runMatrix: normalizeMatrixLayerInput(project.runMatrix),\n root,\n }\n}\n\nfunction normalizeConfig(config: CliConfig | null | undefined, cwd: string): NormalizedCliProjectConfig[] {\n const projects = config?.projects ?? [{ name: 'default' }]\n const inheritedModels = config?.models ?? []\n return projects.map(project => normalizeProjectConfig(project, cwd, inheritedModels))\n}\n\n/**\n * Loads nearest `vieval.config.*` and returns normalized project definitions.\n *\n * Call stack:\n *\n * {@link loadVievalCliConfig}\n * -> {@link resolveVievalConfig}\n * -> {@link normalizeConfig}\n * -> {@link NormalizedCliProjectConfig}[]\n *\n * Use when:\n * - CLI orchestration needs project includes/excludes similar to Vitest\n * - callers want config auto-discovery without manual imports in eval files\n */\nexport async function loadVievalCliConfig(options: LoadVievalCliConfigOptions = {}): Promise<LoadedCliConfig> {\n const cwd = options.cwd ?? process.cwd()\n try {\n const loadedConfig = await resolveVievalConfig(cwd, options.configFilePath)\n if (loadedConfig.configFilePath == null || loadedConfig.config == null) {\n return {\n configFilePath: null,\n env: {},\n projects: normalizeConfig(null, cwd),\n }\n }\n\n const config = await applyVievalPlugins(loadedConfig.config)\n\n return {\n configFilePath: loadedConfig.configFilePath,\n env: config.env ?? {},\n projects: normalizeConfig(config, dirname(loadedConfig.configFilePath)),\n }\n }\n catch (error) {\n const errorMessage = errorMessageFrom(error) ?? 'Unknown config loading error.'\n const configFilePath = options.configFilePath == null\n ? 'vieval.config'\n : (isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath))\n throw new Error(`Failed to load vieval config \"${configFilePath}\": ${errorMessage}`, { cause: error })\n }\n}\n","import type { EvalDefinition } from '../config'\n\nconst registeredDefinitionsByModule = new Map<string, EvalDefinition[]>()\nlet activeModuleHref: string | null = null\n\n/**\n * Starts module-scoped eval registration collection.\n */\nexport function beginModuleRegistration(moduleHref: string): void {\n activeModuleHref = moduleHref\n}\n\n/**\n * Ends module-scoped eval registration collection.\n */\nexport function endModuleRegistration(): void {\n activeModuleHref = null\n}\n\n/**\n * Registers one eval definition against the currently active module.\n */\nexport function registerEvalDefinition(definition: EvalDefinition): void {\n if (activeModuleHref == null) {\n return\n }\n\n const existing = registeredDefinitionsByModule.get(activeModuleHref) ?? []\n existing.push(definition)\n registeredDefinitionsByModule.set(activeModuleHref, existing)\n}\n\n/**\n * Consumes registered definitions for one module and clears stored state.\n */\nexport function consumeModuleRegistrations(moduleHref: string): EvalDefinition[] {\n const definitions = registeredDefinitionsByModule.get(moduleHref) ?? []\n registeredDefinitionsByModule.delete(moduleHref)\n return definitions\n}\n"],"mappings":";;;;;;;;;AAgBA,MAAM,kBAAkB,IAAI,IAAI;CAAC;CAAW;CAAU;CAAW,CAAC;AAClE,MAAM,wCAAwC;AAC9C,MAAM,UAAU,cAAc,OAAO,KAAK,IAAI;;;;AAiK9C,MAAa,eAAe,oBAA+B;;;;;;;;;;;;;;;AAgB3D,SAAgBA,UAAQ,MAAc,QAAgB,WAA8B,IAAuB;AACzG,QAAOC,QAAY,MAAM,QAAQ,SAAS;;AAG5C,eAAe,mBAAmB,QAAuC;CACvE,IAAI,gBAA2B;CAC/B,MAAM,UAAU,cAAc,WAAW,EAAE;AAE3C,MAAK,MAAM,UAAU,SAAS;AAC5B,MAAI,OAAO,gBAAgB,KACzB;EAGF,MAAM,aAAa,MAAM,OAAO,aAAa,cAAc;AAC3D,MAAI,cAAc,KAChB,iBAAgB;GACd,GAAG;GACH,GAAG;GACJ;;AAIL,MAAK,MAAM,UAAU,QACnB,OAAM,OAAO,uBAAuB,cAAc;AAGpD,QAAO;;AAGT,eAAe,eAAe,UAAoC;AAChE,KAAI;AACF,QAAM,OAAO,SAAS;AACtB,SAAO;SAEH;AACJ,SAAO;;;AAIX,SAAS,kCAAkC,WAA4B;AACrE,QAAO,cAAc,UAAU,cAAc;;AAG/C,SAAS,oCAAoC,WAA4B;AACvE,QAAO,cAAc;;AAGvB,eAAe,yBAAyB,UAAoC;CAC1E,MAAM,YAAY,QAAQ,SAAS;AAEnC,KAAI,oCAAoC,UAAU,EAAE;EAClD,MAAM,MAAM,MAAM,SAAS,UAAU,QAAQ;AAC7C,SAAO,KAAK,MAAM,IAAI;;AAGxB,KAAI,kCAAkC,UAAU,CAC9C,QAAO,QAAQ,SAAS;AAG1B,QAAO,OAAO,cAAc,SAAS,CAAC;;AAGxC,SAAS,oBAAoB,aAA+B;AAC1D,KAAI,eAAe,KACjB,QAAO;AAGT,KAAI,OAAO,gBAAgB,SACzB,QAAO;AAGT,KAAI,aAAa,YACf,QAAQ,YAAqC;AAG/C,QAAO;;AAGT,eAAe,sBAAsB,gBAAgD;CACnF,MAAM,qBAAqB;EACzB;EACA;EACA;EACA;EACA;EACA;EACA;EACD;CAED,IAAI,mBAAmB,QAAQ,eAAe;AAE9C,QAAO,MAAM;AACX,OAAK,MAAM,YAAY,oBAAoB;GACzC,MAAM,gBAAgB,KAAK,kBAAkB,SAAS;AACtD,OAAI,MAAM,eAAe,cAAc,CACrC,QAAO;;EAIX,MAAM,kBAAkB,QAAQ,iBAAiB;AACjD,MAAI,oBAAoB,iBACtB,QAAO;AAET,qBAAmB;;;AAIvB,eAAe,oBACb,KACA,wBAIC;CACD,MAAM,yBAAyB,0BAA0B,OACrD,MAAM,sBAAsB,IAAI,GAC/B,WAAW,uBAAuB,GAAG,yBAAyB,QAAQ,KAAK,uBAAuB;AAEvG,KAAI,0BAA0B,QAAQ,0BAA0B,QAAQ,CAAC,MAAM,eAAe,uBAAuB,CACnH,OAAM,IAAI,MAAM,kDAAkD,yBAAyB;AAG7F,KAAI,0BAA0B,KAC5B,QAAO;EACL,QAAQ;EACR,gBAAgB;EACjB;AAcH,QAAO;EACL,SAZa,MAAM,WAAsB;GACzC,YAAY;GACZ;GACA,QAAQ;GACR,SAAS;GACT,QAAQ;GACR,QAAQ;GACR,aAAa;GACb,QAAQ;GACR,eAAe;GAChB,CAAC,EAEe;EACf,gBAAgB;EACjB;;AAGH,SAAS,wBAAwB,QAA+D;CAC9F,MAAM,aAAa,OAAO,KAAK,OAAO;AACtC,QACE,WAAW,SAAS,KACjB,WAAW,OAAM,QAAO,gBAAgB,IAAI,IAAI,CAAC;;AAIxD,SAAS,mCAAmC,QAA8C;CACxF,MAAM,aAAa,OAAO,KAAK,OAAO;CACtC,MAAM,kBAAkB,WAAW,MAAK,QAAO,gBAAgB,IAAI,IAAI,CAAC;CACxE,MAAM,cAAc,WAAW,MAAK,QAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC;AAErE,KAAI,mBAAmB,YACrB,OAAM,IAAI,UAAU,sCAAsC;;AAI9D,SAAS,0BAA0B,QAA6E;AAC9G,KAAI,UAAU,KACZ;AAGF,oCAAmC,OAAO;AAE1C,KAAI,wBAAwB,OAAO,CACjC,QAAO;AAGT,QAAO,EACL,QAAQ,QACT;;AAGH,SAAS,uBACP,SACA,KACA,iBAC4B;CAC5B,MAAM,UAAU,QAAQ,WAAW;EACjC;EACA;EACA;EACA;EACA;EACA;EACD;CACD,MAAM,UAAU,QAAQ,WAAW;EACjC;EACA;EACA;EACD;CACD,MAAM,SAAS,QAAQ,UAAU,CAAC,GAAG,gBAAgB;CACrD,MAAM,qBAAqB,QAAQ,uBACjC,OAAO,SAAS,IACZ,OAAO,KAAI,WAAU,EAAE,IAAI,MAAM,IAAI,EAAE,GACvC,CAAC,EAAE,IAAI,WAAW,CAAC;CAEzB,MAAM,OAAO,QAAQ,QAAQ,OACzB,MACC,WAAW,QAAQ,KAAK,GAAG,QAAQ,OAAO,QAAQ,KAAK,QAAQ,KAAK;AAEzE,QAAO;EACL;EACA,UAAU,QAAQ;EAClB;EACA,YAAY,0BAA0B,QAAQ,WAAW;EACzD;EACA,MAAM,QAAQ;EACd;EACA,WAAW,0BAA0B,QAAQ,UAAU;EACvD;EACD;;AAGH,SAAS,gBAAgB,QAAsC,KAA2C;CACxG,MAAM,WAAW,QAAQ,YAAY,CAAC,EAAE,MAAM,WAAW,CAAC;CAC1D,MAAM,kBAAkB,QAAQ,UAAU,EAAE;AAC5C,QAAO,SAAS,KAAI,YAAW,uBAAuB,SAAS,KAAK,gBAAgB,CAAC;;;;;;;;;;;;;;;;AAiBvF,eAAsB,oBAAoB,UAAsC,EAAE,EAA4B;CAC5G,MAAM,MAAM,QAAQ,OAAO,QAAQ,KAAK;AACxC,KAAI;EACF,MAAM,eAAe,MAAM,oBAAoB,KAAK,QAAQ,eAAe;AAC3E,MAAI,aAAa,kBAAkB,QAAQ,aAAa,UAAU,KAChE,QAAO;GACL,gBAAgB;GAChB,KAAK,EAAE;GACP,UAAU,gBAAgB,MAAM,IAAI;GACrC;EAGH,MAAM,SAAS,MAAM,mBAAmB,aAAa,OAAO;AAE5D,SAAO;GACL,gBAAgB,aAAa;GAC7B,KAAK,OAAO,OAAO,EAAE;GACrB,UAAU,gBAAgB,QAAQ,QAAQ,aAAa,eAAe,CAAC;GACxE;UAEI,OAAO;EACZ,MAAM,eAAe,iBAAiB,MAAM,IAAI;EAChD,MAAM,iBAAiB,QAAQ,kBAAkB,OAC7C,kBACC,WAAW,QAAQ,eAAe,GAAG,QAAQ,iBAAiB,QAAQ,KAAK,QAAQ,eAAe;AACvG,QAAM,IAAI,MAAM,iCAAiC,eAAe,KAAK,gBAAgB,EAAE,OAAO,OAAO,CAAC;;;;;AC3c1G,MAAM,gDAAgC,IAAI,KAA+B;AACzE,IAAI,mBAAkC;;;;AAKtC,SAAgB,wBAAwB,YAA0B;AAChE,oBAAmB;;;;;AAMrB,SAAgB,wBAA8B;AAC5C,oBAAmB;;;;;AAMrB,SAAgB,uBAAuB,YAAkC;AACvE,KAAI,oBAAoB,KACtB;CAGF,MAAM,WAAW,8BAA8B,IAAI,iBAAiB,IAAI,EAAE;AAC1E,UAAS,KAAK,WAAW;AACzB,+BAA8B,IAAI,kBAAkB,SAAS;;;;;AAM/D,SAAgB,2BAA2B,YAAsC;CAC/E,MAAM,cAAc,8BAA8B,IAAI,WAAW,IAAI,EAAE;AACvE,+BAA8B,OAAO,WAAW;AAChD,QAAO"}