vieval 0.0.10 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +31 -31
  2. package/dist/bin/vieval.mjs +1 -1
  3. package/dist/bin/vieval.mjs.map +1 -1
  4. package/dist/cli/index.d.mts +1 -1
  5. package/dist/cli/index.mjs +1 -1
  6. package/dist/{cli-DTDgaqeI.mjs → cli-uzS81IPd.mjs} +1483 -1483
  7. package/dist/cli-uzS81IPd.mjs.map +1 -0
  8. package/dist/config.d.mts +1 -1
  9. package/dist/config.mjs +1 -1
  10. package/dist/config.mjs.map +1 -1
  11. package/dist/core/assertions/index.d.mts +156 -156
  12. package/dist/core/assertions/index.mjs +82 -82
  13. package/dist/core/assertions/index.mjs.map +1 -1
  14. package/dist/core/inference-executors/index.d.mts +37 -37
  15. package/dist/core/inference-executors/index.mjs +54 -53
  16. package/dist/core/inference-executors/index.mjs.map +1 -1
  17. package/dist/core/processors/results/index.d.mts +18 -18
  18. package/dist/core/processors/results/index.mjs.map +1 -1
  19. package/dist/core/runner/index.d.mts +2 -2
  20. package/dist/core/runner/index.mjs +259 -259
  21. package/dist/core/runner/index.mjs.map +1 -1
  22. package/dist/core/scheduler/index.d.mts +1 -1
  23. package/dist/core/scheduler/index.mjs +65 -65
  24. package/dist/core/scheduler/index.mjs.map +1 -1
  25. package/dist/{env-DfWZy_n4.d.mts → env-Br6jaWGL.d.mts} +9 -9
  26. package/dist/{env-nV5rVErX.mjs → env-egxaJtNn.mjs} +8 -8
  27. package/dist/env-egxaJtNn.mjs.map +1 -0
  28. package/dist/{expect-extensions-DCSqlneN.mjs → expect-extensions-BKdEPt3h.mjs} +46 -46
  29. package/dist/expect-extensions-BKdEPt3h.mjs.map +1 -0
  30. package/dist/expect.d.mts +1 -3
  31. package/dist/expect.mjs +1 -1
  32. package/dist/expect.mjs.map +1 -1
  33. package/dist/{index-D_aMeWqO.d.mts → index-BLIlhiWT.d.mts} +565 -565
  34. package/dist/{index-Bg0atWBF.d.mts → index-CIaJClcC.d.mts} +48 -48
  35. package/dist/index.d.mts +208 -197
  36. package/dist/index.mjs +148 -148
  37. package/dist/index.mjs.map +1 -1
  38. package/dist/{models-pBSRUZhY.mjs → models-CaCOUPZw.mjs} +1 -1
  39. package/dist/{models-pBSRUZhY.mjs.map → models-CaCOUPZw.mjs.map} +1 -1
  40. package/dist/plugins/chat-models/index.d.mts +279 -279
  41. package/dist/plugins/chat-models/index.mjs +360 -360
  42. package/dist/plugins/chat-models/index.mjs.map +1 -1
  43. package/dist/{queue-DsZQkZO_.mjs → queue-BL86z2W_.mjs} +1 -1
  44. package/dist/{queue-DsZQkZO_.mjs.map → queue-BL86z2W_.mjs.map} +1 -1
  45. package/dist/{registry-DMnwE_mY.mjs → registry-BK7k6X81.mjs} +294 -294
  46. package/dist/registry-BK7k6X81.mjs.map +1 -0
  47. package/dist/testing/expect-extensions.d.mts +27 -27
  48. package/dist/testing/expect-extensions.mjs +1 -1
  49. package/package.json +12 -12
  50. package/dist/cli-DTDgaqeI.mjs.map +0 -1
  51. package/dist/env-nV5rVErX.mjs.map +0 -1
  52. package/dist/expect-extensions-DCSqlneN.mjs.map +0 -1
  53. package/dist/registry-DMnwE_mY.mjs.map +0 -1
@@ -7,7 +7,7 @@ import { pathToFileURL } from "node:url";
7
7
  import { createDefineConfig, loadConfig } from "c12";
8
8
  import { loadEnv } from "vite";
9
9
  //#region src/cli/config.ts
10
- const matrixLayerKeys = new Set([
10
+ const matrixLayerKeys = /* @__PURE__ */ new Set([
11
11
  "disable",
12
12
  "extend",
13
13
  "override"
@@ -19,6 +19,23 @@ const require = createRequire(import.meta.url);
19
19
  */
20
20
  const defineConfig = createDefineConfig();
21
21
  /**
22
+ * Detects which top-level config mode is active.
23
+ *
24
+ * Expects:
25
+ * - exactly one of `projects`, `workspaces`, or `comparisons`
26
+ *
27
+ * Returns:
28
+ * - active top-level mode key
29
+ */
30
+ function detectCliConfigMode(config) {
31
+ const declaredModes = [];
32
+ if (config.projects != null) declaredModes.push("projects");
33
+ if (config.workspaces != null) declaredModes.push("workspaces");
34
+ if (config.comparisons != null) declaredModes.push("comparisons");
35
+ if (declaredModes.length > 1) throw new Error(`Invalid vieval config: top-level keys are mutually exclusive. Found ${declaredModes.join(", ")}.`);
36
+ return declaredModes[0] ?? "projects";
37
+ }
38
+ /**
22
39
  * Loads `.env*` files using Vite's env resolution behavior.
23
40
  *
24
41
  * Use when:
@@ -35,6 +52,82 @@ const defineConfig = createDefineConfig();
35
52
  function loadEnv$1(mode, envDir, prefixes = "") {
36
53
  return loadEnv(mode, envDir, prefixes);
37
54
  }
55
+ /**
56
+ * Loads nearest `vieval.config.*` without project normalization.
57
+ */
58
+ async function loadRawVievalConfig(options = {}) {
59
+ const cwd = options.cwd ?? process.cwd();
60
+ try {
61
+ const loadedConfig = await resolveVievalConfig(cwd, options.configFilePath);
62
+ if (loadedConfig.configFilePath == null || loadedConfig.config == null) return {
63
+ config: null,
64
+ configFilePath: null
65
+ };
66
+ const config = await applyVievalPlugins(loadedConfig.config);
67
+ detectCliConfigMode(config);
68
+ return {
69
+ config,
70
+ configFilePath: loadedConfig.configFilePath
71
+ };
72
+ } catch (error) {
73
+ const errorMessage = errorMessageFrom(error) ?? "Unknown config loading error.";
74
+ const configFilePath = options.configFilePath == null ? "vieval.config" : isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath);
75
+ throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
76
+ }
77
+ }
78
+ /**
79
+ * Loads nearest `vieval.config.*` and returns normalized project definitions.
80
+ *
81
+ * Call stack:
82
+ *
83
+ * {@link loadVievalCliConfig}
84
+ * -> {@link resolveVievalConfig}
85
+ * -> {@link normalizeConfig}
86
+ * -> {@link NormalizedCliProjectConfig}[]
87
+ *
88
+ * Use when:
89
+ * - CLI orchestration needs project includes/excludes similar to Vitest
90
+ * - callers want config auto-discovery without manual imports in eval files
91
+ */
92
+ async function loadVievalCliConfig(options = {}) {
93
+ const cwd = options.cwd ?? process.cwd();
94
+ try {
95
+ const loadedConfig = await loadRawVievalConfig(options);
96
+ if (loadedConfig.configFilePath == null || loadedConfig.config == null) return {
97
+ concurrency: void 0,
98
+ configFilePath: null,
99
+ env: {},
100
+ projects: await normalizeConfig(null, cwd),
101
+ reporting: void 0
102
+ };
103
+ const config = loadedConfig.config;
104
+ return {
105
+ concurrency: config.concurrency,
106
+ configFilePath: loadedConfig.configFilePath,
107
+ env: config.env ?? {},
108
+ projects: await normalizeConfig(config, dirname(loadedConfig.configFilePath)),
109
+ reporting: normalizeReportingConfig(config.reporting)
110
+ };
111
+ } catch (error) {
112
+ const errorMessage = errorMessageFrom(error) ?? "Unknown config loading error.";
113
+ const configFilePath = options.configFilePath == null ? "vieval.config" : isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath);
114
+ throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
115
+ }
116
+ }
117
+ async function applyProjectPlugins(config, project, normalizedProject, cwd) {
118
+ if (project.plugins == null || project.plugins.length === 0) return normalizedProject;
119
+ const scopedConfig = createProjectScopedConfig(config, project, normalizedProject);
120
+ const resolvedConfig = await applyVievalPlugins(scopedConfig);
121
+ const scopedProject = scopedConfig.projects?.[0];
122
+ if (scopedProject == null) throw new Error("Project-local plugin normalization requires one scoped project.");
123
+ const resolvedProject = resolvedConfig.projects?.[0] ?? scopedProject;
124
+ return normalizeProjectConfig({
125
+ ...resolvedProject,
126
+ concurrency: resolvedProject.concurrency === scopedProject.concurrency ? toProjectConcurrencyDefaults(resolvedConfig.concurrency) : resolvedProject.concurrency,
127
+ models: resolvedProject.models === scopedProject.models ? resolvedConfig.models : resolvedProject.models,
128
+ reporters: resolvedProject.reporters === scopedProject.reporters ? resolvedConfig.reporters : resolvedProject.reporters
129
+ }, cwd, void 0, resolvedConfig.models ?? [], resolvedConfig.reporters ?? []);
130
+ }
38
131
  async function applyVievalPlugins(config) {
39
132
  let currentConfig = config;
40
133
  const plugins = currentConfig.plugins ?? [];
@@ -49,34 +142,38 @@ async function applyVievalPlugins(config) {
49
142
  for (const plugin of plugins) await plugin.configVievalResolved?.(currentConfig);
50
143
  return currentConfig;
51
144
  }
52
- async function isReadableFile(filePath) {
53
- try {
54
- await access(filePath);
55
- return true;
56
- } catch {
57
- return false;
58
- }
59
- }
60
- function isConfigFileExtensionUsingRequire(extension) {
61
- return extension === ".cjs" || extension === ".cts";
62
- }
63
- function isConfigFileExtensionUsingJsonParse(extension) {
64
- return extension === ".json";
65
- }
66
- async function importVievalConfigModule(filePath) {
67
- const extension = extname(filePath);
68
- if (isConfigFileExtensionUsingJsonParse(extension)) {
69
- const raw = await readFile(filePath, "utf-8");
70
- return JSON.parse(raw);
71
- }
72
- if (isConfigFileExtensionUsingRequire(extension)) return require(filePath);
73
- return import(pathToFileURL(filePath).href);
145
+ function assertNonAmbiguousMatrixDefinition(matrix) {
146
+ const matrixKeys = Object.keys(matrix);
147
+ const hasReservedKeys = matrixKeys.some((key) => matrixLayerKeys.has(key));
148
+ const hasAxisKeys = matrixKeys.some((key) => !matrixLayerKeys.has(key));
149
+ if (hasReservedKeys && hasAxisKeys) throw new TypeError(ambiguousMatrixDefinitionErrorMessage);
74
150
  }
75
- function resolveConfigExport(moduleValue) {
76
- if (moduleValue == null) return null;
77
- if (typeof moduleValue !== "object") return moduleValue;
78
- if ("default" in moduleValue) return moduleValue.default;
79
- return moduleValue;
151
+ function createProjectScopedConfig(config, project, normalizedProject) {
152
+ return {
153
+ concurrency: config?.concurrency == null && normalizedProject.concurrency == null ? void 0 : {
154
+ ...normalizedProject.concurrency,
155
+ workspace: config?.concurrency?.workspace
156
+ },
157
+ env: config?.env,
158
+ models: normalizedProject.models,
159
+ plugins: project.plugins,
160
+ projects: [{
161
+ concurrency: normalizedProject.concurrency,
162
+ evalMatrix: normalizedProject.evalMatrix,
163
+ exclude: normalizedProject.exclude,
164
+ executor: normalizedProject.executor,
165
+ include: normalizedProject.include,
166
+ inferenceExecutors: normalizedProject.inferenceExecutors,
167
+ models: normalizedProject.models,
168
+ name: normalizedProject.name,
169
+ plugins: project.plugins,
170
+ reporters: normalizedProject.reporters,
171
+ root: normalizedProject.root,
172
+ runMatrix: normalizedProject.runMatrix
173
+ }],
174
+ reporters: normalizedProject.reporters,
175
+ reporting: config?.reporting
176
+ };
80
177
  }
81
178
  async function findNearestConfigFile(startDirectory) {
82
179
  const supportedFileNames = [
@@ -99,52 +196,32 @@ async function findNearestConfigFile(startDirectory) {
99
196
  currentDirectory = parentDirectory;
100
197
  }
101
198
  }
102
- async function resolveVievalConfig(cwd, explicitConfigFilePath) {
103
- const resolvedConfigFilePath = explicitConfigFilePath == null ? await findNearestConfigFile(cwd) : isAbsolute(explicitConfigFilePath) ? explicitConfigFilePath : resolve(cwd, explicitConfigFilePath);
104
- if (explicitConfigFilePath != null && resolvedConfigFilePath != null && !await isReadableFile(resolvedConfigFilePath)) throw new Error(`Config file does not exist or is not readable: ${resolvedConfigFilePath}`);
105
- if (resolvedConfigFilePath == null) return {
106
- config: null,
107
- configFilePath: null
108
- };
109
- return {
110
- config: (await loadConfig({
111
- configFile: resolvedConfigFilePath,
112
- cwd,
113
- dotenv: false,
114
- envName: false,
115
- extend: false,
116
- import: importVievalConfigModule,
117
- packageJson: false,
118
- rcFile: false,
119
- resolveModule: resolveConfigExport
120
- })).config,
121
- configFilePath: resolvedConfigFilePath
122
- };
199
+ async function importVievalConfigModule(filePath) {
200
+ const extension = extname(filePath);
201
+ if (isConfigFileExtensionUsingJsonParse(extension)) {
202
+ const raw = await readFile(filePath, "utf-8");
203
+ return JSON.parse(raw);
204
+ }
205
+ if (isConfigFileExtensionUsingRequire(extension)) return require(filePath);
206
+ return import(pathToFileURL(filePath).href);
207
+ }
208
+ function isConfigFileExtensionUsingJsonParse(extension) {
209
+ return extension === ".json";
210
+ }
211
+ function isConfigFileExtensionUsingRequire(extension) {
212
+ return extension === ".cjs" || extension === ".cts";
123
213
  }
124
214
  function isLayerMatrixDefinition(matrix) {
125
215
  const matrixKeys = Object.keys(matrix);
126
216
  return matrixKeys.length > 0 && matrixKeys.every((key) => matrixLayerKeys.has(key));
127
217
  }
128
- function assertNonAmbiguousMatrixDefinition(matrix) {
129
- const matrixKeys = Object.keys(matrix);
130
- const hasReservedKeys = matrixKeys.some((key) => matrixLayerKeys.has(key));
131
- const hasAxisKeys = matrixKeys.some((key) => !matrixLayerKeys.has(key));
132
- if (hasReservedKeys && hasAxisKeys) throw new TypeError(ambiguousMatrixDefinitionErrorMessage);
133
- }
134
- function normalizeMatrixLayerInput(matrix) {
135
- if (matrix == null) return;
136
- assertNonAmbiguousMatrixDefinition(matrix);
137
- if (isLayerMatrixDefinition(matrix)) return matrix;
138
- return { extend: matrix };
139
- }
140
- function toProjectConcurrencyDefaults(concurrency) {
141
- if (concurrency == null) return;
142
- return {
143
- attempt: concurrency.attempt,
144
- case: concurrency.case,
145
- project: concurrency.project,
146
- task: concurrency.task
147
- };
218
+ async function isReadableFile(filePath) {
219
+ try {
220
+ await access(filePath);
221
+ return true;
222
+ } catch {
223
+ return false;
224
+ }
148
225
  }
149
226
  function mergeProjectConcurrency(inheritedConcurrency, projectConcurrency) {
150
227
  if (inheritedConcurrency == null && projectConcurrency == null) return;
@@ -155,6 +232,27 @@ function mergeProjectConcurrency(inheritedConcurrency, projectConcurrency) {
155
232
  task: projectConcurrency?.task ?? inheritedConcurrency?.task
156
233
  };
157
234
  }
235
+ async function normalizeConfig(config, cwd) {
236
+ if (config != null) {
237
+ if (detectCliConfigMode(config) === "comparisons") throw new Error("vieval run requires project-mode config. Received comparison-mode config.");
238
+ }
239
+ const projects = config?.workspaces == null ? config?.projects ?? [{ name: "default" }] : config.workspaces.map((workspace) => ({
240
+ name: workspace.id,
241
+ root: workspace.root
242
+ }));
243
+ const inheritedConcurrency = toProjectConcurrencyDefaults(config?.concurrency);
244
+ const inheritedModels = config?.models ?? [];
245
+ const inheritedReporterReferences = config?.reporters ?? [];
246
+ return Promise.all(projects.map(async (project) => {
247
+ return applyProjectPlugins(config, project, normalizeProjectConfig(project, cwd, inheritedConcurrency, inheritedModels, inheritedReporterReferences), cwd);
248
+ }));
249
+ }
250
+ function normalizeMatrixLayerInput(matrix) {
251
+ if (matrix == null) return;
252
+ assertNonAmbiguousMatrixDefinition(matrix);
253
+ if (isLayerMatrixDefinition(matrix)) return matrix;
254
+ return { extend: matrix };
255
+ }
158
256
  function normalizeProjectConfig(project, cwd, inheritedConcurrency, inheritedModels, inheritedReporterReferences) {
159
257
  const include = project.include ?? [
160
258
  "**/*.eval.ts",
@@ -175,159 +273,61 @@ function normalizeProjectConfig(project, cwd, inheritedConcurrency, inheritedMod
175
273
  const reporters = project.reporters ?? [...inheritedReporterReferences];
176
274
  return {
177
275
  concurrency: mergeProjectConcurrency(inheritedConcurrency, project.concurrency),
276
+ evalMatrix: normalizeMatrixLayerInput(project.evalMatrix),
178
277
  exclude,
179
278
  executor: project.executor,
180
- include,
181
- evalMatrix: normalizeMatrixLayerInput(project.evalMatrix),
182
- models,
183
- name: project.name,
184
- inferenceExecutors,
185
- reporters,
186
- runMatrix: normalizeMatrixLayerInput(project.runMatrix),
187
- root
188
- };
189
- }
190
- function createProjectScopedConfig(config, project, normalizedProject) {
191
- return {
192
- concurrency: config?.concurrency == null && normalizedProject.concurrency == null ? void 0 : {
193
- ...normalizedProject.concurrency,
194
- workspace: config?.concurrency?.workspace
195
- },
196
- env: config?.env,
197
- models: normalizedProject.models,
198
- plugins: project.plugins,
199
- projects: [{
200
- concurrency: normalizedProject.concurrency,
201
- exclude: normalizedProject.exclude,
202
- executor: normalizedProject.executor,
203
- include: normalizedProject.include,
204
- evalMatrix: normalizedProject.evalMatrix,
205
- inferenceExecutors: normalizedProject.inferenceExecutors,
206
- models: normalizedProject.models,
207
- name: normalizedProject.name,
208
- plugins: project.plugins,
209
- reporters: normalizedProject.reporters,
210
- root: normalizedProject.root,
211
- runMatrix: normalizedProject.runMatrix
212
- }],
213
- reporters: normalizedProject.reporters,
214
- reporting: config?.reporting
215
- };
216
- }
217
- async function applyProjectPlugins(config, project, normalizedProject, cwd) {
218
- if (project.plugins == null || project.plugins.length === 0) return normalizedProject;
219
- const scopedConfig = createProjectScopedConfig(config, project, normalizedProject);
220
- const resolvedConfig = await applyVievalPlugins(scopedConfig);
221
- const scopedProject = scopedConfig.projects?.[0];
222
- if (scopedProject == null) throw new Error("Project-local plugin normalization requires one scoped project.");
223
- const resolvedProject = resolvedConfig.projects?.[0] ?? scopedProject;
224
- return normalizeProjectConfig({
225
- ...resolvedProject,
226
- concurrency: resolvedProject.concurrency === scopedProject.concurrency ? toProjectConcurrencyDefaults(resolvedConfig.concurrency) : resolvedProject.concurrency,
227
- models: resolvedProject.models === scopedProject.models ? resolvedConfig.models : resolvedProject.models,
228
- reporters: resolvedProject.reporters === scopedProject.reporters ? resolvedConfig.reporters : resolvedProject.reporters
229
- }, cwd, void 0, resolvedConfig.models ?? [], resolvedConfig.reporters ?? []);
230
- }
231
- async function normalizeConfig(config, cwd) {
232
- if (config != null) {
233
- if (detectCliConfigMode(config) === "comparisons") throw new Error("vieval run requires project-mode config. Received comparison-mode config.");
234
- }
235
- const projects = config?.workspaces == null ? config?.projects ?? [{ name: "default" }] : config.workspaces.map((workspace) => ({
236
- name: workspace.id,
237
- root: workspace.root
238
- }));
239
- const inheritedConcurrency = toProjectConcurrencyDefaults(config?.concurrency);
240
- const inheritedModels = config?.models ?? [];
241
- const inheritedReporterReferences = config?.reporters ?? [];
242
- return Promise.all(projects.map(async (project) => {
243
- return applyProjectPlugins(config, project, normalizeProjectConfig(project, cwd, inheritedConcurrency, inheritedModels, inheritedReporterReferences), cwd);
244
- }));
245
- }
246
- function normalizeReportingConfig(config) {
247
- if (config == null) return;
248
- return { openTelemetry: config.openTelemetry == null ? void 0 : {
249
- enabled: config.openTelemetry.enabled ?? false,
250
- onRunEnd: config.openTelemetry.onRunEnd
251
- } };
252
- }
253
- /**
254
- * Detects which top-level config mode is active.
255
- *
256
- * Expects:
257
- * - exactly one of `projects`, `workspaces`, or `comparisons`
258
- *
259
- * Returns:
260
- * - active top-level mode key
261
- */
262
- function detectCliConfigMode(config) {
263
- const declaredModes = [];
264
- if (config.projects != null) declaredModes.push("projects");
265
- if (config.workspaces != null) declaredModes.push("workspaces");
266
- if (config.comparisons != null) declaredModes.push("comparisons");
267
- if (declaredModes.length > 1) throw new Error(`Invalid vieval config: top-level keys are mutually exclusive. Found ${declaredModes.join(", ")}.`);
268
- return declaredModes[0] ?? "projects";
279
+ include,
280
+ inferenceExecutors,
281
+ models,
282
+ name: project.name,
283
+ reporters,
284
+ root,
285
+ runMatrix: normalizeMatrixLayerInput(project.runMatrix)
286
+ };
269
287
  }
270
- /**
271
- * Loads nearest `vieval.config.*` without project normalization.
272
- */
273
- async function loadRawVievalConfig(options = {}) {
274
- const cwd = options.cwd ?? process.cwd();
275
- try {
276
- const loadedConfig = await resolveVievalConfig(cwd, options.configFilePath);
277
- if (loadedConfig.configFilePath == null || loadedConfig.config == null) return {
278
- config: null,
279
- configFilePath: null
280
- };
281
- const config = await applyVievalPlugins(loadedConfig.config);
282
- detectCliConfigMode(config);
283
- return {
284
- config,
285
- configFilePath: loadedConfig.configFilePath
286
- };
287
- } catch (error) {
288
- const errorMessage = errorMessageFrom(error) ?? "Unknown config loading error.";
289
- const configFilePath = options.configFilePath == null ? "vieval.config" : isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath);
290
- throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
291
- }
288
+ function normalizeReportingConfig(config) {
289
+ if (config == null) return;
290
+ return { openTelemetry: config.openTelemetry == null ? void 0 : {
291
+ enabled: config.openTelemetry.enabled ?? false,
292
+ onRunEnd: config.openTelemetry.onRunEnd
293
+ } };
292
294
  }
293
- /**
294
- * Loads nearest `vieval.config.*` and returns normalized project definitions.
295
- *
296
- * Call stack:
297
- *
298
- * {@link loadVievalCliConfig}
299
- * -> {@link resolveVievalConfig}
300
- * -> {@link normalizeConfig}
301
- * -> {@link NormalizedCliProjectConfig}[]
302
- *
303
- * Use when:
304
- * - CLI orchestration needs project includes/excludes similar to Vitest
305
- * - callers want config auto-discovery without manual imports in eval files
306
- */
307
- async function loadVievalCliConfig(options = {}) {
308
- const cwd = options.cwd ?? process.cwd();
309
- try {
310
- const loadedConfig = await loadRawVievalConfig(options);
311
- if (loadedConfig.configFilePath == null || loadedConfig.config == null) return {
312
- concurrency: void 0,
313
- configFilePath: null,
314
- env: {},
315
- projects: await normalizeConfig(null, cwd),
316
- reporting: void 0
317
- };
318
- const config = loadedConfig.config;
319
- return {
320
- concurrency: config.concurrency,
321
- configFilePath: loadedConfig.configFilePath,
322
- env: config.env ?? {},
323
- projects: await normalizeConfig(config, dirname(loadedConfig.configFilePath)),
324
- reporting: normalizeReportingConfig(config.reporting)
325
- };
326
- } catch (error) {
327
- const errorMessage = errorMessageFrom(error) ?? "Unknown config loading error.";
328
- const configFilePath = options.configFilePath == null ? "vieval.config" : isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath);
329
- throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
330
- }
295
+ function resolveConfigExport(moduleValue) {
296
+ if (moduleValue == null) return null;
297
+ if (typeof moduleValue !== "object") return moduleValue;
298
+ if ("default" in moduleValue) return moduleValue.default;
299
+ return moduleValue;
300
+ }
301
+ async function resolveVievalConfig(cwd, explicitConfigFilePath) {
302
+ const resolvedConfigFilePath = explicitConfigFilePath == null ? await findNearestConfigFile(cwd) : isAbsolute(explicitConfigFilePath) ? explicitConfigFilePath : resolve(cwd, explicitConfigFilePath);
303
+ if (explicitConfigFilePath != null && resolvedConfigFilePath != null && !await isReadableFile(resolvedConfigFilePath)) throw new Error(`Config file does not exist or is not readable: ${resolvedConfigFilePath}`);
304
+ if (resolvedConfigFilePath == null) return {
305
+ config: null,
306
+ configFilePath: null
307
+ };
308
+ return {
309
+ config: (await loadConfig({
310
+ configFile: resolvedConfigFilePath,
311
+ cwd,
312
+ dotenv: false,
313
+ envName: false,
314
+ extend: false,
315
+ import: importVievalConfigModule,
316
+ packageJson: false,
317
+ rcFile: false,
318
+ resolveModule: resolveConfigExport
319
+ })).config,
320
+ configFilePath: resolvedConfigFilePath
321
+ };
322
+ }
323
+ function toProjectConcurrencyDefaults(concurrency) {
324
+ if (concurrency == null) return;
325
+ return {
326
+ attempt: concurrency.attempt,
327
+ case: concurrency.case,
328
+ project: concurrency.project,
329
+ task: concurrency.task
330
+ };
331
331
  }
332
332
  //#endregion
333
333
  //#region src/core/telemetry/noop.ts
@@ -346,62 +346,16 @@ async function loadVievalCliConfig(options = {}) {
346
346
  */
347
347
  function createNoopTelemetryRuntime() {
348
348
  return {
349
- async withSpan(_name, _attributes, callback) {
350
- return await callback();
351
- },
352
349
  addEvent() {},
350
+ recordException() {},
353
351
  setAttributes() {},
354
- recordException() {}
352
+ async withSpan(_name, _attributes, callback) {
353
+ return await callback();
354
+ }
355
355
  };
356
356
  }
357
357
  //#endregion
358
358
  //#region src/core/telemetry/otel.ts
359
- async function importOpenTelemetryApi() {
360
- return await import("@opentelemetry/api");
361
- }
362
- function isOpenTelemetryAttributeScalar(value) {
363
- return typeof value === "boolean" || typeof value === "number" || typeof value === "string";
364
- }
365
- function isHomogeneousOpenTelemetryAttributeArray(value) {
366
- if (value.length === 0) return true;
367
- const firstType = typeof value[0];
368
- if (firstType !== "boolean" && firstType !== "number" && firstType !== "string") return false;
369
- return value.every((item) => typeof item === firstType);
370
- }
371
- function stringifyAttributeValue(value) {
372
- try {
373
- return JSON.stringify(value);
374
- } catch {
375
- return String(value);
376
- }
377
- }
378
- /**
379
- * Normalizes JSON-compatible telemetry attributes into OpenTelemetry-safe attributes.
380
- *
381
- * Before:
382
- * - `{ nil: null, nested: ['a', [1, null]], scalarArray: ['a', 1, true] }`
383
- *
384
- * After:
385
- * - `{ nested: '["a",[1,null]]', scalarArray: ['a', 1, true] }`
386
- */
387
- function normalizeOpenTelemetryAttributes(attributes) {
388
- if (attributes == null) return;
389
- const normalized = {};
390
- for (const [key, value] of Object.entries(attributes)) {
391
- if (value == null) continue;
392
- if (isOpenTelemetryAttributeScalar(value)) {
393
- normalized[key] = value;
394
- continue;
395
- }
396
- if (Array.isArray(value)) {
397
- normalized[key] = isHomogeneousOpenTelemetryAttributeArray(value) ? value : stringifyAttributeValue(value) ?? "";
398
- continue;
399
- }
400
- const stringified = stringifyAttributeValue(value);
401
- if (stringified != null) normalized[key] = stringified;
402
- }
403
- return normalized;
404
- }
405
359
  /**
406
360
  * Creates an OpenTelemetry-backed runtime using active spans.
407
361
  *
@@ -428,6 +382,15 @@ function createOpenTelemetryRuntime(options = {}) {
428
382
  return await apiPromise;
429
383
  }
430
384
  return {
385
+ addEvent(name, attributes) {
386
+ loadedApi?.trace.getActiveSpan()?.addEvent(name, normalizeOpenTelemetryAttributes(attributes));
387
+ },
388
+ recordException(error) {
389
+ loadedApi?.trace.getActiveSpan()?.recordException(error);
390
+ },
391
+ setAttributes(attributes) {
392
+ loadedApi?.trace.getActiveSpan()?.setAttributes(normalizeOpenTelemetryAttributes(attributes) ?? {});
393
+ },
431
394
  async withSpan(name, attributes, callback) {
432
395
  const api = await getApi();
433
396
  return await api.trace.getTracer("vieval").startActiveSpan(name, { attributes: normalizeOpenTelemetryAttributes(attributes) ?? {} }, async (span) => {
@@ -444,29 +407,58 @@ function createOpenTelemetryRuntime(options = {}) {
444
407
  span.end();
445
408
  }
446
409
  });
447
- },
448
- addEvent(name, attributes) {
449
- loadedApi?.trace.getActiveSpan()?.addEvent(name, normalizeOpenTelemetryAttributes(attributes));
450
- },
451
- setAttributes(attributes) {
452
- loadedApi?.trace.getActiveSpan()?.setAttributes(normalizeOpenTelemetryAttributes(attributes) ?? {});
453
- },
454
- recordException(error) {
455
- loadedApi?.trace.getActiveSpan()?.recordException(error);
456
410
  }
457
411
  };
458
412
  }
413
+ async function importOpenTelemetryApi() {
414
+ return await import("@opentelemetry/api");
415
+ }
416
+ function isHomogeneousOpenTelemetryAttributeArray(value) {
417
+ if (value.length === 0) return true;
418
+ const firstType = typeof value[0];
419
+ if (firstType !== "boolean" && firstType !== "number" && firstType !== "string") return false;
420
+ return value.every((item) => typeof item === firstType);
421
+ }
422
+ function isOpenTelemetryAttributeScalar(value) {
423
+ return typeof value === "boolean" || typeof value === "number" || typeof value === "string";
424
+ }
425
+ /**
426
+ * Normalizes JSON-compatible telemetry attributes into OpenTelemetry-safe attributes.
427
+ *
428
+ * Before:
429
+ * - `{ nil: null, nested: ['a', [1, null]], scalarArray: ['a', 1, true] }`
430
+ *
431
+ * After:
432
+ * - `{ nested: '["a",[1,null]]', scalarArray: ['a', 1, true] }`
433
+ */
434
+ function normalizeOpenTelemetryAttributes(attributes) {
435
+ if (attributes == null) return;
436
+ const normalized = {};
437
+ for (const [key, value] of Object.entries(attributes)) {
438
+ if (value == null) continue;
439
+ if (isOpenTelemetryAttributeScalar(value)) {
440
+ normalized[key] = value;
441
+ continue;
442
+ }
443
+ if (Array.isArray(value)) {
444
+ normalized[key] = isHomogeneousOpenTelemetryAttributeArray(value) ? value : stringifyAttributeValue(value) ?? "";
445
+ continue;
446
+ }
447
+ const stringified = stringifyAttributeValue(value);
448
+ if (stringified != null) normalized[key] = stringified;
449
+ }
450
+ return normalized;
451
+ }
452
+ function stringifyAttributeValue(value) {
453
+ try {
454
+ return JSON.stringify(value);
455
+ } catch {
456
+ return String(value);
457
+ }
458
+ }
459
459
  //#endregion
460
460
  //#region src/dsl/registry.ts
461
461
  const registryStoreSymbol = Symbol.for("vieval.dsl.registry.store");
462
- function getRegistryStore() {
463
- const processWithStore = process;
464
- processWithStore[registryStoreSymbol] ??= {
465
- activeModuleHref: null,
466
- registeredDefinitionsByModule: /* @__PURE__ */ new Map()
467
- };
468
- return processWithStore[registryStoreSymbol];
469
- }
470
462
  /**
471
463
  * Starts module-scoped eval registration collection.
472
464
  */
@@ -475,6 +467,15 @@ function beginModuleRegistration(moduleHref) {
475
467
  store.activeModuleHref = moduleHref;
476
468
  }
477
469
  /**
470
+ * Consumes registered definitions for one module and clears stored state.
471
+ */
472
+ function consumeModuleRegistrations(moduleHref) {
473
+ const store = getRegistryStore();
474
+ const definitions = store.registeredDefinitionsByModule.get(moduleHref) ?? [];
475
+ store.registeredDefinitionsByModule.delete(moduleHref);
476
+ return definitions;
477
+ }
478
+ /**
478
479
  * Ends module-scoped eval registration collection.
479
480
  */
480
481
  function endModuleRegistration() {
@@ -491,16 +492,15 @@ function registerEvalDefinition(definition) {
491
492
  existing.push(definition);
492
493
  store.registeredDefinitionsByModule.set(store.activeModuleHref, existing);
493
494
  }
494
- /**
495
- * Consumes registered definitions for one module and clears stored state.
496
- */
497
- function consumeModuleRegistrations(moduleHref) {
498
- const store = getRegistryStore();
499
- const definitions = store.registeredDefinitionsByModule.get(moduleHref) ?? [];
500
- store.registeredDefinitionsByModule.delete(moduleHref);
501
- return definitions;
495
+ function getRegistryStore() {
496
+ const processWithStore = process;
497
+ processWithStore[registryStoreSymbol] ??= {
498
+ activeModuleHref: null,
499
+ registeredDefinitionsByModule: /* @__PURE__ */ new Map()
500
+ };
501
+ return processWithStore[registryStoreSymbol];
502
502
  }
503
503
  //#endregion
504
504
  export { createOpenTelemetryRuntime as a, detectCliConfigMode as c, loadVievalCliConfig as d, registerEvalDefinition as i, loadEnv$1 as l, consumeModuleRegistrations as n, createNoopTelemetryRuntime as o, endModuleRegistration as r, defineConfig as s, beginModuleRegistration as t, loadRawVievalConfig as u };
505
505
 
506
- //# sourceMappingURL=registry-DMnwE_mY.mjs.map
506
+ //# sourceMappingURL=registry-BK7k6X81.mjs.map