vieval 0.0.11 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -31
- package/dist/bin/vieval.mjs +1 -1
- package/dist/cli/index.d.mts +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/{cli-CHFCF8UR.mjs → cli-uzS81IPd.mjs} +1529 -1529
- package/dist/cli-uzS81IPd.mjs.map +1 -0
- package/dist/config.d.mts +1 -1
- package/dist/core/assertions/index.d.mts +156 -156
- package/dist/core/assertions/index.mjs +82 -82
- package/dist/core/assertions/index.mjs.map +1 -1
- package/dist/core/inference-executors/index.d.mts +37 -37
- package/dist/core/inference-executors/index.mjs +53 -52
- package/dist/core/inference-executors/index.mjs.map +1 -1
- package/dist/core/processors/results/index.d.mts +18 -18
- package/dist/core/processors/results/index.mjs.map +1 -1
- package/dist/core/runner/index.d.mts +2 -2
- package/dist/core/runner/index.mjs +258 -258
- package/dist/core/runner/index.mjs.map +1 -1
- package/dist/core/scheduler/index.d.mts +1 -1
- package/dist/core/scheduler/index.mjs +64 -64
- package/dist/core/scheduler/index.mjs.map +1 -1
- package/dist/{env-bRH0K6fU.d.mts → env-Br6jaWGL.d.mts} +9 -9
- package/dist/{env-BVYeJhGA.mjs → env-egxaJtNn.mjs} +8 -8
- package/dist/env-egxaJtNn.mjs.map +1 -0
- package/dist/{expect-extensions-Mf1sMNBv.mjs → expect-extensions-BKdEPt3h.mjs} +46 -46
- package/dist/expect-extensions-BKdEPt3h.mjs.map +1 -0
- package/dist/expect.mjs +1 -1
- package/dist/{index-CwKBlCG9.d.mts → index-BLIlhiWT.d.mts} +565 -565
- package/dist/{index-Be5I1ZJL.d.mts → index-CIaJClcC.d.mts} +48 -48
- package/dist/index.d.mts +207 -195
- package/dist/index.mjs +147 -147
- package/dist/index.mjs.map +1 -1
- package/dist/models-CaCOUPZw.mjs.map +1 -1
- package/dist/plugins/chat-models/index.d.mts +279 -279
- package/dist/plugins/chat-models/index.mjs +359 -359
- package/dist/plugins/chat-models/index.mjs.map +1 -1
- package/dist/{registry-BSyjwZFx.mjs → registry-BK7k6X81.mjs} +293 -293
- package/dist/registry-BK7k6X81.mjs.map +1 -0
- package/dist/testing/expect-extensions.d.mts +27 -27
- package/dist/testing/expect-extensions.mjs +1 -1
- package/package.json +3 -3
- package/dist/cli-CHFCF8UR.mjs.map +0 -1
- package/dist/env-BVYeJhGA.mjs.map +0 -1
- package/dist/expect-extensions-Mf1sMNBv.mjs.map +0 -1
- package/dist/registry-BSyjwZFx.mjs.map +0 -1
|
@@ -19,6 +19,23 @@ const require = createRequire(import.meta.url);
|
|
|
19
19
|
*/
|
|
20
20
|
const defineConfig = createDefineConfig();
|
|
21
21
|
/**
|
|
22
|
+
* Detects which top-level config mode is active.
|
|
23
|
+
*
|
|
24
|
+
* Expects:
|
|
25
|
+
* - exactly one of `projects`, `workspaces`, or `comparisons`
|
|
26
|
+
*
|
|
27
|
+
* Returns:
|
|
28
|
+
* - active top-level mode key
|
|
29
|
+
*/
|
|
30
|
+
function detectCliConfigMode(config) {
|
|
31
|
+
const declaredModes = [];
|
|
32
|
+
if (config.projects != null) declaredModes.push("projects");
|
|
33
|
+
if (config.workspaces != null) declaredModes.push("workspaces");
|
|
34
|
+
if (config.comparisons != null) declaredModes.push("comparisons");
|
|
35
|
+
if (declaredModes.length > 1) throw new Error(`Invalid vieval config: top-level keys are mutually exclusive. Found ${declaredModes.join(", ")}.`);
|
|
36
|
+
return declaredModes[0] ?? "projects";
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
22
39
|
* Loads `.env*` files using Vite's env resolution behavior.
|
|
23
40
|
*
|
|
24
41
|
* Use when:
|
|
@@ -35,6 +52,82 @@ const defineConfig = createDefineConfig();
|
|
|
35
52
|
function loadEnv$1(mode, envDir, prefixes = "") {
|
|
36
53
|
return loadEnv(mode, envDir, prefixes);
|
|
37
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* Loads nearest `vieval.config.*` without project normalization.
|
|
57
|
+
*/
|
|
58
|
+
async function loadRawVievalConfig(options = {}) {
|
|
59
|
+
const cwd = options.cwd ?? process.cwd();
|
|
60
|
+
try {
|
|
61
|
+
const loadedConfig = await resolveVievalConfig(cwd, options.configFilePath);
|
|
62
|
+
if (loadedConfig.configFilePath == null || loadedConfig.config == null) return {
|
|
63
|
+
config: null,
|
|
64
|
+
configFilePath: null
|
|
65
|
+
};
|
|
66
|
+
const config = await applyVievalPlugins(loadedConfig.config);
|
|
67
|
+
detectCliConfigMode(config);
|
|
68
|
+
return {
|
|
69
|
+
config,
|
|
70
|
+
configFilePath: loadedConfig.configFilePath
|
|
71
|
+
};
|
|
72
|
+
} catch (error) {
|
|
73
|
+
const errorMessage = errorMessageFrom(error) ?? "Unknown config loading error.";
|
|
74
|
+
const configFilePath = options.configFilePath == null ? "vieval.config" : isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath);
|
|
75
|
+
throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Loads nearest `vieval.config.*` and returns normalized project definitions.
|
|
80
|
+
*
|
|
81
|
+
* Call stack:
|
|
82
|
+
*
|
|
83
|
+
* {@link loadVievalCliConfig}
|
|
84
|
+
* -> {@link resolveVievalConfig}
|
|
85
|
+
* -> {@link normalizeConfig}
|
|
86
|
+
* -> {@link NormalizedCliProjectConfig}[]
|
|
87
|
+
*
|
|
88
|
+
* Use when:
|
|
89
|
+
* - CLI orchestration needs project includes/excludes similar to Vitest
|
|
90
|
+
* - callers want config auto-discovery without manual imports in eval files
|
|
91
|
+
*/
|
|
92
|
+
async function loadVievalCliConfig(options = {}) {
|
|
93
|
+
const cwd = options.cwd ?? process.cwd();
|
|
94
|
+
try {
|
|
95
|
+
const loadedConfig = await loadRawVievalConfig(options);
|
|
96
|
+
if (loadedConfig.configFilePath == null || loadedConfig.config == null) return {
|
|
97
|
+
concurrency: void 0,
|
|
98
|
+
configFilePath: null,
|
|
99
|
+
env: {},
|
|
100
|
+
projects: await normalizeConfig(null, cwd),
|
|
101
|
+
reporting: void 0
|
|
102
|
+
};
|
|
103
|
+
const config = loadedConfig.config;
|
|
104
|
+
return {
|
|
105
|
+
concurrency: config.concurrency,
|
|
106
|
+
configFilePath: loadedConfig.configFilePath,
|
|
107
|
+
env: config.env ?? {},
|
|
108
|
+
projects: await normalizeConfig(config, dirname(loadedConfig.configFilePath)),
|
|
109
|
+
reporting: normalizeReportingConfig(config.reporting)
|
|
110
|
+
};
|
|
111
|
+
} catch (error) {
|
|
112
|
+
const errorMessage = errorMessageFrom(error) ?? "Unknown config loading error.";
|
|
113
|
+
const configFilePath = options.configFilePath == null ? "vieval.config" : isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath);
|
|
114
|
+
throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
async function applyProjectPlugins(config, project, normalizedProject, cwd) {
|
|
118
|
+
if (project.plugins == null || project.plugins.length === 0) return normalizedProject;
|
|
119
|
+
const scopedConfig = createProjectScopedConfig(config, project, normalizedProject);
|
|
120
|
+
const resolvedConfig = await applyVievalPlugins(scopedConfig);
|
|
121
|
+
const scopedProject = scopedConfig.projects?.[0];
|
|
122
|
+
if (scopedProject == null) throw new Error("Project-local plugin normalization requires one scoped project.");
|
|
123
|
+
const resolvedProject = resolvedConfig.projects?.[0] ?? scopedProject;
|
|
124
|
+
return normalizeProjectConfig({
|
|
125
|
+
...resolvedProject,
|
|
126
|
+
concurrency: resolvedProject.concurrency === scopedProject.concurrency ? toProjectConcurrencyDefaults(resolvedConfig.concurrency) : resolvedProject.concurrency,
|
|
127
|
+
models: resolvedProject.models === scopedProject.models ? resolvedConfig.models : resolvedProject.models,
|
|
128
|
+
reporters: resolvedProject.reporters === scopedProject.reporters ? resolvedConfig.reporters : resolvedProject.reporters
|
|
129
|
+
}, cwd, void 0, resolvedConfig.models ?? [], resolvedConfig.reporters ?? []);
|
|
130
|
+
}
|
|
38
131
|
async function applyVievalPlugins(config) {
|
|
39
132
|
let currentConfig = config;
|
|
40
133
|
const plugins = currentConfig.plugins ?? [];
|
|
@@ -49,34 +142,38 @@ async function applyVievalPlugins(config) {
|
|
|
49
142
|
for (const plugin of plugins) await plugin.configVievalResolved?.(currentConfig);
|
|
50
143
|
return currentConfig;
|
|
51
144
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
return false;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
function isConfigFileExtensionUsingRequire(extension) {
|
|
61
|
-
return extension === ".cjs" || extension === ".cts";
|
|
62
|
-
}
|
|
63
|
-
function isConfigFileExtensionUsingJsonParse(extension) {
|
|
64
|
-
return extension === ".json";
|
|
65
|
-
}
|
|
66
|
-
async function importVievalConfigModule(filePath) {
|
|
67
|
-
const extension = extname(filePath);
|
|
68
|
-
if (isConfigFileExtensionUsingJsonParse(extension)) {
|
|
69
|
-
const raw = await readFile(filePath, "utf-8");
|
|
70
|
-
return JSON.parse(raw);
|
|
71
|
-
}
|
|
72
|
-
if (isConfigFileExtensionUsingRequire(extension)) return require(filePath);
|
|
73
|
-
return import(pathToFileURL(filePath).href);
|
|
145
|
+
function assertNonAmbiguousMatrixDefinition(matrix) {
|
|
146
|
+
const matrixKeys = Object.keys(matrix);
|
|
147
|
+
const hasReservedKeys = matrixKeys.some((key) => matrixLayerKeys.has(key));
|
|
148
|
+
const hasAxisKeys = matrixKeys.some((key) => !matrixLayerKeys.has(key));
|
|
149
|
+
if (hasReservedKeys && hasAxisKeys) throw new TypeError(ambiguousMatrixDefinitionErrorMessage);
|
|
74
150
|
}
|
|
75
|
-
function
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
151
|
+
function createProjectScopedConfig(config, project, normalizedProject) {
|
|
152
|
+
return {
|
|
153
|
+
concurrency: config?.concurrency == null && normalizedProject.concurrency == null ? void 0 : {
|
|
154
|
+
...normalizedProject.concurrency,
|
|
155
|
+
workspace: config?.concurrency?.workspace
|
|
156
|
+
},
|
|
157
|
+
env: config?.env,
|
|
158
|
+
models: normalizedProject.models,
|
|
159
|
+
plugins: project.plugins,
|
|
160
|
+
projects: [{
|
|
161
|
+
concurrency: normalizedProject.concurrency,
|
|
162
|
+
evalMatrix: normalizedProject.evalMatrix,
|
|
163
|
+
exclude: normalizedProject.exclude,
|
|
164
|
+
executor: normalizedProject.executor,
|
|
165
|
+
include: normalizedProject.include,
|
|
166
|
+
inferenceExecutors: normalizedProject.inferenceExecutors,
|
|
167
|
+
models: normalizedProject.models,
|
|
168
|
+
name: normalizedProject.name,
|
|
169
|
+
plugins: project.plugins,
|
|
170
|
+
reporters: normalizedProject.reporters,
|
|
171
|
+
root: normalizedProject.root,
|
|
172
|
+
runMatrix: normalizedProject.runMatrix
|
|
173
|
+
}],
|
|
174
|
+
reporters: normalizedProject.reporters,
|
|
175
|
+
reporting: config?.reporting
|
|
176
|
+
};
|
|
80
177
|
}
|
|
81
178
|
async function findNearestConfigFile(startDirectory) {
|
|
82
179
|
const supportedFileNames = [
|
|
@@ -99,52 +196,32 @@ async function findNearestConfigFile(startDirectory) {
|
|
|
99
196
|
currentDirectory = parentDirectory;
|
|
100
197
|
}
|
|
101
198
|
}
|
|
102
|
-
async function
|
|
103
|
-
const
|
|
104
|
-
if (
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
return
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
import: importVievalConfigModule,
|
|
117
|
-
packageJson: false,
|
|
118
|
-
rcFile: false,
|
|
119
|
-
resolveModule: resolveConfigExport
|
|
120
|
-
})).config,
|
|
121
|
-
configFilePath: resolvedConfigFilePath
|
|
122
|
-
};
|
|
199
|
+
async function importVievalConfigModule(filePath) {
|
|
200
|
+
const extension = extname(filePath);
|
|
201
|
+
if (isConfigFileExtensionUsingJsonParse(extension)) {
|
|
202
|
+
const raw = await readFile(filePath, "utf-8");
|
|
203
|
+
return JSON.parse(raw);
|
|
204
|
+
}
|
|
205
|
+
if (isConfigFileExtensionUsingRequire(extension)) return require(filePath);
|
|
206
|
+
return import(pathToFileURL(filePath).href);
|
|
207
|
+
}
|
|
208
|
+
function isConfigFileExtensionUsingJsonParse(extension) {
|
|
209
|
+
return extension === ".json";
|
|
210
|
+
}
|
|
211
|
+
function isConfigFileExtensionUsingRequire(extension) {
|
|
212
|
+
return extension === ".cjs" || extension === ".cts";
|
|
123
213
|
}
|
|
124
214
|
function isLayerMatrixDefinition(matrix) {
|
|
125
215
|
const matrixKeys = Object.keys(matrix);
|
|
126
216
|
return matrixKeys.length > 0 && matrixKeys.every((key) => matrixLayerKeys.has(key));
|
|
127
217
|
}
|
|
128
|
-
function
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
if (matrix == null) return;
|
|
136
|
-
assertNonAmbiguousMatrixDefinition(matrix);
|
|
137
|
-
if (isLayerMatrixDefinition(matrix)) return matrix;
|
|
138
|
-
return { extend: matrix };
|
|
139
|
-
}
|
|
140
|
-
function toProjectConcurrencyDefaults(concurrency) {
|
|
141
|
-
if (concurrency == null) return;
|
|
142
|
-
return {
|
|
143
|
-
attempt: concurrency.attempt,
|
|
144
|
-
case: concurrency.case,
|
|
145
|
-
project: concurrency.project,
|
|
146
|
-
task: concurrency.task
|
|
147
|
-
};
|
|
218
|
+
async function isReadableFile(filePath) {
|
|
219
|
+
try {
|
|
220
|
+
await access(filePath);
|
|
221
|
+
return true;
|
|
222
|
+
} catch {
|
|
223
|
+
return false;
|
|
224
|
+
}
|
|
148
225
|
}
|
|
149
226
|
function mergeProjectConcurrency(inheritedConcurrency, projectConcurrency) {
|
|
150
227
|
if (inheritedConcurrency == null && projectConcurrency == null) return;
|
|
@@ -155,6 +232,27 @@ function mergeProjectConcurrency(inheritedConcurrency, projectConcurrency) {
|
|
|
155
232
|
task: projectConcurrency?.task ?? inheritedConcurrency?.task
|
|
156
233
|
};
|
|
157
234
|
}
|
|
235
|
+
async function normalizeConfig(config, cwd) {
|
|
236
|
+
if (config != null) {
|
|
237
|
+
if (detectCliConfigMode(config) === "comparisons") throw new Error("vieval run requires project-mode config. Received comparison-mode config.");
|
|
238
|
+
}
|
|
239
|
+
const projects = config?.workspaces == null ? config?.projects ?? [{ name: "default" }] : config.workspaces.map((workspace) => ({
|
|
240
|
+
name: workspace.id,
|
|
241
|
+
root: workspace.root
|
|
242
|
+
}));
|
|
243
|
+
const inheritedConcurrency = toProjectConcurrencyDefaults(config?.concurrency);
|
|
244
|
+
const inheritedModels = config?.models ?? [];
|
|
245
|
+
const inheritedReporterReferences = config?.reporters ?? [];
|
|
246
|
+
return Promise.all(projects.map(async (project) => {
|
|
247
|
+
return applyProjectPlugins(config, project, normalizeProjectConfig(project, cwd, inheritedConcurrency, inheritedModels, inheritedReporterReferences), cwd);
|
|
248
|
+
}));
|
|
249
|
+
}
|
|
250
|
+
function normalizeMatrixLayerInput(matrix) {
|
|
251
|
+
if (matrix == null) return;
|
|
252
|
+
assertNonAmbiguousMatrixDefinition(matrix);
|
|
253
|
+
if (isLayerMatrixDefinition(matrix)) return matrix;
|
|
254
|
+
return { extend: matrix };
|
|
255
|
+
}
|
|
158
256
|
function normalizeProjectConfig(project, cwd, inheritedConcurrency, inheritedModels, inheritedReporterReferences) {
|
|
159
257
|
const include = project.include ?? [
|
|
160
258
|
"**/*.eval.ts",
|
|
@@ -175,159 +273,61 @@ function normalizeProjectConfig(project, cwd, inheritedConcurrency, inheritedMod
|
|
|
175
273
|
const reporters = project.reporters ?? [...inheritedReporterReferences];
|
|
176
274
|
return {
|
|
177
275
|
concurrency: mergeProjectConcurrency(inheritedConcurrency, project.concurrency),
|
|
276
|
+
evalMatrix: normalizeMatrixLayerInput(project.evalMatrix),
|
|
178
277
|
exclude,
|
|
179
278
|
executor: project.executor,
|
|
180
|
-
include,
|
|
181
|
-
|
|
182
|
-
models,
|
|
183
|
-
name: project.name,
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
runMatrix: normalizeMatrixLayerInput(project.runMatrix)
|
|
187
|
-
|
|
188
|
-
};
|
|
189
|
-
}
|
|
190
|
-
function createProjectScopedConfig(config, project, normalizedProject) {
|
|
191
|
-
return {
|
|
192
|
-
concurrency: config?.concurrency == null && normalizedProject.concurrency == null ? void 0 : {
|
|
193
|
-
...normalizedProject.concurrency,
|
|
194
|
-
workspace: config?.concurrency?.workspace
|
|
195
|
-
},
|
|
196
|
-
env: config?.env,
|
|
197
|
-
models: normalizedProject.models,
|
|
198
|
-
plugins: project.plugins,
|
|
199
|
-
projects: [{
|
|
200
|
-
concurrency: normalizedProject.concurrency,
|
|
201
|
-
exclude: normalizedProject.exclude,
|
|
202
|
-
executor: normalizedProject.executor,
|
|
203
|
-
include: normalizedProject.include,
|
|
204
|
-
evalMatrix: normalizedProject.evalMatrix,
|
|
205
|
-
inferenceExecutors: normalizedProject.inferenceExecutors,
|
|
206
|
-
models: normalizedProject.models,
|
|
207
|
-
name: normalizedProject.name,
|
|
208
|
-
plugins: project.plugins,
|
|
209
|
-
reporters: normalizedProject.reporters,
|
|
210
|
-
root: normalizedProject.root,
|
|
211
|
-
runMatrix: normalizedProject.runMatrix
|
|
212
|
-
}],
|
|
213
|
-
reporters: normalizedProject.reporters,
|
|
214
|
-
reporting: config?.reporting
|
|
215
|
-
};
|
|
216
|
-
}
|
|
217
|
-
async function applyProjectPlugins(config, project, normalizedProject, cwd) {
|
|
218
|
-
if (project.plugins == null || project.plugins.length === 0) return normalizedProject;
|
|
219
|
-
const scopedConfig = createProjectScopedConfig(config, project, normalizedProject);
|
|
220
|
-
const resolvedConfig = await applyVievalPlugins(scopedConfig);
|
|
221
|
-
const scopedProject = scopedConfig.projects?.[0];
|
|
222
|
-
if (scopedProject == null) throw new Error("Project-local plugin normalization requires one scoped project.");
|
|
223
|
-
const resolvedProject = resolvedConfig.projects?.[0] ?? scopedProject;
|
|
224
|
-
return normalizeProjectConfig({
|
|
225
|
-
...resolvedProject,
|
|
226
|
-
concurrency: resolvedProject.concurrency === scopedProject.concurrency ? toProjectConcurrencyDefaults(resolvedConfig.concurrency) : resolvedProject.concurrency,
|
|
227
|
-
models: resolvedProject.models === scopedProject.models ? resolvedConfig.models : resolvedProject.models,
|
|
228
|
-
reporters: resolvedProject.reporters === scopedProject.reporters ? resolvedConfig.reporters : resolvedProject.reporters
|
|
229
|
-
}, cwd, void 0, resolvedConfig.models ?? [], resolvedConfig.reporters ?? []);
|
|
230
|
-
}
|
|
231
|
-
async function normalizeConfig(config, cwd) {
|
|
232
|
-
if (config != null) {
|
|
233
|
-
if (detectCliConfigMode(config) === "comparisons") throw new Error("vieval run requires project-mode config. Received comparison-mode config.");
|
|
234
|
-
}
|
|
235
|
-
const projects = config?.workspaces == null ? config?.projects ?? [{ name: "default" }] : config.workspaces.map((workspace) => ({
|
|
236
|
-
name: workspace.id,
|
|
237
|
-
root: workspace.root
|
|
238
|
-
}));
|
|
239
|
-
const inheritedConcurrency = toProjectConcurrencyDefaults(config?.concurrency);
|
|
240
|
-
const inheritedModels = config?.models ?? [];
|
|
241
|
-
const inheritedReporterReferences = config?.reporters ?? [];
|
|
242
|
-
return Promise.all(projects.map(async (project) => {
|
|
243
|
-
return applyProjectPlugins(config, project, normalizeProjectConfig(project, cwd, inheritedConcurrency, inheritedModels, inheritedReporterReferences), cwd);
|
|
244
|
-
}));
|
|
245
|
-
}
|
|
246
|
-
function normalizeReportingConfig(config) {
|
|
247
|
-
if (config == null) return;
|
|
248
|
-
return { openTelemetry: config.openTelemetry == null ? void 0 : {
|
|
249
|
-
enabled: config.openTelemetry.enabled ?? false,
|
|
250
|
-
onRunEnd: config.openTelemetry.onRunEnd
|
|
251
|
-
} };
|
|
252
|
-
}
|
|
253
|
-
/**
|
|
254
|
-
* Detects which top-level config mode is active.
|
|
255
|
-
*
|
|
256
|
-
* Expects:
|
|
257
|
-
* - exactly one of `projects`, `workspaces`, or `comparisons`
|
|
258
|
-
*
|
|
259
|
-
* Returns:
|
|
260
|
-
* - active top-level mode key
|
|
261
|
-
*/
|
|
262
|
-
function detectCliConfigMode(config) {
|
|
263
|
-
const declaredModes = [];
|
|
264
|
-
if (config.projects != null) declaredModes.push("projects");
|
|
265
|
-
if (config.workspaces != null) declaredModes.push("workspaces");
|
|
266
|
-
if (config.comparisons != null) declaredModes.push("comparisons");
|
|
267
|
-
if (declaredModes.length > 1) throw new Error(`Invalid vieval config: top-level keys are mutually exclusive. Found ${declaredModes.join(", ")}.`);
|
|
268
|
-
return declaredModes[0] ?? "projects";
|
|
279
|
+
include,
|
|
280
|
+
inferenceExecutors,
|
|
281
|
+
models,
|
|
282
|
+
name: project.name,
|
|
283
|
+
reporters,
|
|
284
|
+
root,
|
|
285
|
+
runMatrix: normalizeMatrixLayerInput(project.runMatrix)
|
|
286
|
+
};
|
|
269
287
|
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
const loadedConfig = await resolveVievalConfig(cwd, options.configFilePath);
|
|
277
|
-
if (loadedConfig.configFilePath == null || loadedConfig.config == null) return {
|
|
278
|
-
config: null,
|
|
279
|
-
configFilePath: null
|
|
280
|
-
};
|
|
281
|
-
const config = await applyVievalPlugins(loadedConfig.config);
|
|
282
|
-
detectCliConfigMode(config);
|
|
283
|
-
return {
|
|
284
|
-
config,
|
|
285
|
-
configFilePath: loadedConfig.configFilePath
|
|
286
|
-
};
|
|
287
|
-
} catch (error) {
|
|
288
|
-
const errorMessage = errorMessageFrom(error) ?? "Unknown config loading error.";
|
|
289
|
-
const configFilePath = options.configFilePath == null ? "vieval.config" : isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath);
|
|
290
|
-
throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
|
|
291
|
-
}
|
|
288
|
+
function normalizeReportingConfig(config) {
|
|
289
|
+
if (config == null) return;
|
|
290
|
+
return { openTelemetry: config.openTelemetry == null ? void 0 : {
|
|
291
|
+
enabled: config.openTelemetry.enabled ?? false,
|
|
292
|
+
onRunEnd: config.openTelemetry.onRunEnd
|
|
293
|
+
} };
|
|
292
294
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
throw new Error(`Failed to load vieval config "${configFilePath}": ${errorMessage}`, { cause: error });
|
|
330
|
-
}
|
|
295
|
+
function resolveConfigExport(moduleValue) {
|
|
296
|
+
if (moduleValue == null) return null;
|
|
297
|
+
if (typeof moduleValue !== "object") return moduleValue;
|
|
298
|
+
if ("default" in moduleValue) return moduleValue.default;
|
|
299
|
+
return moduleValue;
|
|
300
|
+
}
|
|
301
|
+
async function resolveVievalConfig(cwd, explicitConfigFilePath) {
|
|
302
|
+
const resolvedConfigFilePath = explicitConfigFilePath == null ? await findNearestConfigFile(cwd) : isAbsolute(explicitConfigFilePath) ? explicitConfigFilePath : resolve(cwd, explicitConfigFilePath);
|
|
303
|
+
if (explicitConfigFilePath != null && resolvedConfigFilePath != null && !await isReadableFile(resolvedConfigFilePath)) throw new Error(`Config file does not exist or is not readable: ${resolvedConfigFilePath}`);
|
|
304
|
+
if (resolvedConfigFilePath == null) return {
|
|
305
|
+
config: null,
|
|
306
|
+
configFilePath: null
|
|
307
|
+
};
|
|
308
|
+
return {
|
|
309
|
+
config: (await loadConfig({
|
|
310
|
+
configFile: resolvedConfigFilePath,
|
|
311
|
+
cwd,
|
|
312
|
+
dotenv: false,
|
|
313
|
+
envName: false,
|
|
314
|
+
extend: false,
|
|
315
|
+
import: importVievalConfigModule,
|
|
316
|
+
packageJson: false,
|
|
317
|
+
rcFile: false,
|
|
318
|
+
resolveModule: resolveConfigExport
|
|
319
|
+
})).config,
|
|
320
|
+
configFilePath: resolvedConfigFilePath
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
function toProjectConcurrencyDefaults(concurrency) {
|
|
324
|
+
if (concurrency == null) return;
|
|
325
|
+
return {
|
|
326
|
+
attempt: concurrency.attempt,
|
|
327
|
+
case: concurrency.case,
|
|
328
|
+
project: concurrency.project,
|
|
329
|
+
task: concurrency.task
|
|
330
|
+
};
|
|
331
331
|
}
|
|
332
332
|
//#endregion
|
|
333
333
|
//#region src/core/telemetry/noop.ts
|
|
@@ -346,62 +346,16 @@ async function loadVievalCliConfig(options = {}) {
|
|
|
346
346
|
*/
|
|
347
347
|
function createNoopTelemetryRuntime() {
|
|
348
348
|
return {
|
|
349
|
-
async withSpan(_name, _attributes, callback) {
|
|
350
|
-
return await callback();
|
|
351
|
-
},
|
|
352
349
|
addEvent() {},
|
|
350
|
+
recordException() {},
|
|
353
351
|
setAttributes() {},
|
|
354
|
-
|
|
352
|
+
async withSpan(_name, _attributes, callback) {
|
|
353
|
+
return await callback();
|
|
354
|
+
}
|
|
355
355
|
};
|
|
356
356
|
}
|
|
357
357
|
//#endregion
|
|
358
358
|
//#region src/core/telemetry/otel.ts
|
|
359
|
-
async function importOpenTelemetryApi() {
|
|
360
|
-
return await import("@opentelemetry/api");
|
|
361
|
-
}
|
|
362
|
-
function isOpenTelemetryAttributeScalar(value) {
|
|
363
|
-
return typeof value === "boolean" || typeof value === "number" || typeof value === "string";
|
|
364
|
-
}
|
|
365
|
-
function isHomogeneousOpenTelemetryAttributeArray(value) {
|
|
366
|
-
if (value.length === 0) return true;
|
|
367
|
-
const firstType = typeof value[0];
|
|
368
|
-
if (firstType !== "boolean" && firstType !== "number" && firstType !== "string") return false;
|
|
369
|
-
return value.every((item) => typeof item === firstType);
|
|
370
|
-
}
|
|
371
|
-
function stringifyAttributeValue(value) {
|
|
372
|
-
try {
|
|
373
|
-
return JSON.stringify(value);
|
|
374
|
-
} catch {
|
|
375
|
-
return String(value);
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
/**
|
|
379
|
-
* Normalizes JSON-compatible telemetry attributes into OpenTelemetry-safe attributes.
|
|
380
|
-
*
|
|
381
|
-
* Before:
|
|
382
|
-
* - `{ nil: null, nested: ['a', [1, null]], scalarArray: ['a', 1, true] }`
|
|
383
|
-
*
|
|
384
|
-
* After:
|
|
385
|
-
* - `{ nested: '["a",[1,null]]', scalarArray: ['a', 1, true] }`
|
|
386
|
-
*/
|
|
387
|
-
function normalizeOpenTelemetryAttributes(attributes) {
|
|
388
|
-
if (attributes == null) return;
|
|
389
|
-
const normalized = {};
|
|
390
|
-
for (const [key, value] of Object.entries(attributes)) {
|
|
391
|
-
if (value == null) continue;
|
|
392
|
-
if (isOpenTelemetryAttributeScalar(value)) {
|
|
393
|
-
normalized[key] = value;
|
|
394
|
-
continue;
|
|
395
|
-
}
|
|
396
|
-
if (Array.isArray(value)) {
|
|
397
|
-
normalized[key] = isHomogeneousOpenTelemetryAttributeArray(value) ? value : stringifyAttributeValue(value) ?? "";
|
|
398
|
-
continue;
|
|
399
|
-
}
|
|
400
|
-
const stringified = stringifyAttributeValue(value);
|
|
401
|
-
if (stringified != null) normalized[key] = stringified;
|
|
402
|
-
}
|
|
403
|
-
return normalized;
|
|
404
|
-
}
|
|
405
359
|
/**
|
|
406
360
|
* Creates an OpenTelemetry-backed runtime using active spans.
|
|
407
361
|
*
|
|
@@ -428,6 +382,15 @@ function createOpenTelemetryRuntime(options = {}) {
|
|
|
428
382
|
return await apiPromise;
|
|
429
383
|
}
|
|
430
384
|
return {
|
|
385
|
+
addEvent(name, attributes) {
|
|
386
|
+
loadedApi?.trace.getActiveSpan()?.addEvent(name, normalizeOpenTelemetryAttributes(attributes));
|
|
387
|
+
},
|
|
388
|
+
recordException(error) {
|
|
389
|
+
loadedApi?.trace.getActiveSpan()?.recordException(error);
|
|
390
|
+
},
|
|
391
|
+
setAttributes(attributes) {
|
|
392
|
+
loadedApi?.trace.getActiveSpan()?.setAttributes(normalizeOpenTelemetryAttributes(attributes) ?? {});
|
|
393
|
+
},
|
|
431
394
|
async withSpan(name, attributes, callback) {
|
|
432
395
|
const api = await getApi();
|
|
433
396
|
return await api.trace.getTracer("vieval").startActiveSpan(name, { attributes: normalizeOpenTelemetryAttributes(attributes) ?? {} }, async (span) => {
|
|
@@ -444,29 +407,58 @@ function createOpenTelemetryRuntime(options = {}) {
|
|
|
444
407
|
span.end();
|
|
445
408
|
}
|
|
446
409
|
});
|
|
447
|
-
},
|
|
448
|
-
addEvent(name, attributes) {
|
|
449
|
-
loadedApi?.trace.getActiveSpan()?.addEvent(name, normalizeOpenTelemetryAttributes(attributes));
|
|
450
|
-
},
|
|
451
|
-
setAttributes(attributes) {
|
|
452
|
-
loadedApi?.trace.getActiveSpan()?.setAttributes(normalizeOpenTelemetryAttributes(attributes) ?? {});
|
|
453
|
-
},
|
|
454
|
-
recordException(error) {
|
|
455
|
-
loadedApi?.trace.getActiveSpan()?.recordException(error);
|
|
456
410
|
}
|
|
457
411
|
};
|
|
458
412
|
}
|
|
413
|
+
async function importOpenTelemetryApi() {
|
|
414
|
+
return await import("@opentelemetry/api");
|
|
415
|
+
}
|
|
416
|
+
function isHomogeneousOpenTelemetryAttributeArray(value) {
|
|
417
|
+
if (value.length === 0) return true;
|
|
418
|
+
const firstType = typeof value[0];
|
|
419
|
+
if (firstType !== "boolean" && firstType !== "number" && firstType !== "string") return false;
|
|
420
|
+
return value.every((item) => typeof item === firstType);
|
|
421
|
+
}
|
|
422
|
+
function isOpenTelemetryAttributeScalar(value) {
|
|
423
|
+
return typeof value === "boolean" || typeof value === "number" || typeof value === "string";
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* Normalizes JSON-compatible telemetry attributes into OpenTelemetry-safe attributes.
|
|
427
|
+
*
|
|
428
|
+
* Before:
|
|
429
|
+
* - `{ nil: null, nested: ['a', [1, null]], scalarArray: ['a', 1, true] }`
|
|
430
|
+
*
|
|
431
|
+
* After:
|
|
432
|
+
* - `{ nested: '["a",[1,null]]', scalarArray: ['a', 1, true] }`
|
|
433
|
+
*/
|
|
434
|
+
function normalizeOpenTelemetryAttributes(attributes) {
|
|
435
|
+
if (attributes == null) return;
|
|
436
|
+
const normalized = {};
|
|
437
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
438
|
+
if (value == null) continue;
|
|
439
|
+
if (isOpenTelemetryAttributeScalar(value)) {
|
|
440
|
+
normalized[key] = value;
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
if (Array.isArray(value)) {
|
|
444
|
+
normalized[key] = isHomogeneousOpenTelemetryAttributeArray(value) ? value : stringifyAttributeValue(value) ?? "";
|
|
445
|
+
continue;
|
|
446
|
+
}
|
|
447
|
+
const stringified = stringifyAttributeValue(value);
|
|
448
|
+
if (stringified != null) normalized[key] = stringified;
|
|
449
|
+
}
|
|
450
|
+
return normalized;
|
|
451
|
+
}
|
|
452
|
+
function stringifyAttributeValue(value) {
|
|
453
|
+
try {
|
|
454
|
+
return JSON.stringify(value);
|
|
455
|
+
} catch {
|
|
456
|
+
return String(value);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
459
|
//#endregion
|
|
460
460
|
//#region src/dsl/registry.ts
|
|
461
461
|
const registryStoreSymbol = Symbol.for("vieval.dsl.registry.store");
|
|
462
|
-
function getRegistryStore() {
|
|
463
|
-
const processWithStore = process;
|
|
464
|
-
processWithStore[registryStoreSymbol] ??= {
|
|
465
|
-
activeModuleHref: null,
|
|
466
|
-
registeredDefinitionsByModule: /* @__PURE__ */ new Map()
|
|
467
|
-
};
|
|
468
|
-
return processWithStore[registryStoreSymbol];
|
|
469
|
-
}
|
|
470
462
|
/**
|
|
471
463
|
* Starts module-scoped eval registration collection.
|
|
472
464
|
*/
|
|
@@ -475,6 +467,15 @@ function beginModuleRegistration(moduleHref) {
|
|
|
475
467
|
store.activeModuleHref = moduleHref;
|
|
476
468
|
}
|
|
477
469
|
/**
|
|
470
|
+
* Consumes registered definitions for one module and clears stored state.
|
|
471
|
+
*/
|
|
472
|
+
function consumeModuleRegistrations(moduleHref) {
|
|
473
|
+
const store = getRegistryStore();
|
|
474
|
+
const definitions = store.registeredDefinitionsByModule.get(moduleHref) ?? [];
|
|
475
|
+
store.registeredDefinitionsByModule.delete(moduleHref);
|
|
476
|
+
return definitions;
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
478
479
|
* Ends module-scoped eval registration collection.
|
|
479
480
|
*/
|
|
480
481
|
function endModuleRegistration() {
|
|
@@ -491,16 +492,15 @@ function registerEvalDefinition(definition) {
|
|
|
491
492
|
existing.push(definition);
|
|
492
493
|
store.registeredDefinitionsByModule.set(store.activeModuleHref, existing);
|
|
493
494
|
}
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
return definitions;
|
|
495
|
+
function getRegistryStore() {
|
|
496
|
+
const processWithStore = process;
|
|
497
|
+
processWithStore[registryStoreSymbol] ??= {
|
|
498
|
+
activeModuleHref: null,
|
|
499
|
+
registeredDefinitionsByModule: /* @__PURE__ */ new Map()
|
|
500
|
+
};
|
|
501
|
+
return processWithStore[registryStoreSymbol];
|
|
502
502
|
}
|
|
503
503
|
//#endregion
|
|
504
504
|
export { createOpenTelemetryRuntime as a, detectCliConfigMode as c, loadVievalCliConfig as d, registerEvalDefinition as i, loadEnv$1 as l, consumeModuleRegistrations as n, createNoopTelemetryRuntime as o, endModuleRegistration as r, defineConfig as s, beginModuleRegistration as t, loadRawVievalConfig as u };
|
|
505
505
|
|
|
506
|
-
//# sourceMappingURL=registry-
|
|
506
|
+
//# sourceMappingURL=registry-BK7k6X81.mjs.map
|