vieval 0.0.3 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +6 -3
  2. package/dist/bin/vieval.d.mts +1 -0
  3. package/dist/bin/vieval.mjs +33 -0
  4. package/dist/bin/vieval.mjs.map +1 -0
  5. package/dist/cli/index.d.mts +32 -0
  6. package/dist/cli/index.mjs +1 -2576
  7. package/dist/cli-DayPXzHX.mjs +2593 -0
  8. package/dist/cli-DayPXzHX.mjs.map +1 -0
  9. package/dist/config.d.mts +1 -1
  10. package/dist/config.mjs +17 -2
  11. package/dist/config.mjs.map +1 -0
  12. package/dist/core/assertions/index.d.mts +314 -2
  13. package/dist/core/assertions/index.mjs +182 -1
  14. package/dist/core/assertions/index.mjs.map +1 -0
  15. package/dist/core/inference-executors/index.d.mts +1 -1
  16. package/dist/core/inference-executors/index.mjs +1 -1
  17. package/dist/core/processors/results/index.d.mts +1 -1
  18. package/dist/core/runner/index.d.mts +1 -1
  19. package/dist/core/runner/index.mjs +635 -1
  20. package/dist/core/runner/index.mjs.map +1 -0
  21. package/dist/{env-C7X81PWa.mjs → env-BFSjny07.mjs} +1 -1
  22. package/dist/{env-C7X81PWa.mjs.map → env-BFSjny07.mjs.map} +1 -1
  23. package/dist/{env-DtpjACOW.d.mts → env-BTq3dV7C.d.mts} +1 -1
  24. package/dist/{expect-extensions-BOzwV5EJ.mjs → expect-extensions-QLXESWjn.mjs} +2 -2
  25. package/dist/{expect-extensions-BOzwV5EJ.mjs.map → expect-extensions-QLXESWjn.mjs.map} +1 -1
  26. package/dist/expect.d.mts +10 -2
  27. package/dist/expect.mjs +16 -1
  28. package/dist/expect.mjs.map +1 -0
  29. package/dist/{index-BDMEAmf2.d.mts → index-OEdqjQSe.d.mts} +2 -2
  30. package/dist/index.d.mts +3 -3
  31. package/dist/index.mjs +4 -4
  32. package/dist/{models-DIGdOUpJ.mjs → models-D_MsBtYw.mjs} +1 -1
  33. package/dist/{models-DIGdOUpJ.mjs.map → models-D_MsBtYw.mjs.map} +1 -1
  34. package/dist/plugins/chat-models/index.d.mts +1 -1
  35. package/dist/plugins/chat-models/index.mjs +1 -1
  36. package/dist/{registry-CHJcTN2W.mjs → registry-CwcMMjnZ.mjs} +3 -3
  37. package/dist/{registry-CHJcTN2W.mjs.map → registry-CwcMMjnZ.mjs.map} +1 -1
  38. package/dist/testing/expect-extensions.d.mts +1 -1
  39. package/dist/testing/expect-extensions.mjs +1 -1
  40. package/package.json +3 -3
  41. package/dist/assertions-DcAjfVDA.mjs +0 -183
  42. package/dist/assertions-DcAjfVDA.mjs.map +0 -1
  43. package/dist/cli/index.mjs.map +0 -1
  44. package/dist/config-CHN24egi.mjs +0 -17
  45. package/dist/config-CHN24egi.mjs.map +0 -1
  46. package/dist/expect-B2vaoRVZ.d.mts +0 -10
  47. package/dist/expect-CaXiUkwY.mjs +0 -17
  48. package/dist/expect-CaXiUkwY.mjs.map +0 -1
  49. package/dist/index-C3gPFmcR.d.mts +0 -314
  50. package/dist/runner-Dpy-eivM.mjs +0 -636
  51. package/dist/runner-Dpy-eivM.mjs.map +0 -1
@@ -1,2577 +1,2 @@
1
- #!/usr/bin/env node
2
- import { c as loadRawVievalConfig, l as loadVievalCliConfig, n as consumeModuleRegistrations, o as detectCliConfigMode, r as endModuleRegistration, t as beginModuleRegistration } from "../registry-CHJcTN2W.mjs";
3
- import { a as runScheduledTasks, i as RunnerExecutionError, l as createFilesystemTaskCacheRuntime, n as createRunnerSchedule, r as createRunnerRuntimeContext, s as collectEvalEntries, t as createTaskExecutionContext } from "../runner-Dpy-eivM.mjs";
4
- import process from "node:process";
5
- import { access, mkdir, writeFile } from "node:fs/promises";
6
- import path, { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
7
- import { fileURLToPath, pathToFileURL } from "node:url";
8
- import { errorMessageFrom } from "@moeru/std";
9
- import { existsSync, readFileSync, realpathSync } from "node:fs";
10
- import meow from "meow";
11
- import { glob } from "tinyglobby";
12
- import { randomUUID } from "node:crypto";
13
- import c from "tinyrainbow";
14
- import { uniq } from "es-toolkit";
15
- import { createVitest } from "vitest/node";
16
- import { stripVTControlCharacters } from "node:util";
17
- import stringWidth from "fast-string-width";
18
- //#region src/cli/comparison-config.ts
19
- const supportedWorkspaceConfigFileNames = [
20
- "vieval.config.ts",
21
- "vieval.config.mts",
22
- "vieval.config.cts",
23
- "vieval.config.js",
24
- "vieval.config.mjs",
25
- "vieval.config.cjs",
26
- "vieval.config.json"
27
- ];
28
- async function isReadableFile(filePath) {
29
- try {
30
- await access(filePath);
31
- return true;
32
- } catch {
33
- return false;
34
- }
35
- }
36
- function normalizeGlobInput(patterns) {
37
- if (patterns == null) return [];
38
- return (typeof patterns === "string" ? [patterns] : patterns).map((pattern) => pattern.trim()).filter((pattern) => pattern.length > 0);
39
- }
40
- function normalizeMethodShape(method, configDirectory, index) {
41
- const id = method.id.trim();
42
- const workspace = method.workspace.trim();
43
- const project = method.project.trim();
44
- const configFilePath = method.configFilePath?.trim();
45
- if (id.length === 0) throw new Error(`Comparison method #${index + 1} is missing id.`);
46
- if (workspace.length === 0) throw new Error(`Comparison method "${id}" is missing workspace.`);
47
- if (project.length === 0) throw new Error(`Comparison method "${id}" is missing project.`);
48
- const resolvedWorkspace = isAbsolute(workspace) ? workspace : resolve(configDirectory, workspace);
49
- return {
50
- configFilePath: configFilePath == null || configFilePath.length === 0 ? void 0 : isAbsolute(configFilePath) ? configFilePath : resolve(configDirectory, configFilePath),
51
- id,
52
- project,
53
- workspace: resolvedWorkspace
54
- };
55
- }
56
- async function findWorkspaceConfigFile(workspaceDirectory) {
57
- for (const fileName of supportedWorkspaceConfigFileNames) {
58
- const candidate = join(workspaceDirectory, fileName);
59
- if (await isReadableFile(candidate)) return candidate;
60
- }
61
- return null;
62
- }
63
- function createDiscoveredMethodId(configDirectory, workspace, projectName) {
64
- const relativeWorkspace = relative(configDirectory, workspace);
65
- return `${(relativeWorkspace.length > 0 ? relativeWorkspace : basename(workspace)).replaceAll("\\", "/")}:${projectName}`;
66
- }
67
- async function discoverMethodsFromWorkspaceGlobs(args) {
68
- const includes = normalizeGlobInput(args.comparison.includesWorkspaces);
69
- if (includes.length === 0) return [];
70
- const discoveredWorkspaceDirectories = await glob(includes, {
71
- absolute: true,
72
- cwd: args.configDirectory,
73
- ignore: normalizeGlobInput(args.comparison.excludesWorkspaces),
74
- onlyDirectories: true
75
- });
76
- const methods = [];
77
- for (const workspaceDirectory of discoveredWorkspaceDirectories.sort((left, right) => left.localeCompare(right))) {
78
- const configFilePath = await findWorkspaceConfigFile(workspaceDirectory);
79
- if (configFilePath == null) continue;
80
- const loadedWorkspaceConfig = await loadVievalCliConfig({
81
- configFilePath,
82
- cwd: workspaceDirectory
83
- });
84
- for (const project of loadedWorkspaceConfig.projects) methods.push({
85
- configFilePath,
86
- id: createDiscoveredMethodId(args.configDirectory, workspaceDirectory, project.name),
87
- project: project.name,
88
- workspace: workspaceDirectory
89
- });
90
- }
91
- return methods;
92
- }
93
- function validateMethodIdsAreUnique(methods) {
94
- const methodIds = methods.map((method) => method.id);
95
- const duplicatedMethodId = methodIds.find((methodId, index) => methodIds.indexOf(methodId) !== index);
96
- if (duplicatedMethodId != null) throw new Error(`Duplicate comparison method id "${duplicatedMethodId}".`);
97
- }
98
- function assertComparisonMode(config) {
99
- const mode = detectCliConfigMode(config);
100
- if (mode !== "comparisons") throw new Error(`Expected comparison-mode config, but received ${mode}-mode config.`);
101
- }
102
- function selectComparisonConfig(comparisons, comparisonId) {
103
- if (comparisons.length === 0) throw new Error("Comparison config requires at least one comparisons entry.");
104
- if (comparisonId == null || comparisonId.trim().length === 0) {
105
- if (comparisons.length > 1) throw new Error(`Multiple comparisons found. Provide --comparison. Available ids: ${comparisons.map((item) => item.id).join(", ")}`);
106
- return comparisons[0];
107
- }
108
- const selected = comparisons.find((item) => item.id === comparisonId);
109
- if (selected == null) throw new Error(`Unknown comparison id "${comparisonId}".`);
110
- return selected;
111
- }
112
- function normalizeBenchmark(comparison) {
113
- const benchmarkId = comparison.benchmark.id.trim();
114
- const sharedCaseNamespace = comparison.benchmark.sharedCaseNamespace.trim();
115
- if (benchmarkId.length === 0) throw new Error("Comparison config requires benchmark.id.");
116
- if (sharedCaseNamespace.length === 0) throw new Error("Comparison config requires benchmark.sharedCaseNamespace.");
117
- return {
118
- id: benchmarkId,
119
- sharedCaseNamespace
120
- };
121
- }
122
- /**
123
- * Loads and validates comparison-mode data from `vieval.config.*`.
124
- */
125
- async function loadVievalComparisonConfig(options = {}) {
126
- const cwd = options.cwd ?? process.cwd();
127
- try {
128
- const loaded = await loadRawVievalConfig({
129
- configFilePath: options.configFilePath,
130
- cwd
131
- });
132
- if (loaded.configFilePath == null || loaded.config == null) throw new Error("Failed to find vieval config. Expected vieval.config.*");
133
- assertComparisonMode(loaded.config);
134
- const selectedComparison = selectComparisonConfig(loaded.config.comparisons, options.comparisonId);
135
- const configDirectory = dirname(loaded.configFilePath);
136
- const explicitMethods = (selectedComparison.methods ?? []).map((method, index) => normalizeMethodShape(method, configDirectory, index));
137
- const discoveredMethods = await discoverMethodsFromWorkspaceGlobs({
138
- comparison: selectedComparison,
139
- configDirectory
140
- });
141
- const methods = [...explicitMethods, ...discoveredMethods];
142
- if (methods.length === 0) throw new Error("Comparison config resolved zero methods. Configure methods or includesWorkspaces.");
143
- validateMethodIdsAreUnique(methods);
144
- return {
145
- config: {
146
- benchmark: normalizeBenchmark(selectedComparison),
147
- methods
148
- },
149
- configFilePath: loaded.configFilePath
150
- };
151
- } catch (error) {
152
- const errorMessage = errorMessageFrom(error) ?? "Unknown comparison config loading error.";
153
- const resolvedPath = options.configFilePath ?? "vieval.config";
154
- throw new Error(`Failed to load comparison config "${resolvedPath}": ${errorMessage}`);
155
- }
156
- }
157
- //#endregion
158
- //#region src/cli/report-compare.ts
159
- /**
160
- * Builds a compact compare report sorted by hybrid/exact score.
161
- */
162
- function buildCompareReportArtifact(args) {
163
- const rows = args.methods.map((method) => {
164
- const overall = method.output.projects[0]?.result?.overall;
165
- return {
166
- exactAverage: overall?.exactAverage ?? null,
167
- hybridAverage: overall?.hybridAverage ?? null,
168
- methodId: method.methodId,
169
- runCount: overall?.runCount ?? 0
170
- };
171
- });
172
- rows.sort((left, right) => {
173
- const leftHybrid = left.hybridAverage ?? Number.NEGATIVE_INFINITY;
174
- const rightHybrid = right.hybridAverage ?? Number.NEGATIVE_INFINITY;
175
- if (leftHybrid !== rightHybrid) return rightHybrid - leftHybrid;
176
- const leftExact = left.exactAverage ?? Number.NEGATIVE_INFINITY;
177
- return (right.exactAverage ?? Number.NEGATIVE_INFINITY) - leftExact;
178
- });
179
- return {
180
- benchmarkId: args.benchmarkId,
181
- methods: rows,
182
- reportPath: args.reportPath
183
- };
184
- }
185
- /**
186
- * Writes compare report artifact as JSON.
187
- */
188
- async function writeCompareReportArtifact(args) {
189
- const outputPath = resolve(args.outputPath);
190
- await mkdir(dirname(outputPath), { recursive: true });
191
- await writeFile(outputPath, `${JSON.stringify(args.artifact, null, 2)}\n`, "utf-8");
192
- return outputPath;
193
- }
194
- //#endregion
195
- //#region src/cli/discovery.ts
196
- /**
197
- * Discovers eval files using include/exclude globs relative to project root.
198
- *
199
- * Before:
200
- * - Absolute path file list from recursive filesystem walk
201
- *
202
- * After:
203
- * - Filtered absolute path list matching include/exclude rules
204
- */
205
- async function discoverEvalFiles(options) {
206
- return uniq(await glob([...options.include], {
207
- absolute: true,
208
- cwd: options.root,
209
- ignore: [...options.exclude],
210
- onlyFiles: true
211
- })).sort((left, right) => left.localeCompare(right));
212
- }
213
- //#endregion
214
- //#region src/cli/module-runtime.ts
215
- /**
216
- * Loads eval modules and returns a normalized eval-module map.
217
- *
218
- * Use when:
219
- * - CLI collection needs Vite/Vitest-powered module resolution and transforms
220
- * - eval files should be imported with the same runtime semantics as Vitest
221
- *
222
- * Expects:
223
- * - `projectRoot` points at the project that owns the eval files
224
- * - each `evalFilePaths` entry is an absolute file path
225
- *
226
- * Returns:
227
- * - eval modules keyed by stable file href + optional registration suffixes
228
- */
229
- async function loadEvalModulesWithVitestRuntime(evalFilePaths, projectRoot) {
230
- const loadedModules = {};
231
- const runtime = await createVitest("test", {
232
- config: false,
233
- root: projectRoot,
234
- run: false,
235
- silent: true,
236
- watch: false
237
- });
238
- try {
239
- for (const evalFilePath of evalFilePaths) {
240
- const moduleHref = pathToFileURL(evalFilePath).href;
241
- beginModuleRegistration(moduleHref);
242
- try {
243
- const moduleValue = await runtime.import(moduleHref);
244
- const registeredDefinitions = consumeModuleRegistrations(moduleHref);
245
- const defaultDefinition = moduleValue.default;
246
- const definitions = [...registeredDefinitions, ...defaultDefinition == null ? [] : [defaultDefinition]];
247
- const deduplicatedDefinitions = definitions.filter((definition, index) => {
248
- const key = `${definition.name}::${definition.description}::${definition.task?.id ?? ""}`;
249
- return definitions.findIndex((candidate) => `${candidate.name}::${candidate.description}::${candidate.task?.id ?? ""}` === key) === index;
250
- });
251
- if (deduplicatedDefinitions.length === 0) continue;
252
- for (const [definitionIndex, definition] of deduplicatedDefinitions.entries()) {
253
- const moduleKey = definitionIndex === 0 ? moduleHref : `${moduleHref}#registration-${definitionIndex + 1}`;
254
- loadedModules[moduleKey] = { default: definition };
255
- }
256
- } finally {
257
- endModuleRegistration();
258
- }
259
- }
260
- } finally {
261
- await runtime.close();
262
- }
263
- return loadedModules;
264
- }
265
- //#endregion
266
- //#region src/cli/reporters/noop-reporter.ts
267
- /**
268
- * Creates a reporter that intentionally does nothing.
269
- *
270
- * Use when:
271
- * - terminal output should stay silent
272
- * - reporter wiring needs a safe default for tests or non-interactive runs
273
- *
274
- * Expects:
275
- * - callers may invoke any lifecycle method in any order that matches the run
276
- *
277
- * Returns:
278
- * - a stable reporter implementation with no observable side effects
279
- */
280
- function createNoopReporter() {
281
- return {
282
- onRunStart(_payload) {},
283
- onTaskQueued(_payload) {},
284
- onTaskStart(_payload) {},
285
- onCaseStart(_payload) {},
286
- onCaseEnd(_payload) {},
287
- onTaskEnd(_payload) {},
288
- onRunEnd(_payload) {},
289
- dispose() {}
290
- };
291
- }
292
- //#endregion
293
- //#region src/cli/reporters/summary-reporter.ts
294
- const POINTER = "❯";
295
- const TREE_NODE_END = "└";
296
- const TREE_NODE_MIDDLE = "├";
297
- var SummaryReporterStateMachine = class {
298
- options;
299
- taskCounters = createCounterState();
300
- caseCounters = createCounterState();
301
- tasks = /* @__PURE__ */ new Map();
302
- queueOrderCounter = 0;
303
- startedAtMs = 0;
304
- startTime = "";
305
- constructor(options) {
306
- this.options = options;
307
- }
308
- /**
309
- * Handles run startup.
310
- *
311
- * Use when:
312
- * - a new CLI run is starting and the summary state must reset
313
- *
314
- * Expects:
315
- * - `totalTasks` matches the scheduled task count for the run
316
- *
317
- * Returns:
318
- * - no direct value
319
- */
320
- onRunStart(payload) {
321
- this.tasks.clear();
322
- this.queueOrderCounter = 0;
323
- resetCounterState(this.taskCounters, payload.totalTasks);
324
- resetCounterState(this.caseCounters, 0);
325
- this.startedAtMs = this.options.getNow();
326
- this.startTime = formatTimeString(new Date(this.options.getWallClockNow()));
327
- }
328
- /**
329
- * Handles task queue events.
330
- *
331
- * Use when:
332
- * - a scheduled task becomes visible in the live summary before it starts
333
- *
334
- * Expects:
335
- * - `taskId` is stable across later lifecycle events
336
- *
337
- * Returns:
338
- * - no direct value
339
- */
340
- onTaskQueued(payload) {
341
- const task = this.getOrCreateTaskState(payload.taskId);
342
- if (task.state === "finished") return;
343
- task.displayName = payload.displayName ?? task.displayName;
344
- task.projectName = payload.projectName ?? task.projectName;
345
- this.syncTaskTotalCases(task, payload.totalCases);
346
- }
347
- /**
348
- * Handles task start events.
349
- *
350
- * Use when:
351
- * - a queued task begins executing
352
- *
353
- * Expects:
354
- * - the task was previously queued or can be synthesized from its identifier
355
- *
356
- * Returns:
357
- * - no direct value
358
- */
359
- onTaskStart(payload) {
360
- const task = this.getOrCreateTaskState(payload.taskId);
361
- if (task.state === "finished") return;
362
- task.state = "running";
363
- task.startedAt ??= this.options.getNow();
364
- }
365
- /**
366
- * Handles case start events.
367
- *
368
- * Use when:
369
- * - a running task starts one case and slow-case tracking may begin
370
- *
371
- * Expects:
372
- * - `caseId` is stable for the lifetime of the running case
373
- *
374
- * Returns:
375
- * - no direct value
376
- */
377
- onCaseStart(payload) {
378
- const task = this.getOrCreateTaskState(payload.taskId);
379
- if (task.state === "finished") return;
380
- task.state = "running";
381
- task.startedAt ??= this.options.getNow();
382
- if (task.settledCaseIds.has(payload.caseId) || task.runningCases.has(payload.caseId)) return;
383
- task.caseOrderCounter += 1;
384
- task.runningCases.set(payload.caseId, {
385
- caseId: payload.caseId,
386
- caseName: payload.caseName ?? payload.caseId,
387
- order: task.caseOrderCounter,
388
- startedAt: this.options.getNow()
389
- });
390
- this.syncTaskTotalCases(task);
391
- }
392
- /**
393
- * Handles case completion.
394
- *
395
- * Use when:
396
- * - a running case settles and counters must advance
397
- *
398
- * Expects:
399
- * - duplicate completion for the same `caseId` is ignored
400
- *
401
- * Returns:
402
- * - no direct value
403
- */
404
- onCaseEnd(payload) {
405
- const task = this.getOrCreateTaskState(payload.taskId);
406
- if (task.state === "finished") return;
407
- if (task.settledCaseIds.has(payload.caseId)) {
408
- task.runningCases.delete(payload.caseId);
409
- return;
410
- }
411
- task.settledCaseIds.add(payload.caseId);
412
- task.runningCases.delete(payload.caseId);
413
- task.completedCases += 1;
414
- this.syncTaskTotalCases(task);
415
- this.caseCounters.completed += 1;
416
- if (payload.state === "passed") {
417
- this.caseCounters.passed += 1;
418
- return;
419
- }
420
- if (payload.state === "failed") {
421
- this.caseCounters.failed += 1;
422
- return;
423
- }
424
- this.caseCounters.skipped += 1;
425
- }
426
- /**
427
- * Handles task completion.
428
- *
429
- * Use when:
430
- * - a task leaves the active window and contributes to terminal totals
431
- *
432
- * Expects:
433
- * - duplicate task completion for the same task is ignored
434
- *
435
- * Returns:
436
- * - no direct value
437
- */
438
- onTaskEnd(payload) {
439
- const task = this.getOrCreateTaskState(payload.taskId);
440
- if (task.state === "finished") return;
441
- this.syncTaskTotalCases(task);
442
- task.state = "finished";
443
- task.taskResult = payload.state;
444
- task.runningCases.clear();
445
- this.taskCounters.completed += 1;
446
- if (payload.state === "passed") {
447
- this.taskCounters.passed += 1;
448
- return;
449
- }
450
- if (payload.state === "failed") {
451
- this.taskCounters.failed += 1;
452
- return;
453
- }
454
- this.taskCounters.skipped += 1;
455
- }
456
- /**
457
- * Handles run completion.
458
- *
459
- * Use when:
460
- * - the caller has final task totals and wants the footer normalized
461
- *
462
- * Expects:
463
- * - payload counters are final terminal task totals
464
- *
465
- * Returns:
466
- * - no direct value
467
- */
468
- onRunEnd(payload) {
469
- this.taskCounters.total = payload.totalTasks;
470
- this.taskCounters.passed = payload.passedTasks;
471
- this.taskCounters.failed = payload.failedTasks;
472
- this.taskCounters.skipped = payload.skippedTasks;
473
- this.taskCounters.completed = payload.passedTasks + payload.failedTasks + payload.skippedTasks;
474
- }
475
- /**
476
- * Releases reporter resources.
477
- *
478
- * Use when:
479
- * - CLI cleanup runs from a `finally` block
480
- *
481
- * Expects:
482
- * - repeated calls are safe
483
- *
484
- * Returns:
485
- * - no direct value
486
- */
487
- dispose() {}
488
- /**
489
- * Builds the current live summary window rows.
490
- *
491
- * Use when:
492
- * - the live reporter or tests need a snapshot of the active window
493
- *
494
- * Expects:
495
- * - `maxRows`, when present, keeps footer rows visible
496
- *
497
- * Returns:
498
- * - terminal rows in display order
499
- */
500
- getWindowRows(options) {
501
- const activeRows = this.createActiveRows();
502
- const footerRows = this.createFooterRows();
503
- const maxRows = options?.maxRows;
504
- const activeBlock = [
505
- "",
506
- ...activeRows,
507
- ...activeRows.length > 0 ? [""] : []
508
- ];
509
- const footerBlock = [...footerRows, ""];
510
- if (maxRows == null || maxRows <= 0) return [...activeBlock, ...footerBlock];
511
- if (maxRows <= footerBlock.length) return footerBlock.slice(-maxRows);
512
- const availableActiveRows = Math.max(0, maxRows - footerBlock.length);
513
- return [...activeBlock.slice(0, availableActiveRows), ...footerBlock];
514
- }
515
- createActiveRows() {
516
- const activeTasks = Array.from(this.tasks.values()).filter((task) => task.state !== "finished").sort(compareActiveTasks);
517
- const rows = [];
518
- for (const task of activeTasks) {
519
- const suffix = task.state === "queued" ? c.dim(" [queued]") : ` ${task.completedCases}/${task.totalCases}`;
520
- const badge = formatProjectBadge(task.projectName, this.options.isTTY);
521
- rows.push(c.bold(c.yellow(` ${POINTER} `)) + badge + task.displayName + c.dim(suffix));
522
- const slowCases = Array.from(task.runningCases.values()).filter((activeCase) => this.options.getNow() - activeCase.startedAt >= this.options.slowThresholdMs).sort((left, right) => left.order - right.order);
523
- for (const [index, activeCase] of slowCases.entries()) {
524
- const icon = index === slowCases.length - 1 ? TREE_NODE_END : TREE_NODE_MIDDLE;
525
- const elapsed = Math.max(0, this.options.getNow() - activeCase.startedAt);
526
- rows.push(c.bold(c.yellow(` ${icon} `)) + activeCase.caseName + c.bold(c.yellow(` ${formatDuration$1(elapsed)}`)));
527
- }
528
- }
529
- return rows;
530
- }
531
- createFooterRows() {
532
- return [
533
- padSummaryTitle("Tasks") + formatCounterState(this.taskCounters),
534
- padSummaryTitle("Cases") + formatCounterState(this.caseCounters),
535
- padSummaryTitle("Start at") + this.startTime,
536
- padSummaryTitle("Duration") + formatDuration$1(Math.max(0, this.options.getNow() - this.startedAtMs))
537
- ];
538
- }
539
- getOrCreateTaskState(taskId) {
540
- const existing = this.tasks.get(taskId);
541
- if (existing != null) return existing;
542
- const created = {
543
- caseOrderCounter: 0,
544
- completedCases: 0,
545
- displayName: taskId,
546
- projectName: void 0,
547
- queueOrder: this.queueOrderCounter,
548
- runningCases: /* @__PURE__ */ new Map(),
549
- settledCaseIds: /* @__PURE__ */ new Set(),
550
- startedAt: void 0,
551
- state: "queued",
552
- taskId,
553
- taskResult: void 0,
554
- totalCases: 0
555
- };
556
- this.queueOrderCounter += 1;
557
- this.tasks.set(taskId, created);
558
- return created;
559
- }
560
- syncTaskTotalCases(task, reportedTotalCases) {
561
- const observedTotalCases = task.completedCases + task.runningCases.size;
562
- task.totalCases = Math.max(task.totalCases, reportedTotalCases ?? 0, observedTotalCases);
563
- this.caseCounters.total = sumTaskCaseTotals(this.tasks.values());
564
- }
565
- };
566
- /**
567
- * Creates the live summary reporter state machine for `vieval` CLI runs.
568
- *
569
- * Use when:
570
- * - the CLI wants Vitest-style active rows and live counters
571
- * - tests need a deterministic reporter surface without touching the terminal
572
- *
573
- * Expects:
574
- * - queue/start/end events describe task lifecycle in order
575
- * - `getNow()` remains monotonic within one run
576
- * - `getWallClockNow()` returns the wall-clock run start timestamp
577
- *
578
- * Returns:
579
- * - a reporter compatible with the base CLI lifecycle plus `getWindowRows()`
580
- *
581
- * Call stack:
582
- *
583
- * {@link createSummaryReporter}
584
- * -> {@link SummaryReporterStateMachine.onTaskQueued}
585
- * -> {@link SummaryReporterStateMachine.onCaseStart}
586
- * -> {@link SummaryReporterStateMachine.getWindowRows}
587
- */
588
- function createSummaryReporter(options) {
589
- return new SummaryReporterStateMachine(options);
590
- }
591
- function createCounterState() {
592
- return {
593
- completed: 0,
594
- failed: 0,
595
- passed: 0,
596
- skipped: 0,
597
- total: 0
598
- };
599
- }
600
- function resetCounterState(counter, total) {
601
- counter.completed = 0;
602
- counter.failed = 0;
603
- counter.passed = 0;
604
- counter.skipped = 0;
605
- counter.total = total;
606
- }
607
- function sumTaskCaseTotals(tasks) {
608
- let total = 0;
609
- for (const task of tasks) total += task.totalCases;
610
- return total;
611
- }
612
- function compareActiveTasks(left, right) {
613
- const leftProject = left.projectName ?? "";
614
- const rightProject = right.projectName ?? "";
615
- if (leftProject !== rightProject) return leftProject.localeCompare(rightProject);
616
- const displayNameOrder = left.displayName.localeCompare(right.displayName);
617
- if (displayNameOrder !== 0) return displayNameOrder;
618
- return left.queueOrder - right.queueOrder;
619
- }
620
- function padSummaryTitle(label) {
621
- return `${c.dim(label.padEnd(8))} `;
622
- }
623
- function formatCounterState(counter) {
624
- return [
625
- c.bold(c.green(`${counter.passed} passed`)),
626
- counter.failed > 0 ? c.bold(c.red(`${counter.failed} failed`)) : c.dim(`${counter.failed} failed`),
627
- counter.skipped > 0 ? c.yellow(`${counter.skipped} skipped`) : c.dim(`${counter.skipped} skipped`)
628
- ].join(c.dim(" | ")) + c.gray(` (${counter.total})`);
629
- }
630
- function formatTimeString(date) {
631
- return date.toTimeString().split(" ")[0] ?? "";
632
- }
633
- function formatDuration$1(durationMs) {
634
- if (durationMs >= 1e3) return `${(durationMs / 1e3).toFixed(2)}s`;
635
- return `${Math.round(durationMs)}ms`;
636
- }
637
- function formatProjectBadge(projectName, isTTY) {
638
- if (projectName == null || projectName.length === 0) return "";
639
- if (!isTTY || !c.isColorSupported) return `|${projectName}| `;
640
- const backgroundPool = [
641
- c.bgYellow,
642
- c.bgCyan,
643
- c.bgGreen,
644
- c.bgMagenta
645
- ];
646
- const background = backgroundPool[projectName.split("").reduce((accumulator, character, index) => accumulator + character.charCodeAt(0) + index, 0) % backgroundPool.length];
647
- return `${c.black(background(` ${projectName} `))} `;
648
- }
649
- //#endregion
650
- //#region src/cli/reporters/index.ts
651
- /**
652
- * Creates the default CLI reporter for the current output mode.
653
- *
654
- * Use when:
655
- * - interactive terminals should use the live summary reporter
656
- * - non-interactive environments should stay silent with the noop reporter
657
- *
658
- * Expects:
659
- * - `isTTY` decides whether the live summary reporter can be used
660
- *
661
- * Returns:
662
- * - a summary reporter for TTY runs, otherwise a noop reporter
663
- */
664
- function createCliReporter(options) {
665
- if (!options.isTTY) return createNoopReporter();
666
- return createSummaryReporter(options);
667
- }
668
- //#endregion
669
- //#region src/cli/reporters/renderers/windowed-renderer.ts
670
- const DEFAULT_RENDER_INTERVAL_MS = 1e3;
671
- const ESC = "\x1B[";
672
- const CARRIAGE_RETURN = "\r";
673
- const CLEAR_LINE = `${ESC}K`;
674
- const MOVE_CURSOR_ONE_ROW_UP = `${ESC}1A`;
675
- const SYNC_START = `${ESC}?2026h`;
676
- const SYNC_END = `${ESC}?2026l`;
677
- /**
678
- * Renders a dynamic window at the bottom of the terminal.
679
- *
680
- * Use when:
681
- * - a reporter needs in-place TTY updates without leaking terminal control codes into tests
682
- * - callers want Vitest-style redraw behavior with injected output/timer dependencies
683
- *
684
- * Expects:
685
- * - `start()` runs before `schedule()`
686
- * - `finish()` or `dispose()` may be called multiple times safely
687
- *
688
- * Returns:
689
- * - no direct value; all effects are emitted through the injected callbacks
690
- *
691
- * Call stack:
692
- *
693
- * {@link WindowRenderer.start}
694
- * -> periodic schedule callback
695
- * -> {@link WindowRenderer.schedule}
696
- * -> {@link WindowRenderer.renderWindow}
697
- */
698
- var WindowRenderer = class {
699
- options;
700
- renderInterval;
701
- renderScheduled = false;
702
- renderScheduleVersion = 0;
703
- windowHeight = 0;
704
- started = false;
705
- finished = false;
706
- bufferedOutput = "";
707
- constructor(options) {
708
- if (options.createInterval && options.clearInterval) {
709
- this.options = {
710
- createInterval: (callback, intervalMs) => {
711
- const timer = options.createInterval(callback, intervalMs);
712
- return {
713
- clear: () => options.clearInterval(timer),
714
- unref: timer.unref?.bind(timer)
715
- };
716
- },
717
- getColumns: options.getColumns,
718
- getWindow: options.getWindow,
719
- intervalMs: options.intervalMs ?? DEFAULT_RENDER_INTERVAL_MS,
720
- queueRenderReset: options.queueRenderReset ?? defaultQueueRenderReset,
721
- supportsAnsiWindowing: options.supportsAnsiWindowing ?? true,
722
- writeOutput: options.writeOutput
723
- };
724
- return;
725
- }
726
- this.options = {
727
- createInterval: defaultCreateInterval,
728
- getColumns: options.getColumns,
729
- getWindow: options.getWindow,
730
- intervalMs: options.intervalMs ?? DEFAULT_RENDER_INTERVAL_MS,
731
- queueRenderReset: options.queueRenderReset ?? defaultQueueRenderReset,
732
- supportsAnsiWindowing: options.supportsAnsiWindowing ?? true,
733
- writeOutput: options.writeOutput
734
- };
735
- }
736
- /**
737
- * Starts the periodic refresh loop.
738
- *
739
- * Use when:
740
- * - the live reporter is about to emit in-place updates
741
- *
742
- * Expects:
743
- * - repeated calls are harmless and keep the existing timer
744
- *
745
- * Returns:
746
- * - no direct value
747
- */
748
- start() {
749
- if (this.started && !this.finished) return;
750
- this.started = true;
751
- this.finished = false;
752
- this.renderScheduleVersion += 1;
753
- if (!this.renderInterval) {
754
- this.renderInterval = this.options.createInterval(() => this.schedule(), this.options.intervalMs);
755
- this.renderInterval.unref?.();
756
- }
757
- }
758
- /**
759
- * Queues a render if one is not already in flight.
760
- *
761
- * Use when:
762
- * - reporter state changes and the bottom window should refresh
763
- *
764
- * Expects:
765
- * - the renderer has been started
766
- *
767
- * Returns:
768
- * - no direct value
769
- */
770
- schedule() {
771
- if (!this.started || this.finished || this.renderScheduled) return;
772
- const renderScheduleVersion = this.renderScheduleVersion;
773
- this.renderScheduled = true;
774
- this.renderWindow();
775
- this.options.queueRenderReset(() => {
776
- if (this.renderScheduleVersion !== renderScheduleVersion) return;
777
- this.renderScheduled = false;
778
- });
779
- }
780
- /**
781
- * Clears the rendered window and stops the refresh loop.
782
- *
783
- * Use when:
784
- * - the live reporter is transitioning to final static output
785
- *
786
- * Expects:
787
- * - repeated calls are safe
788
- *
789
- * Returns:
790
- * - no direct value
791
- */
792
- finish() {
793
- if (this.finished) return;
794
- this.finished = true;
795
- this.started = false;
796
- this.renderScheduleVersion += 1;
797
- this.renderScheduled = false;
798
- this.stopInterval();
799
- this.clearWindow();
800
- this.flushBufferedOutput();
801
- }
802
- /**
803
- * Stops the renderer and clears any visible window state.
804
- *
805
- * Use when:
806
- * - cleanup needs to happen from a `finally` block or interrupted run
807
- *
808
- * Expects:
809
- * - callers may invoke it more than once
810
- *
811
- * Returns:
812
- * - no direct value
813
- */
814
- dispose() {
815
- this.finish();
816
- }
817
- /**
818
- * Alias for disposal to match Vitest's renderer lifecycle naming.
819
- *
820
- * Use when:
821
- * - adapting code that expects `stop()`
822
- *
823
- * Expects:
824
- * - callers want the same semantics as `dispose()`
825
- *
826
- * Returns:
827
- * - no direct value
828
- */
829
- stop() {
830
- this.dispose();
831
- }
832
- /**
833
- * Writes reporter output through the renderer lifecycle.
834
- *
835
- * Use when:
836
- * - emitting log lines that must appear above the live ANSI window
837
- * - callers need deterministic buffering behavior in tests
838
- *
839
- * Expects:
840
- * - active ANSI window mode buffers until `schedule()` or `finish()`
841
- * - inactive or non-windowed mode writes directly
842
- *
843
- * Returns:
844
- * - no direct value
845
- */
846
- write(message) {
847
- if (!this.isActiveWindowMode()) {
848
- this.writeOutput(message);
849
- return;
850
- }
851
- this.bufferedOutput += message;
852
- }
853
- renderWindow() {
854
- const windowContent = this.options.getWindow();
855
- const rowCount = getRenderedRowCount(windowContent, this.options.getColumns());
856
- if (this.options.supportsAnsiWindowing) {
857
- this.writeOutput(SYNC_START);
858
- this.clearWindow();
859
- }
860
- this.flushBufferedOutput();
861
- this.writeOutput(windowContent.join("\n"));
862
- if (this.options.supportsAnsiWindowing) {
863
- this.writeOutput(SYNC_END);
864
- this.windowHeight = rowCount;
865
- return;
866
- }
867
- this.writeOutput("\n");
868
- this.windowHeight = 0;
869
- }
870
- clearWindow() {
871
- if (!this.options.supportsAnsiWindowing || this.windowHeight === 0) return;
872
- this.writeOutput(`${CARRIAGE_RETURN}${CLEAR_LINE}`);
873
- for (let rowIndex = 1; rowIndex < this.windowHeight; rowIndex += 1) this.writeOutput(`${CARRIAGE_RETURN}${MOVE_CURSOR_ONE_ROW_UP}${CLEAR_LINE}`);
874
- this.windowHeight = 0;
875
- }
876
- stopInterval() {
877
- if (!this.renderInterval) return;
878
- this.renderInterval.clear();
879
- this.renderInterval = void 0;
880
- }
881
- writeOutput(message) {
882
- this.options.writeOutput(message);
883
- }
884
- flushBufferedOutput() {
885
- if (this.bufferedOutput.length === 0) return;
886
- this.writeOutput(this.bufferedOutput);
887
- this.bufferedOutput = "";
888
- }
889
- isActiveWindowMode() {
890
- return this.started && !this.finished && this.options.supportsAnsiWindowing;
891
- }
892
- };
893
- function defaultCreateInterval(callback, intervalMs) {
894
- const timer = globalThis.setInterval(callback, intervalMs);
895
- return {
896
- clear: () => globalThis.clearInterval(timer),
897
- unref: timer.unref?.bind(timer)
898
- };
899
- }
900
- function defaultQueueRenderReset(callback) {
901
- setTimeout(callback, 100).unref();
902
- }
903
- /** Calculate the rendered row count for the supplied rows and terminal width. */
904
- function getRenderedRowCount(rows, columns) {
905
- const safeColumns = Math.max(1, columns);
906
- let count = 0;
907
- for (const row of rows) {
908
- const text = stripVTControlCharacters(row);
909
- count += Math.max(1, Math.ceil(getTextDisplayWidth(text) / safeColumns));
910
- }
911
- return count;
912
- }
913
- function getTextDisplayWidth(text) {
914
- return stringWidth(stripVTControlCharacters(text));
915
- }
916
- //#endregion
917
- //#region src/cli/reporters/vitest-compat-reporter.ts
918
- function isReporterReferenceTuple(reference) {
919
- return Array.isArray(reference);
920
- }
921
- function isAbsoluteLikePath(value) {
922
- return value.startsWith("/") || value.startsWith("./") || value.startsWith("../") || /^[A-Z]:[\\/]/i.test(value);
923
- }
924
- async function loadReporterModule(path) {
925
- if (isAbsoluteLikePath(path)) return import(pathToFileURL(path).href);
926
- return import(path);
927
- }
928
- function normalizeReporterReference(reference) {
929
- if (isReporterReferenceTuple(reference)) return {
930
- options: reference[1],
931
- value: reference[0]
932
- };
933
- return {
934
- options: void 0,
935
- value: reference
936
- };
937
- }
938
- function createReporterInstance(moduleValue, options) {
939
- const value = moduleValue.default ?? moduleValue;
940
- if (value == null) return null;
941
- if (typeof value === "function") return new value(options);
942
- if (typeof value === "object") return value;
943
- return null;
944
- }
945
- async function emitToReporters(reporters, callback) {
946
- await Promise.all(reporters.map(async (reporter) => {
947
- try {
948
- await callback(reporter);
949
- } catch {}
950
- }));
951
- }
952
- /**
953
- * Creates a project-level vitest-compatible reporter bridge.
954
- *
955
- * Use when:
956
- * - `vieval` should reuse vitest-like reporter callbacks without changing CLI output contracts
957
- *
958
- * Expects:
959
- * - references point to modules whose default export is a reporter instance or constructor
960
- *
961
- * Returns:
962
- * - `null` when no reporter references are configured
963
- */
964
- async function createVievalVitestCompatReporterBridge(options) {
965
- if (options.references.length === 0) return null;
966
- const loadedReporters = [];
967
- for (const reference of options.references) {
968
- const normalized = normalizeReporterReference(reference);
969
- try {
970
- const instance = createReporterInstance(typeof normalized.value === "string" ? await loadReporterModule(normalized.value) : normalized.value, normalized.options);
971
- if (instance != null) loadedReporters.push(instance);
972
- } catch {}
973
- }
974
- if (loadedReporters.length === 0) return null;
975
- const modulesByTaskId = /* @__PURE__ */ new Map();
976
- const casesByCompositeId = /* @__PURE__ */ new Map();
977
- function getOrCreateModule(taskId) {
978
- const existing = modulesByTaskId.get(taskId);
979
- if (existing != null) return existing;
980
- const created = {
981
- id: taskId,
982
- name: taskId,
983
- projectName: options.projectName
984
- };
985
- modulesByTaskId.set(taskId, created);
986
- return created;
987
- }
988
- function getOrCreateCase(taskId, caseId) {
989
- const compositeId = `${taskId}::${caseId}`;
990
- const existing = casesByCompositeId.get(compositeId);
991
- if (existing != null) return existing;
992
- const created = {
993
- id: caseId,
994
- module: getOrCreateModule(taskId),
995
- name: caseId,
996
- state: "pending"
997
- };
998
- casesByCompositeId.set(compositeId, created);
999
- return created;
1000
- }
1001
- return {
1002
- async onCaseEnd(payload) {
1003
- const taskCase = getOrCreateCase(payload.taskId, payload.caseId);
1004
- taskCase.state = payload.state;
1005
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestCaseResult?.(taskCase));
1006
- },
1007
- async onCaseStart(payload) {
1008
- const taskCase = getOrCreateCase(payload.taskId, payload.caseId);
1009
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestCaseReady?.(taskCase));
1010
- },
1011
- async onRunEnd(run) {
1012
- const modules = [...modulesByTaskId.values()];
1013
- const errors = run.failed ? [{ message: "vieval run failed" }] : [];
1014
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestRunEnd?.(modules, errors, run.failed ? "failed" : "passed"));
1015
- },
1016
- async onRunStart() {
1017
- const specifications = [...modulesByTaskId.values()].map((module) => ({
1018
- moduleId: module.id,
1019
- projectName: module.projectName
1020
- }));
1021
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestRunStart?.(specifications));
1022
- },
1023
- async onTaskEnd(payload) {
1024
- const module = getOrCreateModule(payload.taskId);
1025
- if (payload.state === "failed") {
1026
- const syntheticCase = getOrCreateCase(payload.taskId, `${payload.taskId}:task`);
1027
- syntheticCase.state = "failed";
1028
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestCaseResult?.(syntheticCase));
1029
- }
1030
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestModuleEnd?.(module));
1031
- },
1032
- async onTaskQueued(payload) {
1033
- const module = getOrCreateModule(payload.taskId);
1034
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestModuleQueued?.(module));
1035
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestModuleCollected?.(module));
1036
- },
1037
- async onTaskStart(payload) {
1038
- const module = getOrCreateModule(payload.taskId);
1039
- await emitToReporters(loadedReporters, (reporter) => reporter.onTestModuleStart?.(module));
1040
- }
1041
- };
1042
- }
1043
- //#endregion
1044
- //#region src/cli/run.ts
1045
- /**
1046
- * Returns true when output contains at least one failing project/task/case outcome.
1047
- */
1048
- function hasRunFailures(output) {
1049
- return output.projects.some((project) => {
1050
- if (project.errorMessage != null) return true;
1051
- if (project.caseSummary != null && project.caseSummary.failed > 0) return true;
1052
- return (project.caseFailures?.length ?? 0) > 0;
1053
- });
1054
- }
1055
- function shouldUseColor() {
1056
- if (process.env.NO_COLOR != null) return false;
1057
- const forceColor = process.env.FORCE_COLOR;
1058
- if (forceColor != null) return forceColor !== "0";
1059
- return process.stdout.isTTY === true;
1060
- }
1061
- function createColorPalette(enabled) {
1062
- if (!enabled) return {
1063
- black: (value) => value,
1064
- bgCyan: (value) => value,
1065
- bgGreen: (value) => value,
1066
- bgMagenta: (value) => value,
1067
- bgYellow: (value) => value,
1068
- dim: (value) => value,
1069
- gray: (value) => value,
1070
- green: (value) => value,
1071
- red: (value) => value,
1072
- yellow: (value) => value
1073
- };
1074
- return {
1075
- black: (value) => c.black(value),
1076
- bgCyan: (value) => c.bgCyan(value),
1077
- bgGreen: (value) => c.bgGreen(value),
1078
- bgMagenta: (value) => c.bgMagenta(value),
1079
- bgYellow: (value) => c.bgYellow(value),
1080
- dim: (value) => c.dim(value),
1081
- gray: (value) => c.gray(value),
1082
- green: (value) => c.green(value),
1083
- red: (value) => c.red(value),
1084
- yellow: (value) => c.yellow(value)
1085
- };
1086
- }
1087
- function createProjectBadge(name, colors, colorEnabled) {
1088
- if (!colorEnabled || !c.isColorSupported) return `|${name}| `;
1089
- const labelColorPool = [
1090
- colors.bgYellow,
1091
- colors.bgCyan,
1092
- colors.bgGreen,
1093
- colors.bgMagenta
1094
- ];
1095
- const background = labelColorPool[name.split("").reduce((accumulator, char, index) => accumulator + char.charCodeAt(0) + index, 0) % labelColorPool.length];
1096
- return `${colors.black(background(` ${name} `))} `;
1097
- }
1098
- function formatDuration(durationMs, colors) {
1099
- if (durationMs == null) return "";
1100
- const rounded = Math.round(durationMs);
1101
- return (rounded > 1e3 ? colors.yellow : colors.green)(` ${rounded}${colors.dim("ms")}`);
1102
- }
1103
- function filterProjectsByName(projects, names) {
1104
- if (names.length === 0) return [...projects];
1105
- const nameSet = new Set(names);
1106
- return projects.filter((project) => nameSet.has(project.name));
1107
- }
1108
- function sanitizeIdentitySegment(value) {
1109
- const normalized = value.trim();
1110
- if (normalized.length === 0) return "default";
1111
- return normalized.replace(/[^\w.-]+/g, "-");
1112
- }
1113
- function createRunIdentity(options) {
1114
- const workspaceId = sanitizeIdentitySegment(options.workspace ?? "default-workspace");
1115
- const experimentId = sanitizeIdentitySegment(options.experiment ?? "default-experiment");
1116
- return {
1117
- attemptId: sanitizeIdentitySegment(options.attempt ?? `attempt-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`),
1118
- experimentId,
1119
- runId: `run-${Date.now()}-${randomUUID().slice(0, 8)}`,
1120
- workspaceId
1121
- };
1122
- }
1123
- function deriveReportProjectId(output) {
1124
- const uniqueProjectNames = [...new Set(output.projects.map((project) => project.name))];
1125
- if (uniqueProjectNames.length === 1) return sanitizeIdentitySegment(uniqueProjectNames[0] ?? "default-project");
1126
- return "multi-project";
1127
- }
1128
- function createEventRecorder(identity) {
1129
- const events = [];
1130
- const taskProjectMap = /* @__PURE__ */ new Map();
1131
- return {
1132
- events,
1133
- record(event, payload, metadata) {
1134
- const maybeTaskPayload = payload;
1135
- const taskId = metadata?.taskId ?? maybeTaskPayload?.taskId;
1136
- const caseId = metadata?.caseId ?? payload?.caseId;
1137
- const projectName = metadata?.projectName ?? maybeTaskPayload?.projectName;
1138
- if (taskId != null && projectName != null) taskProjectMap.set(taskId, projectName);
1139
- events.push({
1140
- attemptId: identity.attemptId,
1141
- caseId,
1142
- data: payload,
1143
- event,
1144
- experimentId: identity.experimentId,
1145
- projectId: taskId == null ? void 0 : taskProjectMap.get(taskId),
1146
- runId: identity.runId,
1147
- schemaVersion: 1,
1148
- taskId,
1149
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
1150
- version: 1,
1151
- workspaceId: identity.workspaceId
1152
- });
1153
- }
1154
- };
1155
- }
1156
- function createReporterWithEventCapture(reporter, recordEvent) {
1157
- return {
1158
- dispose() {
1159
- reporter.dispose();
1160
- },
1161
- onCaseEnd(payload) {
1162
- recordEvent("CaseEnded", payload);
1163
- reporter.onCaseEnd(payload);
1164
- },
1165
- onCaseStart(payload) {
1166
- recordEvent("CaseStarted", payload);
1167
- reporter.onCaseStart(payload);
1168
- },
1169
- onRunEnd(payload) {
1170
- recordEvent("RunEnded", payload);
1171
- reporter.onRunEnd(payload);
1172
- },
1173
- onRunStart(payload) {
1174
- recordEvent("RunStarted", payload);
1175
- reporter.onRunStart(payload);
1176
- },
1177
- onTaskEnd(payload) {
1178
- recordEvent("TaskEnded", payload);
1179
- reporter.onTaskEnd(payload);
1180
- },
1181
- onTaskQueued(payload) {
1182
- recordEvent("TaskQueued", payload);
1183
- reporter.onTaskQueued(payload);
1184
- },
1185
- onTaskStart(payload) {
1186
- recordEvent("TaskStarted", payload);
1187
- reporter.onTaskStart(payload);
1188
- }
1189
- };
1190
- }
1191
- function applyRunEnvironment(env) {
1192
- const envEntries = Object.entries(env);
1193
- if (envEntries.length === 0) return () => {};
1194
- const snapshot = /* @__PURE__ */ new Map();
1195
- for (const [key, value] of envEntries) {
1196
- snapshot.set(key, {
1197
- existed: Object.hasOwn(process.env, key),
1198
- value: process.env[key]
1199
- });
1200
- if (value == null) {
1201
- delete process.env[key];
1202
- continue;
1203
- }
1204
- process.env[key] = value;
1205
- }
1206
- return () => {
1207
- for (const [key, previous] of snapshot.entries()) {
1208
- if (previous.existed) {
1209
- if (previous.value == null) {
1210
- delete process.env[key];
1211
- continue;
1212
- }
1213
- process.env[key] = previous.value;
1214
- continue;
1215
- }
1216
- delete process.env[key];
1217
- }
1218
- };
1219
- }
1220
- function isSummaryReporter(reporter) {
1221
- return "getWindowRows" in reporter;
1222
- }
1223
- function createRunReporter(options) {
1224
- const reporter = createCliReporter({
1225
- getColumns: options?.getColumns ?? (() => process.stdout.columns ?? 80),
1226
- getNow: options?.getNow ?? (() => Date.now()),
1227
- getWallClockNow: options?.getWallClockNow ?? (() => Date.now()),
1228
- isTTY: options?.isTTY ?? process.stdout.isTTY === true,
1229
- slowThresholdMs: options?.slowThresholdMs ?? 300,
1230
- writeError: options?.writeError ?? ((value) => process.stderr.write(value)),
1231
- writeOutput: options?.writeOutput ?? ((value) => process.stdout.write(value))
1232
- });
1233
- if (!isSummaryReporter(reporter)) return {
1234
- ...reporter,
1235
- onCaseStart(payload) {
1236
- reporter.onCaseStart(payload);
1237
- },
1238
- onTaskQueued(payload) {
1239
- reporter.onTaskQueued(payload);
1240
- }
1241
- };
1242
- const rendererBaseOptions = {
1243
- getColumns: options?.getColumns ?? (() => process.stdout.columns ?? 80),
1244
- getWindow: () => reporter.getWindowRows(),
1245
- queueRenderReset: options?.queueRenderReset,
1246
- supportsAnsiWindowing: options?.supportsAnsiWindowing,
1247
- writeOutput: options?.writeOutput ?? ((value) => process.stdout.write(value))
1248
- };
1249
- const renderer = options?.clearInterval != null && options.createInterval != null ? new WindowRenderer({
1250
- ...rendererBaseOptions,
1251
- clearInterval: options.clearInterval,
1252
- createInterval: options.createInterval
1253
- }) : new WindowRenderer(rendererBaseOptions);
1254
- renderer.start();
1255
- function scheduleRender() {
1256
- renderer.schedule();
1257
- }
1258
- return {
1259
- dispose() {
1260
- reporter.dispose();
1261
- renderer.dispose();
1262
- },
1263
- onCaseEnd(payload) {
1264
- reporter.onCaseEnd(payload);
1265
- scheduleRender();
1266
- },
1267
- onCaseStart(payload) {
1268
- reporter.onCaseStart(payload);
1269
- scheduleRender();
1270
- },
1271
- onRunEnd(payload) {
1272
- reporter.onRunEnd(payload);
1273
- scheduleRender();
1274
- },
1275
- onRunStart(payload) {
1276
- reporter.onRunStart(payload);
1277
- scheduleRender();
1278
- },
1279
- onTaskEnd(payload) {
1280
- reporter.onTaskEnd(payload);
1281
- scheduleRender();
1282
- },
1283
- onTaskQueued(payload) {
1284
- reporter.onTaskQueued(payload);
1285
- scheduleRender();
1286
- },
1287
- onTaskStart(payload) {
1288
- reporter.onTaskStart(payload);
1289
- scheduleRender();
1290
- }
1291
- };
1292
- }
1293
- function createTaskQueuePayload(task, projectName) {
1294
- return {
1295
- displayName: task.entry.name,
1296
- projectName,
1297
- taskId: task.id
1298
- };
1299
- }
1300
- function createTaskCaseReporterId(payload) {
1301
- return `${payload.index}:${encodeURIComponent(payload.name)}`;
1302
- }
1303
- function createTaskReporterHooks(task, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter) {
1304
- function syncCaseTotal(total) {
1305
- reporter.onTaskQueued({
1306
- taskId: task.id,
1307
- totalCases: total
1308
- });
1309
- }
1310
- return {
1311
- onCaseEnd(payload) {
1312
- const caseId = createTaskCaseReporterId(payload);
1313
- if (projectCaseCounters != null) {
1314
- const projectCaseId = `${task.id}:${caseId}`;
1315
- if (!projectCaseCounters.seenCaseIds.has(projectCaseId)) {
1316
- projectCaseCounters.seenCaseIds.add(projectCaseId);
1317
- if (payload.state === "passed") projectCaseCounters.passed += 1;
1318
- else if (payload.state === "failed") projectCaseCounters.failed += 1;
1319
- else projectCaseCounters.skipped += 1;
1320
- }
1321
- }
1322
- syncCaseTotal(payload.total);
1323
- if (payload.state === "failed" && payload.errorMessage != null && projectCaseFailures != null) projectCaseFailures.push({
1324
- caseId,
1325
- caseName: payload.name,
1326
- errorMessage: payload.errorMessage,
1327
- taskId: task.id
1328
- });
1329
- reporter.onCaseEnd({
1330
- caseId,
1331
- errorMessage: payload.errorMessage,
1332
- state: payload.state,
1333
- taskId: task.id
1334
- });
1335
- vitestCompatReporter?.onCaseEnd({
1336
- caseId,
1337
- errorMessage: payload.errorMessage,
1338
- state: payload.state,
1339
- taskId: task.id
1340
- });
1341
- },
1342
- onCaseStart(payload) {
1343
- const caseId = createTaskCaseReporterId(payload);
1344
- syncCaseTotal(payload.total);
1345
- reporter.onCaseStart({
1346
- caseId,
1347
- caseName: payload.name,
1348
- taskId: task.id
1349
- });
1350
- vitestCompatReporter?.onCaseStart({
1351
- caseId,
1352
- taskId: task.id
1353
- });
1354
- },
1355
- onEvent(payload) {
1356
- recordEvent(payload.event, payload.data, {
1357
- caseId: payload.caseId,
1358
- projectName,
1359
- taskId: task.id
1360
- });
1361
- }
1362
- };
1363
- }
1364
- function createCliTaskExecutionContext(task, models, cacheRootDirectory, cacheProjectName, workspaceId, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter) {
1365
- return {
1366
- ...createTaskExecutionContext({
1367
- cache: createFilesystemTaskCacheRuntime({
1368
- cacheRootDirectory,
1369
- projectName: cacheProjectName,
1370
- workspaceId
1371
- }),
1372
- models,
1373
- task
1374
- }),
1375
- reporterHooks: createTaskReporterHooks(task, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter)
1376
- };
1377
- }
1378
- function resolveTaskReporterHooks(task, context, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter) {
1379
- return context.reporterHooks ?? createTaskReporterHooks(task, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter);
1380
- }
1381
- function getFailedTaskId(error) {
1382
- if (error instanceof RunnerExecutionError) return error.taskId;
1383
- return null;
1384
- }
1385
- function createAutoTaskExecutor(reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter) {
1386
- return async (task, context) => {
1387
- const taskDefinition = task.entry.task;
1388
- if (taskDefinition == null) throw new Error(`Missing eval task definition for entry "${task.entry.id}".`);
1389
- const output = await taskDefinition.run({
1390
- cache: context.cache,
1391
- model: context.model,
1392
- reporterHooks: resolveTaskReporterHooks(task, context, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter),
1393
- task
1394
- });
1395
- return {
1396
- entryId: task.entry.id,
1397
- id: task.id,
1398
- matrix: task.matrix,
1399
- inferenceExecutorId: task.inferenceExecutor.id,
1400
- scores: [...output.scores]
1401
- };
1402
- };
1403
- }
1404
- function cloneScheduledTaskMatrix(task) {
1405
- return {
1406
- eval: { ...task.matrix.eval },
1407
- meta: { ...task.matrix.meta },
1408
- run: { ...task.matrix.run }
1409
- };
1410
- }
1411
- function createProjectMatrixSummary(tasks) {
1412
- if (tasks.length === 0) return null;
1413
- const runAxes = /* @__PURE__ */ new Set();
1414
- const evalAxes = /* @__PURE__ */ new Set();
1415
- const runRows = /* @__PURE__ */ new Set();
1416
- const evalRows = /* @__PURE__ */ new Set();
1417
- for (const task of tasks) {
1418
- Object.keys(task.matrix.run).forEach((axis) => runAxes.add(axis));
1419
- Object.keys(task.matrix.eval).forEach((axis) => evalAxes.add(axis));
1420
- runRows.add(task.matrix.meta.runRowId);
1421
- evalRows.add(task.matrix.meta.evalRowId);
1422
- }
1423
- return {
1424
- evalAxes: [...evalAxes].sort(),
1425
- evalRows: evalRows.size,
1426
- runAxes: [...runAxes].sort(),
1427
- runRows: runRows.size
1428
- };
1429
- }
1430
- async function prepareProject(project) {
1431
- const startedAt = Date.now();
1432
- try {
1433
- const runtimeContext = await createRunnerRuntimeContext({
1434
- cwd: project.root,
1435
- fallbackProjectRootDirectory: project.root
1436
- });
1437
- const evalFilePaths = await discoverEvalFiles({
1438
- exclude: project.exclude,
1439
- include: project.include,
1440
- root: project.root
1441
- });
1442
- const entries = collectEvalEntries(await loadEvalModulesWithVitestRuntime(evalFilePaths, project.root), runtimeContext);
1443
- const tasks = createRunnerSchedule({
1444
- evalMatrix: project.evalMatrix,
1445
- entries,
1446
- inferenceExecutors: project.inferenceExecutors,
1447
- runMatrix: project.runMatrix
1448
- });
1449
- const canAutoExecuteEntryTasks = entries.some((entry) => entry.task != null) && project.models.length > 0;
1450
- if (project.executor == null && !canAutoExecuteEntryTasks) return {
1451
- kind: "summary",
1452
- summary: {
1453
- caseSummary: null,
1454
- caseFailures: [],
1455
- discoveredEvalFileCount: evalFilePaths.length,
1456
- durationMs: Date.now() - startedAt,
1457
- entryCount: entries.length,
1458
- errorMessage: null,
1459
- executed: false,
1460
- matrixSummary: createProjectMatrixSummary(tasks),
1461
- name: project.name,
1462
- result: null,
1463
- taskCount: tasks.length
1464
- }
1465
- };
1466
- return {
1467
- kind: "prepared",
1468
- prepared: {
1469
- discoveredEvalFileCount: evalFilePaths.length,
1470
- entryCount: entries.length,
1471
- name: project.name,
1472
- project,
1473
- startedAt,
1474
- tasks
1475
- }
1476
- };
1477
- } catch (error) {
1478
- return {
1479
- kind: "summary",
1480
- summary: {
1481
- caseSummary: null,
1482
- caseFailures: [],
1483
- discoveredEvalFileCount: 0,
1484
- durationMs: Date.now() - startedAt,
1485
- entryCount: 0,
1486
- errorMessage: errorMessageFrom(error) ?? "Unknown project execution error.",
1487
- executed: false,
1488
- matrixSummary: null,
1489
- name: project.name,
1490
- result: null,
1491
- taskCount: 0
1492
- }
1493
- };
1494
- }
1495
- }
1496
- async function executePreparedProject(prepared, identity, cacheProjectName, reporter, counters, recordEvent) {
1497
- const settledTaskIds = /* @__PURE__ */ new Set();
1498
- const projectCaseCounters = {
1499
- failed: 0,
1500
- passed: 0,
1501
- seenCaseIds: /* @__PURE__ */ new Set(),
1502
- skipped: 0
1503
- };
1504
- const projectCaseFailures = [];
1505
- const vitestCompatReporter = await createVievalVitestCompatReporterBridge({
1506
- projectName: prepared.name,
1507
- references: prepared.project.reporters
1508
- });
1509
- const rawTaskExecutor = prepared.project.executor ?? createAutoTaskExecutor(reporter, prepared.name, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter);
1510
- const taskExecutor = async (task, context) => {
1511
- return {
1512
- ...await rawTaskExecutor(task, context),
1513
- matrix: cloneScheduledTaskMatrix(task)
1514
- };
1515
- };
1516
- for (const task of prepared.tasks) await vitestCompatReporter?.onTaskQueued({ taskId: task.id });
1517
- await vitestCompatReporter?.onRunStart();
1518
- try {
1519
- const aggregated = await runScheduledTasks(prepared.tasks, taskExecutor, {
1520
- createExecutionContext(task) {
1521
- return createCliTaskExecutionContext(task, prepared.project.models, resolve(prepared.project.root, ".vieval", "cache"), cacheProjectName ?? prepared.name, identity.workspaceId, reporter, prepared.name, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter);
1522
- },
1523
- onTaskEnd(task, state) {
1524
- settledTaskIds.add(task.id);
1525
- reporter.onTaskEnd({
1526
- state,
1527
- taskId: task.id
1528
- });
1529
- vitestCompatReporter?.onTaskEnd({
1530
- state,
1531
- taskId: task.id
1532
- });
1533
- if (state === "passed") {
1534
- counters.passedTasks += 1;
1535
- return;
1536
- }
1537
- counters.failedTasks += 1;
1538
- },
1539
- onTaskStart(task) {
1540
- reporter.onTaskStart({ taskId: task.id });
1541
- vitestCompatReporter?.onTaskStart({ taskId: task.id });
1542
- }
1543
- });
1544
- await vitestCompatReporter?.onRunEnd({ failed: false });
1545
- return {
1546
- caseSummary: {
1547
- failed: projectCaseCounters.failed,
1548
- passed: projectCaseCounters.passed,
1549
- skipped: projectCaseCounters.skipped,
1550
- total: projectCaseCounters.seenCaseIds.size
1551
- },
1552
- caseFailures: projectCaseFailures,
1553
- discoveredEvalFileCount: prepared.discoveredEvalFileCount,
1554
- durationMs: Date.now() - prepared.startedAt,
1555
- entryCount: prepared.entryCount,
1556
- errorMessage: null,
1557
- executed: true,
1558
- matrixSummary: createProjectMatrixSummary(prepared.tasks),
1559
- name: prepared.name,
1560
- result: aggregated,
1561
- taskCount: prepared.tasks.length
1562
- };
1563
- } catch (error) {
1564
- const failedTaskId = getFailedTaskId(error);
1565
- if (failedTaskId != null && !settledTaskIds.has(failedTaskId)) {
1566
- counters.failedTasks += 1;
1567
- settledTaskIds.add(failedTaskId);
1568
- reporter.onTaskEnd({
1569
- state: "failed",
1570
- taskId: failedTaskId
1571
- });
1572
- await vitestCompatReporter?.onTaskEnd({
1573
- state: "failed",
1574
- taskId: failedTaskId
1575
- });
1576
- }
1577
- for (const task of prepared.tasks) {
1578
- if (settledTaskIds.has(task.id)) continue;
1579
- counters.skippedTasks += 1;
1580
- settledTaskIds.add(task.id);
1581
- reporter.onTaskEnd({
1582
- state: "skipped",
1583
- taskId: task.id
1584
- });
1585
- await vitestCompatReporter?.onTaskEnd({
1586
- state: "skipped",
1587
- taskId: task.id
1588
- });
1589
- }
1590
- await vitestCompatReporter?.onRunEnd({ failed: true });
1591
- return {
1592
- caseSummary: {
1593
- failed: projectCaseCounters.failed,
1594
- passed: projectCaseCounters.passed,
1595
- skipped: projectCaseCounters.skipped,
1596
- total: projectCaseCounters.seenCaseIds.size
1597
- },
1598
- caseFailures: projectCaseFailures,
1599
- discoveredEvalFileCount: prepared.discoveredEvalFileCount,
1600
- durationMs: Date.now() - prepared.startedAt,
1601
- entryCount: prepared.entryCount,
1602
- errorMessage: errorMessageFrom(error) ?? "Unknown project execution error.",
1603
- executed: false,
1604
- matrixSummary: createProjectMatrixSummary(prepared.tasks),
1605
- name: prepared.name,
1606
- result: null,
1607
- taskCount: prepared.tasks.length
1608
- };
1609
- }
1610
- }
1611
- async function writeRunReportArtifacts(output, events, identity, reportOut) {
1612
- const projectId = deriveReportProjectId(output);
1613
- const reportDirectory = resolve(reportOut, identity.workspaceId, projectId, identity.experimentId, identity.attemptId, identity.runId);
1614
- await mkdir(reportDirectory, { recursive: true });
1615
- await writeFile(resolve(reportDirectory, "run-summary.json"), `${JSON.stringify(output, null, 2)}\n`, "utf-8");
1616
- await writeFile(resolve(reportDirectory, "events.jsonl"), events.map((event) => JSON.stringify(event)).join("\n").concat(events.length > 0 ? "\n" : ""), "utf-8");
1617
- return reportDirectory;
1618
- }
1619
- /**
1620
- * Runs vieval orchestration from config and returns project-level summaries.
1621
- *
1622
- * Call stack:
1623
- *
1624
- * {@link runVievalCli}
1625
- * -> {@link loadVievalCliConfig}
1626
- * -> {@link discoverEvalFiles}
1627
- * -> {@link collectEvalEntries}
1628
- * -> {@link createRunnerSchedule}
1629
- * -> {@link runScheduledTasks} (optional)
1630
- *
1631
- * Use when:
1632
- * - running eval collection and scheduling from a single command
1633
- * - keeping business-agent eval files near their implementation packages
1634
- */
1635
- async function runVievalCli(options = {}) {
1636
- const identity = createRunIdentity(options);
1637
- const loadedConfig = await loadVievalCliConfig({
1638
- configFilePath: options.configFilePath,
1639
- cwd: options.cwd
1640
- });
1641
- const restoreEnvironment = applyRunEnvironment(loadedConfig.env);
1642
- const eventRecorder = createEventRecorder(identity);
1643
- const reporter = createReporterWithEventCapture(createRunReporter(options.reporter), eventRecorder.record);
1644
- try {
1645
- const selectedProjects = filterProjectsByName(loadedConfig.projects, options.project ?? []);
1646
- const preparedProjects = await Promise.all(selectedProjects.map(async (project) => prepareProject(project)));
1647
- const executableProjects = preparedProjects.filter((project) => project.kind === "prepared").map((project) => project.prepared);
1648
- const totalTasks = preparedProjects.reduce((sum, project) => {
1649
- if (project.kind === "prepared") return sum + project.prepared.tasks.length;
1650
- return sum + project.summary.taskCount;
1651
- }, 0);
1652
- const skippedSummaryTasks = preparedProjects.reduce((sum, project) => {
1653
- if (project.kind === "summary") return sum + project.summary.taskCount;
1654
- return sum;
1655
- }, 0);
1656
- const reporterCounters = {
1657
- failedTasks: 0,
1658
- passedTasks: 0,
1659
- skippedTasks: 0
1660
- };
1661
- reporter.onRunStart({ totalTasks });
1662
- for (const project of executableProjects) for (const task of project.tasks) reporter.onTaskQueued(createTaskQueuePayload(task, project.name));
1663
- const projectSummaries = [];
1664
- for (const preparedProject of preparedProjects) {
1665
- if (preparedProject.kind === "summary") {
1666
- projectSummaries.push(preparedProject.summary);
1667
- continue;
1668
- }
1669
- projectSummaries.push(await executePreparedProject(preparedProject.prepared, identity, options.cacheProjectName, reporter, reporterCounters, eventRecorder.record));
1670
- }
1671
- reporter.onRunEnd({
1672
- failedTasks: reporterCounters.failedTasks,
1673
- passedTasks: reporterCounters.passedTasks,
1674
- skippedTasks: reporterCounters.skippedTasks + skippedSummaryTasks,
1675
- totalTasks
1676
- });
1677
- const output = {
1678
- attemptId: identity.attemptId,
1679
- configFilePath: loadedConfig.configFilePath,
1680
- experimentId: identity.experimentId,
1681
- projects: projectSummaries,
1682
- reportDirectory: null,
1683
- runId: identity.runId,
1684
- workspaceId: identity.workspaceId
1685
- };
1686
- if (options.reportOut != null) output.reportDirectory = await writeRunReportArtifacts(output, eventRecorder.events, identity, options.reportOut);
1687
- return output;
1688
- } finally {
1689
- reporter.dispose();
1690
- restoreEnvironment();
1691
- }
1692
- }
1693
- /**
1694
- * Formats CLI run output as human-readable lines.
1695
- */
1696
- function formatVievalCliRunOutput(output) {
1697
- const colorEnabled = shouldUseColor();
1698
- const colors = createColorPalette(colorEnabled);
1699
- const lines = [];
1700
- lines.push(` ${colors.dim("RUN")} ${colors.yellow("vieval")}`);
1701
- lines.push(` ${colors.dim("Config")} ${output.configFilePath ?? "(not found, using defaults)"}`);
1702
- lines.push("");
1703
- let passedProjects = 0;
1704
- let skippedProjects = 0;
1705
- let failedProjects = 0;
1706
- let totalTasks = 0;
1707
- let executedTasks = 0;
1708
- function formatMatrixSummary(summary) {
1709
- if (summary == null) return null;
1710
- const runAxesLabel = summary.runAxes.length === 0 ? "-" : summary.runAxes.join("|");
1711
- const evalAxesLabel = summary.evalAxes.length === 0 ? "-" : summary.evalAxes.join("|");
1712
- return `matrix run ${summary.runRows} [${runAxesLabel}] / eval ${summary.evalRows} [${evalAxesLabel}]`;
1713
- }
1714
- function formatScheduleBreakdown(project) {
1715
- const summary = project.matrixSummary;
1716
- if (summary == null) return null;
1717
- if (project.taskCount <= 0 || project.entryCount <= 0 || summary.runRows <= 0 || summary.evalRows <= 0) return null;
1718
- const denominator = project.entryCount * summary.runRows * summary.evalRows;
1719
- if (denominator <= 0 || project.taskCount % denominator !== 0) return null;
1720
- const providerCount = project.taskCount / denominator;
1721
- return [
1722
- colors.dim("schedule "),
1723
- colors.yellow(String(project.entryCount)),
1724
- colors.dim(" entries × "),
1725
- colors.yellow(String(providerCount)),
1726
- colors.dim(" inferenceExecutors × "),
1727
- colors.yellow(String(summary.runRows)),
1728
- colors.dim(" run rows × "),
1729
- colors.yellow(String(summary.evalRows)),
1730
- colors.dim(" eval rows = "),
1731
- colors.green(String(project.taskCount)),
1732
- colors.dim(" tasks")
1733
- ].join("");
1734
- }
1735
- for (const project of output.projects) {
1736
- totalTasks += project.taskCount;
1737
- executedTasks += project.result?.overall.runCount ?? 0;
1738
- const badge = createProjectBadge(project.name, colors, colorEnabled);
1739
- const isFailed = project.errorMessage != null;
1740
- const hasFailedCases = (project.caseSummary?.failed ?? 0) > 0 || (project.caseFailures?.length ?? 0) > 0;
1741
- if (isFailed) {
1742
- failedProjects += 1;
1743
- lines.push(` ${colors.red("❯")} ${badge}${formatDuration(project.durationMs, colors)}`);
1744
- lines.push(` ${project.errorMessage}`);
1745
- continue;
1746
- }
1747
- if (!project.executed) {
1748
- skippedProjects += 1;
1749
- const countLabel = colors.dim(`(${project.taskCount} tasks)`);
1750
- const detailsLabel = colors.dim(` ${project.discoveredEvalFileCount} files, ${project.entryCount} entries, 0 runs, hybrid n/a`);
1751
- const matrixSummary = formatMatrixSummary(project.matrixSummary);
1752
- lines.push(` ${colors.dim("○")} ${badge}${countLabel}${detailsLabel}${formatDuration(project.durationMs, colors)}`);
1753
- if (matrixSummary != null) lines.push(` ${colors.dim(matrixSummary)}`);
1754
- const scheduleBreakdown = formatScheduleBreakdown(project);
1755
- if (scheduleBreakdown != null) lines.push(` ${scheduleBreakdown}`);
1756
- continue;
1757
- }
1758
- if (hasFailedCases) failedProjects += 1;
1759
- else passedProjects += 1;
1760
- const hybridAverage = project.result?.overall.hybridAverage;
1761
- const hybridAverageLabel = hybridAverage == null ? "n/a" : String(hybridAverage);
1762
- const runCount = project.result?.overall.runCount ?? 0;
1763
- const countLabel = colors.dim(`(${project.taskCount} tasks)`);
1764
- const caseSummaryLabel = project.caseSummary == null ? "" : `, cases ${project.caseSummary.passed} passed | ${project.caseSummary.failed} failed`;
1765
- const detailsLabel = colors.dim(` ${project.discoveredEvalFileCount} files, ${project.entryCount} entries, ${runCount} runs${caseSummaryLabel}, hybrid ${hybridAverageLabel}`);
1766
- const matrixSummary = formatMatrixSummary(project.matrixSummary);
1767
- lines.push(` ${hasFailedCases ? colors.red("❯") : colors.green("✓")} ${badge}${countLabel}${detailsLabel}${formatDuration(project.durationMs, colors)}`);
1768
- if (matrixSummary != null) lines.push(` ${colors.dim(matrixSummary)}`);
1769
- const scheduleBreakdown = formatScheduleBreakdown(project);
1770
- if (scheduleBreakdown != null) lines.push(` ${scheduleBreakdown}`);
1771
- if ((project.caseFailures?.length ?? 0) > 0) {
1772
- lines.push(` ${colors.red("Failed cases:")}`);
1773
- for (const failure of project.caseFailures.slice(0, 5)) {
1774
- lines.push(` ${colors.red(`- ${failure.caseName} (${failure.taskId})`)}`);
1775
- for (const line of failure.errorMessage.split("\n")) lines.push(` ${colors.red(line)}`);
1776
- }
1777
- if (project.caseFailures.length > 5) lines.push(` ${colors.dim(`... ${project.caseFailures.length - 5} more failed cases`)}`);
1778
- }
1779
- }
1780
- lines.push("");
1781
- if (failedProjects > 0 || skippedProjects > 0) {
1782
- const summarySegments = [`${colors.green(String(passedProjects))} passed`];
1783
- if (skippedProjects > 0) summarySegments.push(`${colors.dim(String(skippedProjects))} skipped`);
1784
- if (failedProjects > 0) summarySegments.push(`${colors.red(String(failedProjects))} failed`);
1785
- lines.push(` ${colors.dim("Projects")} ${summarySegments.join(" | ")} (${output.projects.length})`);
1786
- } else lines.push(` ${colors.dim("Projects")} ${colors.green(String(passedProjects))} passed (${output.projects.length})`);
1787
- lines.push(` ${colors.dim("Tasks")} ${executedTasks} executed / ${totalTasks} scheduled`);
1788
- return lines.join("\n");
1789
- }
1790
- //#endregion
1791
- //#region src/cli/compare.ts
1792
- const compareHelpText = `
1793
- Compare multiple methods on one benchmark.
1794
-
1795
- Usage
1796
- $ vieval compare [--config <path>] [--comparison <id>] [--output <path>] [--format <format>]
1797
-
1798
- Options
1799
- --config Config file path (default: nearest vieval.config.*)
1800
- --comparison Comparison entry id from config.comparisons
1801
- --output Optional output artifact path
1802
- --format Console output format: table | json (default: table)
1803
- `;
1804
- function normalizeCliArgv$4(argv) {
1805
- const normalizedArgv = argv[0] === "--" ? argv.slice(1) : [...argv];
1806
- if (normalizedArgv[0] === "compare") return normalizedArgv.slice(1);
1807
- return normalizedArgv;
1808
- }
1809
- function parseCompareCliArguments(argv) {
1810
- const cli = meow(compareHelpText, {
1811
- argv: normalizeCliArgv$4(argv),
1812
- flags: {
1813
- config: { type: "string" },
1814
- comparison: { type: "string" },
1815
- format: {
1816
- default: "table",
1817
- type: "string"
1818
- },
1819
- output: { type: "string" }
1820
- },
1821
- importMeta: import.meta
1822
- });
1823
- return {
1824
- comparisonId: cli.flags.comparison,
1825
- configFilePath: cli.flags.config,
1826
- format: cli.flags.format === "json" ? "json" : "table",
1827
- output: cli.flags.output
1828
- };
1829
- }
1830
- /**
1831
- * Runs one compare session from `vieval.config.*` comparison-mode config.
1832
- */
1833
- async function runCompareCli(argv) {
1834
- const parsed = parseCompareCliArguments(argv);
1835
- const loaded = await loadVievalComparisonConfig({
1836
- comparisonId: parsed.comparisonId,
1837
- configFilePath: parsed.configFilePath,
1838
- cwd: parsed.cwd
1839
- });
1840
- const methodResults = [];
1841
- for (const method of loaded.config.methods) {
1842
- const methodWorkspace = resolve(method.workspace);
1843
- const output = await runVievalCli({
1844
- cacheProjectName: loaded.config.benchmark.sharedCaseNamespace,
1845
- configFilePath: method.configFilePath ?? resolve(methodWorkspace, "vieval.config.ts"),
1846
- cwd: methodWorkspace,
1847
- project: [method.project],
1848
- workspace: loaded.config.benchmark.id
1849
- });
1850
- const failedProject = output.projects.find((project) => project.errorMessage != null);
1851
- if (failedProject != null) throw new Error(`Comparison method "${method.id}" failed: ${failedProject.errorMessage}`);
1852
- methodResults.push({
1853
- methodId: method.id,
1854
- output
1855
- });
1856
- }
1857
- const runOutput = {
1858
- benchmarkId: loaded.config.benchmark.id,
1859
- methods: methodResults
1860
- };
1861
- const artifact = buildCompareReportArtifact({
1862
- benchmarkId: runOutput.benchmarkId,
1863
- methods: runOutput.methods,
1864
- reportPath: loaded.configFilePath
1865
- });
1866
- if (parsed.output != null) await writeCompareReportArtifact({
1867
- artifact,
1868
- outputPath: parsed.output
1869
- });
1870
- if (parsed.format === "json") process.stdout.write(`${JSON.stringify(artifact, null, 2)}\n`);
1871
- else process.stdout.write([
1872
- "COMPARE vieval",
1873
- `Benchmark ${artifact.benchmarkId}`,
1874
- ...artifact.methods.map((method, index) => {
1875
- const hybrid = method.hybridAverage == null ? "n/a" : method.hybridAverage.toFixed(3);
1876
- const exact = method.exactAverage == null ? "n/a" : method.exactAverage.toFixed(3);
1877
- return `${index + 1}. ${method.methodId} hybrid=${hybrid} exact=${exact} runs=${method.runCount}`;
1878
- })
1879
- ].join("\n").concat("\n"));
1880
- return runOutput;
1881
- }
1882
- async function runCompareCliOrExit(argv) {
1883
- try {
1884
- await runCompareCli(argv);
1885
- } catch (error) {
1886
- const errorMessage = errorMessageFrom(error) ?? "Unknown compare command failure.";
1887
- process.stderr.write(`[vieval compare] ${errorMessage}\n`);
1888
- process.exitCode = 1;
1889
- }
1890
- }
1891
- //#endregion
1892
- //#region package.json
1893
- var name = "vieval";
1894
- //#endregion
1895
- //#region src/cli/eval-run.ts
1896
- const evalRunHelpText = `
1897
- Execute vieval projects from discovered or explicit config.
1898
-
1899
- Usage
1900
- $ vieval run [--config <path>] [--project <name>] [--json] [--report-out <path>]
1901
-
1902
- Options
1903
- --config Config file path
1904
- --project Project name to execute; may be repeated
1905
- --workspace Workspace id used in report artifacts
1906
- --experiment Experiment id used in report artifacts
1907
- --attempt Attempt id used in report artifacts
1908
- --report-out Report output root directory
1909
- --json Print machine-readable JSON output
1910
- `;
1911
- function normalizeCliArgv$3(argv) {
1912
- const normalizedArgv = argv[0] === "--" ? argv.slice(1) : [...argv];
1913
- return normalizedArgv[0] === "run" ? normalizedArgv.slice(1) : normalizedArgv;
1914
- }
1915
- function normalizeProjectNames(projectNames) {
1916
- if (typeof projectNames === "string") return [projectNames];
1917
- return projectNames ?? [];
1918
- }
1919
- function parseCliArguments(argv) {
1920
- const cli = meow(evalRunHelpText, {
1921
- argv: normalizeCliArgv$3(argv),
1922
- importMeta: import.meta,
1923
- flags: {
1924
- config: { type: "string" },
1925
- json: {
1926
- default: false,
1927
- type: "boolean"
1928
- },
1929
- project: {
1930
- isMultiple: true,
1931
- type: "string"
1932
- },
1933
- workspace: { type: "string" },
1934
- experiment: { type: "string" },
1935
- attempt: { type: "string" },
1936
- reportOut: { type: "string" }
1937
- }
1938
- });
1939
- return {
1940
- attempt: cli.flags.attempt,
1941
- configFilePath: cli.flags.config,
1942
- experiment: cli.flags.experiment,
1943
- json: cli.flags.json === true,
1944
- project: normalizeProjectNames(cli.flags.project),
1945
- reportOut: cli.flags.reportOut,
1946
- workspace: cli.flags.workspace
1947
- };
1948
- }
1949
- function isDirectExecution$1() {
1950
- if (!process.argv[1]) return false;
1951
- return path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
1952
- }
1953
- /**
1954
- * CLI entrypoint for `vieval run`.
1955
- *
1956
- * Call stack:
1957
- *
1958
- * {@link main}
1959
- * -> {@link parseCliArguments}(`process.argv`)
1960
- * -> {@link runVievalCli}
1961
- * -> `process.stdout.write(...)` / `process.stderr.write(...)`
1962
- * -> `process.exitCode`
1963
- *
1964
- * Use when:
1965
- * - developers want project-style eval discovery and execution from one command
1966
- * - manual `import.meta.glob` and runner wiring should stay internal
1967
- */
1968
- async function main$1() {
1969
- const parsed = parseCliArguments(process.argv.slice(2));
1970
- try {
1971
- const output = await runVievalCli({
1972
- attempt: parsed.attempt,
1973
- configFilePath: parsed.configFilePath,
1974
- experiment: parsed.experiment,
1975
- project: parsed.project,
1976
- reportOut: parsed.reportOut,
1977
- workspace: parsed.workspace
1978
- });
1979
- if (parsed.json) {
1980
- process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
1981
- if (hasRunFailures(output)) process.exitCode = 1;
1982
- return;
1983
- }
1984
- process.stdout.write(`${formatVievalCliRunOutput(output)}\n`);
1985
- if (hasRunFailures(output)) process.exitCode = 1;
1986
- } catch (error) {
1987
- const errorMessage = errorMessageFrom(error) ?? "Unknown CLI failure.";
1988
- process.stderr.write(`[${name}] ${errorMessage}\n`);
1989
- process.exitCode = 1;
1990
- }
1991
- }
1992
- if (isDirectExecution$1()) await main$1();
1993
- //#endregion
1994
- //#region src/cli/report-artifacts.ts
1995
- /**
1996
- * Resolves one or more `run-summary.json` paths from a report location.
1997
- *
1998
- * Use when:
1999
- * - callers may pass a run directory, summary file path, or a report root
2000
- *
2001
- * Returns:
2002
- * - sorted absolute summary file paths
2003
- */
2004
- async function resolveRunSummaryPaths(reportPath) {
2005
- const absoluteReportPath = resolve(reportPath);
2006
- const directSummaryPath = resolve(absoluteReportPath, "run-summary.json");
2007
- if (existsSync(absoluteReportPath) && absoluteReportPath.endsWith(".json")) return [absoluteReportPath];
2008
- if (existsSync(directSummaryPath)) return [directSummaryPath];
2009
- return (await glob("**/run-summary.json", {
2010
- absolute: true,
2011
- cwd: absoluteReportPath
2012
- })).sort((left, right) => left.localeCompare(right));
2013
- }
2014
- /**
2015
- * Reads one run report artifact set from `run-summary.json` and sibling `events.jsonl`.
2016
- *
2017
- * Use when:
2018
- * - report analysis needs both run aggregate output and event count metadata
2019
- */
2020
- function readReportRunArtifact(summaryFilePath) {
2021
- const reportDirectory = resolve(summaryFilePath, "..");
2022
- const summary = JSON.parse(readFileSync(summaryFilePath, "utf-8"));
2023
- const eventsFilePath = resolve(reportDirectory, "events.jsonl");
2024
- const events = existsSync(eventsFilePath) ? readFileSync(eventsFilePath, "utf-8").split("\n").filter((line) => line.trim().length > 0).map((line) => {
2025
- const event = JSON.parse(line);
2026
- return {
2027
- caseId: event.caseId,
2028
- data: event.data,
2029
- event: event.event,
2030
- taskId: event.taskId
2031
- };
2032
- }) : [];
2033
- return {
2034
- events,
2035
- eventsCount: events.length,
2036
- reportDirectory,
2037
- summary,
2038
- summaryFilePath
2039
- };
2040
- }
2041
- /**
2042
- * Reads all run artifacts found under `reportPath`.
2043
- *
2044
- * Use when:
2045
- * - callers need multi-run analysis from a directory root
2046
- */
2047
- async function readReportArtifacts(reportPath) {
2048
- return (await resolveRunSummaryPaths(reportPath)).map((summaryFilePath) => readReportRunArtifact(summaryFilePath));
2049
- }
2050
- /**
2051
- * Creates a compact summary row for one run artifact.
2052
- *
2053
- * Use when:
2054
- * - table/csv/jsonl exports should stay stable and cheap to parse
2055
- */
2056
- function summarizeReportRunArtifact(artifact) {
2057
- const totalProjects = artifact.summary.projects.length;
2058
- const failedProjects = artifact.summary.projects.filter((project) => project.errorMessage != null).length;
2059
- const executedProjects = artifact.summary.projects.filter((project) => project.executed).length;
2060
- const totalTasks = artifact.summary.projects.reduce((sum, project) => sum + project.taskCount, 0);
2061
- const projectNames = artifact.summary.projects.map((project) => project.name);
2062
- return {
2063
- attemptId: artifact.summary.attemptId ?? null,
2064
- eventsCount: artifact.eventsCount,
2065
- executedProjects,
2066
- experimentId: artifact.summary.experimentId ?? null,
2067
- failedProjects,
2068
- projectNames,
2069
- reportDirectory: artifact.reportDirectory,
2070
- runId: artifact.summary.runId ?? null,
2071
- totalProjects,
2072
- totalTasks,
2073
- workspaceId: artifact.summary.workspaceId ?? null
2074
- };
2075
- }
2076
- //#endregion
2077
- //#region src/cli/report-analyze.ts
2078
- const reportAnalyzeHelpText = `
2079
- Analyze generated vieval report artifacts.
2080
-
2081
- Usage
2082
- $ vieval report analyze <reportPath> [options]
2083
-
2084
- Options
2085
- --format Output format: table | json | jsonl | csv (default: table)
2086
- --workspace Workspace id filter
2087
- --project Project name filter (exact)
2088
- --experiment Experiment id filter
2089
- --attempt Attempt id filter
2090
- --run Run id filter
2091
- --task-state Keep runs containing at least one task in this state
2092
- --case-state Keep runs containing at least one case in this state
2093
- --contains Keep runs containing this text in event name or payload
2094
- --error-contains Keep runs containing this text in project errors or event payload
2095
- --run-matrix Keep runs matching run-matrix selector "key=value[,key=value]"
2096
- --eval-matrix Keep runs matching eval-matrix selector "key=value[,key=value]"
2097
- `;
2098
- function normalizeCliArgv$2(argv) {
2099
- const normalizedArgv = argv[0] === "--" ? argv.slice(1) : [...argv];
2100
- if (normalizedArgv[0] === "report" && normalizedArgv[1] === "analyze") return normalizedArgv.slice(2);
2101
- if (normalizedArgv[0] === "analyze") return normalizedArgv.slice(1);
2102
- return normalizedArgv;
2103
- }
2104
- function parseReportAnalyzeCliArguments(argv) {
2105
- const cli = meow(reportAnalyzeHelpText, {
2106
- argv: normalizeCliArgv$2(argv),
2107
- flags: {
2108
- attempt: { type: "string" },
2109
- caseState: { type: "string" },
2110
- contains: { type: "string" },
2111
- evalMatrix: { type: "string" },
2112
- errorContains: { type: "string" },
2113
- experiment: { type: "string" },
2114
- format: {
2115
- default: "table",
2116
- type: "string"
2117
- },
2118
- project: { type: "string" },
2119
- runMatrix: { type: "string" },
2120
- run: { type: "string" },
2121
- taskState: { type: "string" },
2122
- workspace: { type: "string" }
2123
- },
2124
- importMeta: import.meta
2125
- });
2126
- const reportPath = cli.input[0];
2127
- if (reportPath == null || reportPath.length === 0) throw new Error("Missing required <reportPath> argument.");
2128
- const normalizedFormat = cli.flags.format.toLowerCase();
2129
- const format = normalizedFormat === "json" ? "json" : normalizedFormat === "jsonl" ? "jsonl" : normalizedFormat === "csv" ? "csv" : "table";
2130
- return {
2131
- attempt: cli.flags.attempt,
2132
- caseState: normalizeStateFilter(cli.flags.caseState),
2133
- contains: cli.flags.contains,
2134
- evalMatrix: parseMatrixSelector(cli.flags.evalMatrix),
2135
- errorContains: cli.flags.errorContains,
2136
- experiment: cli.flags.experiment,
2137
- format,
2138
- project: cli.flags.project,
2139
- reportPath,
2140
- runMatrix: parseMatrixSelector(cli.flags.runMatrix),
2141
- run: cli.flags.run,
2142
- taskState: normalizeStateFilter(cli.flags.taskState),
2143
- workspace: cli.flags.workspace
2144
- };
2145
- }
2146
- function normalizeStateFilter(value) {
2147
- if (value == null) return;
2148
- const normalized = value.trim().toLowerCase();
2149
- if (normalized === "passed" || normalized === "failed" || normalized === "skipped") return normalized;
2150
- throw new Error(`Unsupported state filter "${value}". Expected "passed", "failed", or "skipped".`);
2151
- }
2152
- function parseMatrixSelector(value) {
2153
- if (value == null) return;
2154
- const selector = {};
2155
- const segments = value.split(",").map((segment) => segment.trim()).filter((segment) => segment.length > 0);
2156
- for (const segment of segments) {
2157
- const separatorIndex = segment.indexOf("=");
2158
- if (separatorIndex <= 0 || separatorIndex === segment.length - 1) throw new Error(`Invalid matrix selector segment "${segment}". Expected "key=value".`);
2159
- const key = segment.slice(0, separatorIndex).trim();
2160
- const parsedValue = segment.slice(separatorIndex + 1).trim();
2161
- if (key.length === 0 || parsedValue.length === 0) throw new Error(`Invalid matrix selector segment "${segment}". Expected "key=value".`);
2162
- selector[key] = parsedValue;
2163
- }
2164
- return selector;
2165
- }
2166
- function filterAnalyzeRows(rows, parsed) {
2167
- return rows.filter((row) => {
2168
- if (parsed.workspace != null && row.workspaceId !== parsed.workspace) return false;
2169
- if (parsed.experiment != null && row.experimentId !== parsed.experiment) return false;
2170
- if (parsed.attempt != null && row.attemptId !== parsed.attempt) return false;
2171
- if (parsed.run != null && row.runId !== parsed.run) return false;
2172
- if (parsed.project != null && !row.projectNames.includes(parsed.project)) return false;
2173
- return true;
2174
- });
2175
- }
2176
- function includesNeedle(value, needle) {
2177
- const normalizedNeedle = needle.trim().toLowerCase();
2178
- if (normalizedNeedle.length === 0) return true;
2179
- return JSON.stringify(value).toLowerCase().includes(normalizedNeedle);
2180
- }
2181
- function hasTaskState(artifact, targetState) {
2182
- return artifact.events.some((event) => {
2183
- if (event.event !== "TaskEnded") return false;
2184
- return event.data?.state === targetState;
2185
- });
2186
- }
2187
- function hasCaseState(artifact, targetState) {
2188
- return artifact.events.some((event) => {
2189
- if (event.event !== "CaseEnded") return false;
2190
- return event.data?.state === targetState;
2191
- });
2192
- }
2193
- function matchesMatrixSelector(matrix, selector) {
2194
- return Object.entries(selector).every(([key, expectedValue]) => String(matrix[key]) === expectedValue);
2195
- }
2196
- function hasRunMatrixMatch(artifact, selector) {
2197
- return artifact.summary.projects.some((project) => project.result?.runs.some((run) => matchesMatrixSelector(run.matrix.run, selector)) === true);
2198
- }
2199
- function hasEvalMatrixMatch(artifact, selector) {
2200
- return artifact.summary.projects.some((project) => project.result?.runs.some((run) => matchesMatrixSelector(run.matrix.eval, selector)) === true);
2201
- }
2202
- function matchesOutcomeFilters(artifact, parsed) {
2203
- if (parsed.runMatrix != null && !hasRunMatrixMatch(artifact, parsed.runMatrix)) return false;
2204
- if (parsed.evalMatrix != null && !hasEvalMatrixMatch(artifact, parsed.evalMatrix)) return false;
2205
- if (parsed.taskState != null && !hasTaskState(artifact, parsed.taskState)) return false;
2206
- if (parsed.caseState != null && !hasCaseState(artifact, parsed.caseState)) return false;
2207
- if (parsed.contains != null) {
2208
- if (!artifact.events.some((event) => includesNeedle({
2209
- data: event.data,
2210
- event: event.event
2211
- }, parsed.contains))) return false;
2212
- }
2213
- if (parsed.errorContains != null) {
2214
- if (!(artifact.summary.projects.map((project) => project.errorMessage).filter((errorMessage) => errorMessage != null).some((errorMessage) => includesNeedle(errorMessage, parsed.errorContains)) || artifact.events.some((event) => includesNeedle(event.data, parsed.errorContains)))) return false;
2215
- }
2216
- return true;
2217
- }
2218
- async function readReportAnalyzeOutput(parsed) {
2219
- const artifacts = await readReportArtifacts(parsed.reportPath);
2220
- const rows = artifacts.map((artifact) => summarizeReportRunArtifact(artifact));
2221
- const identityFilteredRows = filterAnalyzeRows(rows, parsed);
2222
- const rowByDirectory = new Map(identityFilteredRows.map((row) => [row.reportDirectory, row]));
2223
- const filteredRows = artifacts.filter((artifact) => rowByDirectory.has(artifact.reportDirectory)).filter((artifact) => matchesOutcomeFilters(artifact, parsed)).map((artifact) => rowByDirectory.get(artifact.reportDirectory)).filter((row) => row != null);
2224
- return {
2225
- experimentSummaries: buildExperimentSummaries(filteredRows),
2226
- filteredRunCount: filteredRows.length,
2227
- runs: filteredRows,
2228
- totalRunCount: rows.length
2229
- };
2230
- }
2231
- function roundMetric(value) {
2232
- return Number(value.toFixed(6));
2233
- }
2234
- function computeAverage(values) {
2235
- if (values.length === 0) return 0;
2236
- return values.reduce((sum, value) => sum + value, 0) / values.length;
2237
- }
2238
- function computeStandardDeviation(values) {
2239
- if (values.length === 0) return 0;
2240
- const average = computeAverage(values);
2241
- const variance = computeAverage(values.map((value) => (value - average) ** 2));
2242
- return Math.sqrt(variance);
2243
- }
2244
- function createExperimentGroupKey(row) {
2245
- return `${row.workspaceId ?? "unknown-workspace"}::${row.experimentId ?? "unknown-experiment"}`;
2246
- }
2247
- /**
2248
- * Builds experiment-level rollups from filtered run rows.
2249
- *
2250
- * Use when:
2251
- * - CLI consumers need stability and reliability summaries above per-run data
2252
- *
2253
- * Returns:
2254
- * - one summary row per `workspaceId + experimentId` group
2255
- */
2256
- function buildExperimentSummaries(rows) {
2257
- const grouped = /* @__PURE__ */ new Map();
2258
- for (const row of rows) {
2259
- const groupKey = createExperimentGroupKey(row);
2260
- const existing = grouped.get(groupKey);
2261
- if (existing == null) {
2262
- grouped.set(groupKey, [row]);
2263
- continue;
2264
- }
2265
- existing.push(row);
2266
- }
2267
- return [...grouped.entries()].map(([groupKey, groupRows]) => {
2268
- const [workspaceId, experimentId] = groupKey.split("::");
2269
- const failedProjects = groupRows.reduce((sum, row) => sum + row.failedProjects, 0);
2270
- const totalTasks = groupRows.reduce((sum, row) => sum + row.totalTasks, 0);
2271
- const totalEvents = groupRows.reduce((sum, row) => sum + row.eventsCount, 0);
2272
- const successfulRunCount = groupRows.filter((row) => row.failedProjects === 0).length;
2273
- const successRate = groupRows.length === 0 ? 0 : successfulRunCount / groupRows.length;
2274
- const attemptToRuns = /* @__PURE__ */ new Map();
2275
- for (const row of groupRows) {
2276
- const attemptId = row.attemptId ?? "unknown-attempt";
2277
- const attemptRows = attemptToRuns.get(attemptId);
2278
- if (attemptRows == null) {
2279
- attemptToRuns.set(attemptId, [row]);
2280
- continue;
2281
- }
2282
- attemptRows.push(row);
2283
- }
2284
- const attemptSummaries = [...attemptToRuns.entries()].map(([attemptId, attemptRows]) => {
2285
- const successCount = attemptRows.filter((row) => row.failedProjects === 0).length;
2286
- const runCount = attemptRows.length;
2287
- const failedProjectCount = attemptRows.reduce((sum, row) => sum + row.failedProjects, 0);
2288
- const totalTaskCount = attemptRows.reduce((sum, row) => sum + row.totalTasks, 0);
2289
- const totalEventCount = attemptRows.reduce((sum, row) => sum + row.eventsCount, 0);
2290
- return {
2291
- attemptId,
2292
- failedProjects: failedProjectCount,
2293
- runCount,
2294
- runIds: attemptRows.map((row) => row.runId).filter((runId) => runId != null).sort((left, right) => left.localeCompare(right)),
2295
- successRate: roundMetric(runCount === 0 ? 0 : successCount / runCount),
2296
- totalEvents: totalEventCount,
2297
- totalTasks: totalTaskCount
2298
- };
2299
- }).sort((left, right) => left.attemptId.localeCompare(right.attemptId));
2300
- const attemptSuccessRates = attemptSummaries.map((summary) => summary.successRate);
2301
- const minAttemptSuccessRate = attemptSuccessRates.length === 0 ? 0 : Math.min(...attemptSuccessRates);
2302
- const maxAttemptSuccessRate = attemptSuccessRates.length === 0 ? 0 : Math.max(...attemptSuccessRates);
2303
- const avgAttemptSuccessRate = computeAverage(attemptSuccessRates);
2304
- const stdevAttemptSuccessRate = computeStandardDeviation(attemptSuccessRates);
2305
- return {
2306
- attemptCount: attemptToRuns.size,
2307
- attemptSummaries,
2308
- attemptSuccessRateStats: {
2309
- avg: roundMetric(avgAttemptSuccessRate),
2310
- max: roundMetric(maxAttemptSuccessRate),
2311
- min: roundMetric(minAttemptSuccessRate),
2312
- stdev: roundMetric(stdevAttemptSuccessRate)
2313
- },
2314
- experimentId,
2315
- failedProjects,
2316
- runCount: groupRows.length,
2317
- successRate: roundMetric(successRate),
2318
- totalEvents,
2319
- totalTasks,
2320
- workspaceId
2321
- };
2322
- }).sort((left, right) => {
2323
- const workspaceCompare = left.workspaceId.localeCompare(right.workspaceId);
2324
- if (workspaceCompare !== 0) return workspaceCompare;
2325
- return left.experimentId.localeCompare(right.experimentId);
2326
- });
2327
- }
2328
- function formatTableOutput$1(output) {
2329
- const header = "Run ID | Workspace | Experiment | Attempt | Projects(executed/total) | FailedProjects | Tasks | Events";
2330
- const lines = output.runs.map((row) => {
2331
- return `${row.runId ?? "n/a"} | ${row.workspaceId ?? "n/a"} | ${row.experimentId ?? "n/a"} | ${row.attemptId ?? "n/a"} | ${`${row.executedProjects}/${row.totalProjects}`} | ${row.failedProjects} | ${row.totalTasks} | ${row.eventsCount}`;
2332
- });
2333
- return [
2334
- `ANALYZE vieval report: ${output.filteredRunCount}/${output.totalRunCount} runs (${output.experimentSummaries.length} experiment groups)`,
2335
- header,
2336
- ...lines
2337
- ].join("\n");
2338
- }
2339
- function formatCsvOutput(output) {
2340
- return [[
2341
- "runId",
2342
- "workspaceId",
2343
- "experimentId",
2344
- "attemptId",
2345
- "totalProjects",
2346
- "executedProjects",
2347
- "failedProjects",
2348
- "totalTasks",
2349
- "eventsCount",
2350
- "reportDirectory",
2351
- "projectNames"
2352
- ].join(","), ...output.runs.map((row) => {
2353
- const escapedProjectNames = `"${row.projectNames.join("|").replaceAll("\"", "\"\"")}"`;
2354
- const escapedDirectory = `"${row.reportDirectory.replaceAll("\"", "\"\"")}"`;
2355
- return [
2356
- row.runId ?? "",
2357
- row.workspaceId ?? "",
2358
- row.experimentId ?? "",
2359
- row.attemptId ?? "",
2360
- row.totalProjects.toString(),
2361
- row.executedProjects.toString(),
2362
- row.failedProjects.toString(),
2363
- row.totalTasks.toString(),
2364
- row.eventsCount.toString(),
2365
- escapedDirectory,
2366
- escapedProjectNames
2367
- ].join(",");
2368
- })].join("\n");
2369
- }
2370
- async function runReportAnalyzeCli(argv) {
2371
- try {
2372
- const parsed = parseReportAnalyzeCliArguments(argv);
2373
- const output = await readReportAnalyzeOutput(parsed);
2374
- if (parsed.format === "json") {
2375
- process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
2376
- return;
2377
- }
2378
- if (parsed.format === "jsonl") {
2379
- const jsonl = output.runs.map((run) => JSON.stringify(run)).join("\n");
2380
- process.stdout.write(`${jsonl}${jsonl.length > 0 ? "\n" : ""}`);
2381
- return;
2382
- }
2383
- if (parsed.format === "csv") {
2384
- process.stdout.write(`${formatCsvOutput(output)}\n`);
2385
- return;
2386
- }
2387
- process.stdout.write(`${formatTableOutput$1(output)}\n`);
2388
- } catch (error) {
2389
- const errorMessage = errorMessageFrom(error) ?? "Unknown report analyze failure.";
2390
- process.stderr.write(`[vieval report analyze] ${errorMessage}\n`);
2391
- process.exitCode = 1;
2392
- }
2393
- }
2394
- //#endregion
2395
- //#region src/cli/report-index.ts
2396
- const reportIndexHelpText = `
2397
- Build report indexes from generated vieval artifacts.
2398
-
2399
- Usage
2400
- $ vieval report index <reportPath> [--output <path>] [--format <format>]
2401
-
2402
- Options
2403
- --output Output file path (default: <reportPath>/index/runs.jsonl)
2404
- --format Console output format: table | json | jsonl (default: table)
2405
- `;
2406
- function normalizeCliArgv$1(argv) {
2407
- const normalizedArgv = argv[0] === "--" ? argv.slice(1) : [...argv];
2408
- if (normalizedArgv[0] === "report" && normalizedArgv[1] === "index") return normalizedArgv.slice(2);
2409
- if (normalizedArgv[0] === "index") return normalizedArgv.slice(1);
2410
- return normalizedArgv;
2411
- }
2412
- function parseReportIndexCliArguments(argv) {
2413
- const cli = meow(reportIndexHelpText, {
2414
- argv: normalizeCliArgv$1(argv),
2415
- flags: {
2416
- format: {
2417
- default: "table",
2418
- type: "string"
2419
- },
2420
- output: { type: "string" }
2421
- },
2422
- importMeta: import.meta
2423
- });
2424
- const reportPath = cli.input[0];
2425
- if (reportPath == null || reportPath.length === 0) throw new Error("Missing required <reportPath> argument.");
2426
- const normalizedFormat = cli.flags.format.toLowerCase();
2427
- return {
2428
- format: normalizedFormat === "json" ? "json" : normalizedFormat === "jsonl" ? "jsonl" : "table",
2429
- output: cli.flags.output,
2430
- reportPath
2431
- };
2432
- }
2433
- async function writeIndexFile(parsed) {
2434
- const rows = (await readReportArtifacts(parsed.reportPath)).map((artifact) => summarizeReportRunArtifact(artifact));
2435
- const indexFilePath = resolve(parsed.output ?? resolve(parsed.reportPath, "index", "runs.jsonl"));
2436
- await mkdir(dirname(indexFilePath), { recursive: true });
2437
- const indexContents = rows.map((row) => JSON.stringify(row)).join("\n");
2438
- await writeFile(indexFilePath, `${indexContents}${indexContents.length > 0 ? "\n" : ""}`, "utf-8");
2439
- return {
2440
- indexFilePath,
2441
- indexedRunCount: rows.length,
2442
- rows
2443
- };
2444
- }
2445
- function formatTableOutput(output) {
2446
- return [
2447
- "INDEX vieval report",
2448
- `Path ${output.indexFilePath}`,
2449
- `Run count ${output.indexedRunCount}`
2450
- ].join("\n");
2451
- }
2452
- async function runReportIndexCli(argv) {
2453
- try {
2454
- const parsed = parseReportIndexCliArguments(argv);
2455
- const output = await writeIndexFile(parsed);
2456
- if (parsed.format === "json") {
2457
- process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
2458
- return;
2459
- }
2460
- if (parsed.format === "jsonl") {
2461
- const jsonl = output.rows.map((row) => JSON.stringify(row)).join("\n");
2462
- process.stdout.write(`${jsonl}${jsonl.length > 0 ? "\n" : ""}`);
2463
- return;
2464
- }
2465
- process.stdout.write(`${formatTableOutput(output)}\n`);
2466
- } catch (error) {
2467
- const errorMessage = errorMessageFrom(error) ?? "Unknown report index failure.";
2468
- process.stderr.write(`[vieval report index] ${errorMessage}\n`);
2469
- process.exitCode = 1;
2470
- }
2471
- }
2472
- //#endregion
2473
- //#region src/cli/index.ts
2474
- const topLevelHelpText = `
2475
- Execute and report evaluation projects.
2476
-
2477
- Usage
2478
- $ vieval <command> [options]
2479
-
2480
- Commands
2481
- run Discover and execute eval projects
2482
- compare Compare multiple workspaces/methods on one benchmark
2483
- report Analyze and index generated report artifacts
2484
-
2485
- Examples
2486
- $ vieval run
2487
- $ vieval run --config vieval.config.ts --project chess --json --report-out .vieval/reports
2488
- $ vieval compare --config vieval.config.ts --comparison agent-memory
2489
- $ vieval report analyze .vieval/reports/my-run
2490
- $ vieval report index .vieval/reports --output .vieval/reports/index/runs.jsonl
2491
- `;
2492
- function normalizeCliArgv(argv) {
2493
- return argv[0] === "--" ? argv.slice(1) : [...argv];
2494
- }
2495
- function parseTopLevelCliArguments(argv) {
2496
- const normalizedArgv = normalizeCliArgv(argv);
2497
- const command = normalizedArgv[0];
2498
- meow(topLevelHelpText, {
2499
- autoHelp: false,
2500
- autoVersion: false,
2501
- argv: normalizedArgv,
2502
- importMeta: import.meta
2503
- });
2504
- if (command == null || command === "help" || command === "--help" || command === "-h") return {
2505
- command: "help",
2506
- commandArgv: []
2507
- };
2508
- if (command !== "run" && command !== "report" && command !== "compare") throw new Error(`Unsupported vieval command "${command ?? "(none)"}". Expected "run", "compare", or "report".`);
2509
- return {
2510
- command,
2511
- commandArgv: normalizedArgv.slice(1)
2512
- };
2513
- }
2514
- async function runTopLevelCli(argv) {
2515
- const parsed = parseTopLevelCliArguments(argv);
2516
- if (parsed.command === "help") {
2517
- process.stdout.write(`${topLevelHelpText.trim()}\n`);
2518
- return;
2519
- }
2520
- if (parsed.command === "report") {
2521
- const reportSubcommand = parsed.commandArgv[0];
2522
- if (reportSubcommand === "analyze") {
2523
- await runReportAnalyzeCli(parsed.commandArgv);
2524
- return;
2525
- }
2526
- if (reportSubcommand === "index") {
2527
- await runReportIndexCli(parsed.commandArgv);
2528
- return;
2529
- }
2530
- throw new Error(`Unsupported vieval report command "${reportSubcommand ?? "(none)"}". Expected "analyze" or "index".`);
2531
- }
2532
- if (parsed.command === "compare") {
2533
- await runCompareCliOrExit(parsed.commandArgv);
2534
- return;
2535
- }
2536
- const runArguments = parseCliArguments(parsed.commandArgv);
2537
- const output = await runVievalCli({
2538
- attempt: runArguments.attempt,
2539
- configFilePath: runArguments.configFilePath,
2540
- experiment: runArguments.experiment,
2541
- project: runArguments.project,
2542
- reportOut: runArguments.reportOut,
2543
- workspace: runArguments.workspace
2544
- });
2545
- if (runArguments.json) {
2546
- process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
2547
- if (hasRunFailures(output)) process.exitCode = 1;
2548
- return;
2549
- }
2550
- process.stdout.write(`${formatVievalCliRunOutput(output)}\n`);
2551
- if (hasRunFailures(output)) process.exitCode = 1;
2552
- }
2553
- function isDirectExecution() {
2554
- if (!process.argv[1]) return false;
2555
- const resolvedArgvPath = path.resolve(process.argv[1]);
2556
- const currentModulePath = fileURLToPath(import.meta.url);
2557
- try {
2558
- if (realpathSync.native(resolvedArgvPath) === realpathSync.native(currentModulePath)) return true;
2559
- } catch {
2560
- if (resolvedArgvPath === currentModulePath) return true;
2561
- }
2562
- return resolvedArgvPath.replaceAll("\\", "/").endsWith("/.bin/vieval");
2563
- }
2564
- async function main() {
2565
- try {
2566
- await runTopLevelCli(process.argv.slice(2));
2567
- } catch (error) {
2568
- const errorMessage = errorMessageFrom(error) ?? "Unknown CLI failure.";
2569
- process.stderr.write(`[vieval] ${errorMessage}\n`);
2570
- process.exitCode = 1;
2571
- }
2572
- }
2573
- if (isDirectExecution()) await main();
2574
- //#endregion
1
+ import { n as runTopLevelCli, t as parseTopLevelCliArguments } from "../cli-DayPXzHX.mjs";
2575
2
  export { parseTopLevelCliArguments, runTopLevelCli };
2576
-
2577
- //# sourceMappingURL=index.mjs.map