vieval 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/bin/vieval.mjs +1 -1
  2. package/dist/cli/index.mjs +1 -1
  3. package/dist/{cli-DayPXzHX.mjs → cli-sanbKtQq.mjs} +277 -49
  4. package/dist/cli-sanbKtQq.mjs.map +1 -0
  5. package/dist/config.d.mts +2 -2
  6. package/dist/config.mjs +1 -1
  7. package/dist/core/assertions/index.d.mts +1 -1
  8. package/dist/core/inference-executors/index.d.mts +1 -1
  9. package/dist/core/inference-executors/index.mjs +1 -1
  10. package/dist/core/processors/results/index.d.mts +1 -1
  11. package/dist/core/runner/index.d.mts +3 -2
  12. package/dist/core/runner/index.mjs +3 -2
  13. package/dist/core/runner/index.mjs.map +1 -1
  14. package/dist/core/scheduler/index.d.mts +2 -0
  15. package/dist/core/scheduler/index.mjs +188 -0
  16. package/dist/core/scheduler/index.mjs.map +1 -0
  17. package/dist/{env-BFSjny07.mjs → env--94B0UtW.mjs} +1 -1
  18. package/dist/{env-BFSjny07.mjs.map → env--94B0UtW.mjs.map} +1 -1
  19. package/dist/{env-BTq3dV7C.d.mts → env-BeHv_5mo.d.mts} +1 -1
  20. package/dist/{expect-extensions-QLXESWjn.mjs → expect-extensions-DCSqlneN.mjs} +1 -1
  21. package/dist/{expect-extensions-QLXESWjn.mjs.map → expect-extensions-DCSqlneN.mjs.map} +1 -1
  22. package/dist/expect.mjs +1 -1
  23. package/dist/{index-OEdqjQSe.d.mts → index-DBZKkpBe.d.mts} +105 -3
  24. package/dist/index-fakXoZEe.d.mts +147 -0
  25. package/dist/index.d.mts +110 -11
  26. package/dist/index.mjs +214 -53
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{models-D_MsBtYw.mjs → models-DIGdOUpJ.mjs} +1 -1
  29. package/dist/models-DIGdOUpJ.mjs.map +1 -0
  30. package/dist/plugins/chat-models/index.d.mts +21 -1
  31. package/dist/plugins/chat-models/index.mjs +27 -1
  32. package/dist/plugins/chat-models/index.mjs.map +1 -1
  33. package/dist/queue-DsZQkZO_.mjs +21 -0
  34. package/dist/queue-DsZQkZO_.mjs.map +1 -0
  35. package/dist/{registry-CwcMMjnZ.mjs → registry-CcKZqDJY.mjs} +25 -3
  36. package/dist/registry-CcKZqDJY.mjs.map +1 -0
  37. package/dist/testing/expect-extensions.mjs +1 -1
  38. package/package.json +7 -1
  39. package/dist/cli-DayPXzHX.mjs.map +0 -1
  40. package/dist/models-D_MsBtYw.mjs.map +0 -1
  41. package/dist/registry-CwcMMjnZ.mjs.map +0 -1
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env node
2
- import { n as runTopLevelCli } from "../cli-DayPXzHX.mjs";
2
+ import { n as runTopLevelCli } from "../cli-sanbKtQq.mjs";
3
3
  import process from "node:process";
4
4
  import { errorMessageFrom } from "@moeru/std";
5
5
  //#region src/bin/vieval.ts
@@ -1,2 +1,2 @@
1
- import { n as runTopLevelCli, t as parseTopLevelCliArguments } from "../cli-DayPXzHX.mjs";
1
+ import { n as runTopLevelCli, t as parseTopLevelCliArguments } from "../cli-sanbKtQq.mjs";
2
2
  export { parseTopLevelCliArguments, runTopLevelCli };
@@ -1,4 +1,5 @@
1
- import { c as loadRawVievalConfig, l as loadVievalCliConfig, n as consumeModuleRegistrations, o as detectCliConfigMode, r as endModuleRegistration, t as beginModuleRegistration } from "./registry-CwcMMjnZ.mjs";
1
+ import { c as loadRawVievalConfig, l as loadVievalCliConfig, n as consumeModuleRegistrations, o as detectCliConfigMode, r as endModuleRegistration, t as beginModuleRegistration } from "./registry-CcKZqDJY.mjs";
2
+ import { createSchedulerRuntime } from "./core/scheduler/index.mjs";
2
3
  import { RunnerExecutionError, collectEvalEntries, createFilesystemTaskCacheRuntime, createRunnerRuntimeContext, createRunnerSchedule, createTaskExecutionContext, runScheduledTasks } from "./core/runner/index.mjs";
3
4
  import process from "node:process";
4
5
  import { errorMessageFrom } from "@moeru/std";
@@ -12,6 +13,7 @@ import c from "tinyrainbow";
12
13
  import { existsSync, readFileSync } from "node:fs";
13
14
  import { uniq } from "es-toolkit";
14
15
  import { createVitest } from "vitest/node";
16
+ import { formatDuration, intervalToDuration } from "date-fns";
15
17
  import { stripVTControlCharacters } from "node:util";
16
18
  import stringWidth from "fast-string-width";
17
19
  //#region src/cli/comparison-config.ts
@@ -378,12 +380,21 @@ var SummaryReporterStateMachine = class {
378
380
  if (task.state === "finished") return;
379
381
  task.state = "running";
380
382
  task.startedAt ??= this.options.getNow();
381
- if (task.settledCaseIds.has(payload.caseId) || task.runningCases.has(payload.caseId)) return;
383
+ if (task.settledCaseIds.has(payload.caseId)) return;
384
+ const existingCase = task.runningCases.get(payload.caseId);
385
+ if (existingCase != null) {
386
+ existingCase.autoRetry = payload.autoRetry;
387
+ existingCase.caseName = payload.caseName ?? payload.caseId;
388
+ existingCase.retryIndex = payload.retryIndex;
389
+ return;
390
+ }
382
391
  task.caseOrderCounter += 1;
383
392
  task.runningCases.set(payload.caseId, {
393
+ autoRetry: payload.autoRetry,
384
394
  caseId: payload.caseId,
385
395
  caseName: payload.caseName ?? payload.caseId,
386
396
  order: task.caseOrderCounter,
397
+ retryIndex: payload.retryIndex,
387
398
  startedAt: this.options.getNow()
388
399
  });
389
400
  this.syncTaskTotalCases(task);
@@ -420,6 +431,10 @@ var SummaryReporterStateMachine = class {
420
431
  this.caseCounters.failed += 1;
421
432
  return;
422
433
  }
434
+ if (payload.state === "timeout") {
435
+ this.caseCounters.timeout += 1;
436
+ return;
437
+ }
423
438
  this.caseCounters.skipped += 1;
424
439
  }
425
440
  /**
@@ -500,39 +515,52 @@ var SummaryReporterStateMachine = class {
500
515
  const activeRows = this.createActiveRows();
501
516
  const footerRows = this.createFooterRows();
502
517
  const maxRows = options?.maxRows;
503
- const activeBlock = [
518
+ const footerBlock = [...footerRows, ""];
519
+ if (maxRows == null || maxRows <= 0) return [...[
504
520
  "",
505
521
  ...activeRows,
506
522
  ...activeRows.length > 0 ? [""] : []
507
- ];
508
- const footerBlock = [...footerRows, ""];
509
- if (maxRows == null || maxRows <= 0) return [...activeBlock, ...footerBlock];
523
+ ], ...footerBlock];
510
524
  if (maxRows <= footerBlock.length) return footerBlock.slice(-maxRows);
511
- const availableActiveRows = Math.max(0, maxRows - footerBlock.length);
512
- return [...activeBlock.slice(0, availableActiveRows), ...footerBlock];
525
+ return [...createBoundedActiveBlock(activeRows, Math.max(0, maxRows - footerBlock.length)), ...footerBlock];
513
526
  }
514
527
  createActiveRows() {
515
528
  const activeTasks = Array.from(this.tasks.values()).filter((task) => task.state !== "finished").sort(compareActiveTasks);
516
529
  const rows = [];
517
530
  for (const task of activeTasks) {
518
- const suffix = task.state === "queued" ? c.dim(" [queued]") : ` ${task.completedCases}/${task.totalCases}`;
531
+ const now = this.options.getNow();
532
+ const suffix = task.state === "queued" ? c.dim(" [queued]") : formatTaskProgressSuffix(task, now);
519
533
  const badge = formatProjectBadge(task.projectName, this.options.isTTY);
520
534
  rows.push(c.bold(c.yellow(` ${POINTER} `)) + badge + task.displayName + c.dim(suffix));
521
- const slowCases = Array.from(task.runningCases.values()).filter((activeCase) => this.options.getNow() - activeCase.startedAt >= this.options.slowThresholdMs).sort((left, right) => left.order - right.order);
535
+ const slowCases = Array.from(task.runningCases.values()).filter((activeCase) => now - activeCase.startedAt >= this.options.slowThresholdMs).sort((left, right) => left.order - right.order);
522
536
  for (const [index, activeCase] of slowCases.entries()) {
523
537
  const icon = index === slowCases.length - 1 ? TREE_NODE_END : TREE_NODE_MIDDLE;
524
- const elapsed = Math.max(0, this.options.getNow() - activeCase.startedAt);
525
- rows.push(c.bold(c.yellow(` ${icon} `)) + activeCase.caseName + c.bold(c.yellow(` ${formatDuration$1(elapsed)}`)));
538
+ const elapsed = Math.max(0, now - activeCase.startedAt);
539
+ rows.push(c.bold(c.yellow(` ${icon} `)) + activeCase.caseName + formatRetrySuffix(activeCase) + c.bold(c.yellow(` ${formatDuration$2(elapsed)}`)));
526
540
  }
527
541
  }
528
542
  return rows;
529
543
  }
530
544
  createFooterRows() {
545
+ const now = this.options.getNow();
546
+ const runElapsedDurationMs = Math.max(0, now - this.startedAtMs);
547
+ const taskRunningCount = countRunningTasks(this.tasks.values());
548
+ const caseRunningCount = countRunningCases(this.tasks.values());
531
549
  return [
532
- padSummaryTitle("Tasks") + formatCounterState(this.taskCounters),
533
- padSummaryTitle("Cases") + formatCounterState(this.caseCounters),
550
+ padSummaryTitle("Tasks") + formatCounterState(this.taskCounters, taskRunningCount, {
551
+ elapsedDurationMs: runElapsedDurationMs,
552
+ estimatedDurationMs: estimateTotalDurationMs(this.taskCounters.completed, this.taskCounters.total, runElapsedDurationMs)
553
+ }),
554
+ padSummaryTitle("Cases") + formatCounterState(this.caseCounters, caseRunningCount, {
555
+ elapsedDurationMs: runElapsedDurationMs,
556
+ estimatedDurationMs: estimateTotalDurationMs(this.caseCounters.completed, this.caseCounters.total, runElapsedDurationMs)
557
+ }),
558
+ padSummaryTitle("Concurrency") + formatActiveConcurrencyState({
559
+ caseRunningCount,
560
+ taskRunningCount
561
+ }),
534
562
  padSummaryTitle("Start at") + this.startTime,
535
- padSummaryTitle("Duration") + formatDuration$1(Math.max(0, this.options.getNow() - this.startedAtMs))
563
+ padSummaryTitle("Duration") + formatHumanDuration(runElapsedDurationMs)
536
564
  ];
537
565
  }
538
566
  getOrCreateTaskState(taskId) {
@@ -563,6 +591,39 @@ var SummaryReporterStateMachine = class {
563
591
  }
564
592
  };
565
593
  /**
594
+ * Creates the active task block while keeping room for summary footer rows.
595
+ *
596
+ * Use when:
597
+ * - the live TTY window is smaller than the number of running task/case rows
598
+ * - active rows need a visible truncation marker instead of silently disappearing
599
+ *
600
+ * Expects:
601
+ * - `activeRows` contains already-formatted task and slow-case rows
602
+ * - `maxRows` counts the leading spacer and truncation marker
603
+ *
604
+ * Returns:
605
+ * - rows that fit inside `maxRows`
606
+ * - a final hidden-row marker when active rows were omitted
607
+ */
608
+ function createBoundedActiveBlock(activeRows, maxRows) {
609
+ if (maxRows <= 0) return [];
610
+ if (activeRows.length === 0) return [""];
611
+ const fullBlock = [
612
+ "",
613
+ ...activeRows,
614
+ ""
615
+ ];
616
+ if (fullBlock.length <= maxRows) return fullBlock;
617
+ if (maxRows === 1) return [""];
618
+ const visibleActiveRows = Math.max(0, maxRows - 2);
619
+ const hiddenRows = Math.max(0, activeRows.length - visibleActiveRows);
620
+ return [
621
+ "",
622
+ ...activeRows.slice(0, visibleActiveRows),
623
+ c.dim(` ${TREE_NODE_END} ... ${hiddenRows} more running rows hidden`)
624
+ ];
625
+ }
626
+ /**
566
627
  * Creates the live summary reporter state machine for `vieval` CLI runs.
567
628
  *
568
629
  * Use when:
@@ -593,6 +654,7 @@ function createCounterState() {
593
654
  failed: 0,
594
655
  passed: 0,
595
656
  skipped: 0,
657
+ timeout: 0,
596
658
  total: 0
597
659
  };
598
660
  }
@@ -601,6 +663,7 @@ function resetCounterState(counter, total) {
601
663
  counter.failed = 0;
602
664
  counter.passed = 0;
603
665
  counter.skipped = 0;
666
+ counter.timeout = 0;
604
667
  counter.total = total;
605
668
  }
606
669
  function sumTaskCaseTotals(tasks) {
@@ -619,19 +682,48 @@ function compareActiveTasks(left, right) {
619
682
  function padSummaryTitle(label) {
620
683
  return `${c.dim(label.padEnd(8))} `;
621
684
  }
622
- function formatCounterState(counter) {
685
+ function formatCounterState(counter, runningCount, timing) {
686
+ const plannedCount = Math.max(0, counter.total - counter.completed - runningCount);
623
687
  return [
688
+ plannedCount > 0 ? c.bold(c.blue(`${plannedCount} planned`)) : c.dim(`${plannedCount} planned`),
689
+ runningCount > 0 ? c.bold(c.yellow(`${runningCount} running`)) : c.dim(`${runningCount} running`),
624
690
  c.bold(c.green(`${counter.passed} passed`)),
625
691
  counter.failed > 0 ? c.bold(c.red(`${counter.failed} failed`)) : c.dim(`${counter.failed} failed`),
692
+ counter.timeout > 0 ? c.bold(c.yellow(`${counter.timeout} timeout`)) : c.dim(`${counter.timeout} timeout`),
626
693
  counter.skipped > 0 ? c.yellow(`${counter.skipped} skipped`) : c.dim(`${counter.skipped} skipped`)
627
- ].join(c.dim(" | ")) + c.gray(` (${counter.total})`);
694
+ ].join(c.dim(" | ")) + c.gray(` (${counter.total})`) + formatTimingSuffix(timing);
695
+ }
696
+ function formatActiveConcurrencyState(options) {
697
+ return [options.taskRunningCount > 0 ? c.bold(c.yellow(`${options.taskRunningCount} ${pluralize("task", options.taskRunningCount)} running`)) : c.dim("0 tasks running"), options.caseRunningCount > 0 ? c.bold(c.yellow(`${options.caseRunningCount} ${pluralize("case", options.caseRunningCount)} running`)) : c.dim("0 cases running")].join(c.dim(" | "));
698
+ }
699
+ function pluralize(noun, count) {
700
+ return count === 1 ? noun : `${noun}s`;
701
+ }
702
+ function formatRetrySuffix(activeCase) {
703
+ if (activeCase.retryIndex == null || activeCase.retryIndex <= 0 || activeCase.autoRetry == null || activeCase.autoRetry <= 0) return "";
704
+ return c.dim(` retry ${activeCase.retryIndex}/${activeCase.autoRetry}`);
628
705
  }
629
706
  function formatTimeString(date) {
630
707
  return date.toTimeString().split(" ")[0] ?? "";
631
708
  }
632
- function formatDuration$1(durationMs) {
633
- if (durationMs >= 1e3) return `${(durationMs / 1e3).toFixed(2)}s`;
634
- return `${Math.round(durationMs)}ms`;
709
+ function formatDuration$2(durationMs) {
710
+ return formatHumanDuration(durationMs);
711
+ }
712
+ function formatHumanDuration(durationMs) {
713
+ if (durationMs < 1e3) return `${Math.round(durationMs)}ms`;
714
+ const formatted = formatDuration(intervalToDuration({
715
+ end: durationMs,
716
+ start: 0
717
+ }), {
718
+ delimiter: " ",
719
+ format: [
720
+ "hours",
721
+ "minutes",
722
+ "seconds"
723
+ ],
724
+ zero: false
725
+ });
726
+ return formatted.length > 0 ? formatted : "0 seconds";
635
727
  }
636
728
  function formatProjectBadge(projectName, isTTY) {
637
729
  if (projectName == null || projectName.length === 0) return "";
@@ -645,6 +737,37 @@ function formatProjectBadge(projectName, isTTY) {
645
737
  const background = backgroundPool[projectName.split("").reduce((accumulator, character, index) => accumulator + character.charCodeAt(0) + index, 0) % backgroundPool.length];
646
738
  return `${c.black(background(` ${projectName} `))} `;
647
739
  }
740
+ function countRunningCases(tasks) {
741
+ let runningCount = 0;
742
+ for (const task of tasks) runningCount += task.runningCases.size;
743
+ return runningCount;
744
+ }
745
+ function countRunningTasks(tasks) {
746
+ let runningCount = 0;
747
+ for (const task of tasks) if (task.state === "running") runningCount += 1;
748
+ return runningCount;
749
+ }
750
+ function estimateTaskDurationMs(task, now) {
751
+ if (task.startedAt == null) return;
752
+ return estimateTotalDurationMs(task.completedCases, task.totalCases, Math.max(0, now - task.startedAt));
753
+ }
754
+ function estimateTotalDurationMs(completedCount, totalCount, elapsedDurationMs) {
755
+ if (completedCount === 0 || totalCount === 0) return;
756
+ const averageDurationMs = elapsedDurationMs / completedCount;
757
+ return Math.round(averageDurationMs * totalCount);
758
+ }
759
+ function formatTaskProgressSuffix(task, now) {
760
+ const elapsedDurationMs = task.startedAt == null ? 0 : Math.max(0, now - task.startedAt);
761
+ return ` ${task.completedCases}/${task.totalCases}, ${task.runningCases.size} ${pluralize("case", task.runningCases.size)} running${formatTimingSuffix({
762
+ elapsedDurationMs,
763
+ estimatedDurationMs: estimateTaskDurationMs(task, now)
764
+ })}`;
765
+ }
766
+ function formatTimingSuffix(timing) {
767
+ const parts = [`elapsed ${formatHumanDuration(timing.elapsedDurationMs)}`];
768
+ if (timing.estimatedDurationMs != null) parts.push(`estimated ${formatHumanDuration(timing.estimatedDurationMs)}`);
769
+ return ` (${parts.join(", ")})`;
770
+ }
648
771
  //#endregion
649
772
  //#region src/cli/reporters/index.ts
650
773
  /**
@@ -1000,7 +1123,7 @@ async function createVievalVitestCompatReporterBridge(options) {
1000
1123
  return {
1001
1124
  async onCaseEnd(payload) {
1002
1125
  const taskCase = getOrCreateCase(payload.taskId, payload.caseId);
1003
- taskCase.state = payload.state;
1126
+ taskCase.state = payload.state === "timeout" ? "failed" : payload.state;
1004
1127
  await emitToReporters(loadedReporters, (reporter) => reporter.onTestCaseResult?.(taskCase));
1005
1128
  },
1006
1129
  async onCaseStart(payload) {
@@ -1047,10 +1170,61 @@ async function createVievalVitestCompatReporterBridge(options) {
1047
1170
  function hasRunFailures(output) {
1048
1171
  return output.projects.some((project) => {
1049
1172
  if (project.errorMessage != null) return true;
1050
- if (project.caseSummary != null && project.caseSummary.failed > 0) return true;
1173
+ if (project.caseSummary != null && (project.caseSummary.failed > 0 || project.caseSummary.timeout > 0)) return true;
1051
1174
  return (project.caseFailures?.length ?? 0) > 0;
1052
1175
  });
1053
1176
  }
1177
+ function resolveCappedConcurrency(defaultConcurrency, cliConcurrency, fallback) {
1178
+ const effectiveDefault = defaultConcurrency ?? fallback;
1179
+ if (cliConcurrency == null) return effectiveDefault;
1180
+ return Math.min(effectiveDefault, cliConcurrency);
1181
+ }
1182
+ function resolveOptionalRuntimeTaskConcurrency(defaultConcurrency, cliConcurrency) {
1183
+ return cliConcurrency ?? defaultConcurrency;
1184
+ }
1185
+ function resolveWorkspaceConcurrency(loadedConfig, options) {
1186
+ return resolveCappedConcurrency(loadedConfig.concurrency?.workspace, options.workspaceConcurrency, 1);
1187
+ }
1188
+ function resolveProjectConcurrency(project, options) {
1189
+ return resolveCappedConcurrency(project.concurrency?.project, options.projectConcurrency, Number.POSITIVE_INFINITY);
1190
+ }
1191
+ function resolveTaskConcurrency(project, options) {
1192
+ return resolveCappedConcurrency(project.concurrency?.task, options.taskConcurrency, 1);
1193
+ }
1194
+ function resolveScheduledTaskConcurrency(project, options) {
1195
+ return Math.min(resolveProjectConcurrency(project, options), resolveTaskConcurrency(project, options));
1196
+ }
1197
+ function resolveRuntimeTaskConcurrency(taskConcurrency, project, options) {
1198
+ const attempt = resolveOptionalRuntimeTaskConcurrency(taskConcurrency?.attempt ?? project.concurrency?.attempt, options.attemptConcurrency);
1199
+ const caseConcurrency = resolveOptionalRuntimeTaskConcurrency(taskConcurrency?.case ?? project.concurrency?.case, options.caseConcurrency);
1200
+ if (attempt == null && caseConcurrency == null) return;
1201
+ return {
1202
+ attempt,
1203
+ case: caseConcurrency
1204
+ };
1205
+ }
1206
+ function createScheduledTaskWithRuntimeConcurrency(task, project, options) {
1207
+ const taskDefinition = task.entry.task;
1208
+ if (taskDefinition == null) return task;
1209
+ const concurrency = resolveRuntimeTaskConcurrency(taskDefinition.concurrency, project, options);
1210
+ return {
1211
+ ...task,
1212
+ entry: {
1213
+ ...task.entry,
1214
+ task: {
1215
+ ...taskDefinition,
1216
+ concurrency
1217
+ }
1218
+ }
1219
+ };
1220
+ }
1221
+ function resolveCliRuntimeConcurrency(options) {
1222
+ if (options.attemptConcurrency == null && options.caseConcurrency == null) return;
1223
+ return {
1224
+ attempt: options.attemptConcurrency,
1225
+ case: options.caseConcurrency
1226
+ };
1227
+ }
1054
1228
  function shouldUseColor() {
1055
1229
  if (process.env.NO_COLOR != null) return false;
1056
1230
  const forceColor = process.env.FORCE_COLOR;
@@ -1094,7 +1268,7 @@ function createProjectBadge(name, colors, colorEnabled) {
1094
1268
  const background = labelColorPool[name.split("").reduce((accumulator, char, index) => accumulator + char.charCodeAt(0) + index, 0) % labelColorPool.length];
1095
1269
  return `${colors.black(background(` ${name} `))} `;
1096
1270
  }
1097
- function formatDuration(durationMs, colors) {
1271
+ function formatDuration$1(durationMs, colors) {
1098
1272
  if (durationMs == null) return "";
1099
1273
  const rounded = Math.round(durationMs);
1100
1274
  return (rounded > 1e3 ? colors.yellow : colors.green)(` ${rounded}${colors.dim("ms")}`);
@@ -1220,6 +1394,7 @@ function isSummaryReporter(reporter) {
1220
1394
  return "getWindowRows" in reporter;
1221
1395
  }
1222
1396
  function createRunReporter(options) {
1397
+ const getRows = options?.getRows ?? (() => process.stdout.rows);
1223
1398
  const reporter = createCliReporter({
1224
1399
  getColumns: options?.getColumns ?? (() => process.stdout.columns ?? 80),
1225
1400
  getNow: options?.getNow ?? (() => Date.now()),
@@ -1240,7 +1415,7 @@ function createRunReporter(options) {
1240
1415
  };
1241
1416
  const rendererBaseOptions = {
1242
1417
  getColumns: options?.getColumns ?? (() => process.stdout.columns ?? 80),
1243
- getWindow: () => reporter.getWindowRows(),
1418
+ getWindow: () => reporter.getWindowRows({ maxRows: normalizeLiveReporterMaxRows(getRows()) }),
1244
1419
  queueRenderReset: options?.queueRenderReset,
1245
1420
  supportsAnsiWindowing: options?.supportsAnsiWindowing,
1246
1421
  writeOutput: options?.writeOutput ?? ((value) => process.stdout.write(value))
@@ -1289,6 +1464,22 @@ function createRunReporter(options) {
1289
1464
  }
1290
1465
  };
1291
1466
  }
1467
+ /**
1468
+ * Normalizes terminal row count into the live reporter window height.
1469
+ *
1470
+ * Before:
1471
+ * - undefined
1472
+ * - 4
1473
+ * - 40
1474
+ *
1475
+ * After:
1476
+ * - 23
1477
+ * - 6
1478
+ * - 39
1479
+ */
1480
+ function normalizeLiveReporterMaxRows(rows) {
1481
+ return Math.max(6, (rows == null || !Number.isFinite(rows) || rows <= 0 ? 24 : Math.floor(rows)) - 1);
1482
+ }
1292
1483
  function createTaskQueuePayload(task, projectName) {
1293
1484
  return {
1294
1485
  displayName: task.entry.name,
@@ -1315,11 +1506,12 @@ function createTaskReporterHooks(task, reporter, projectName, recordEvent, proje
1315
1506
  projectCaseCounters.seenCaseIds.add(projectCaseId);
1316
1507
  if (payload.state === "passed") projectCaseCounters.passed += 1;
1317
1508
  else if (payload.state === "failed") projectCaseCounters.failed += 1;
1509
+ else if (payload.state === "timeout") projectCaseCounters.timeout += 1;
1318
1510
  else projectCaseCounters.skipped += 1;
1319
1511
  }
1320
1512
  }
1321
1513
  syncCaseTotal(payload.total);
1322
- if (payload.state === "failed" && payload.errorMessage != null && projectCaseFailures != null) projectCaseFailures.push({
1514
+ if ((payload.state === "failed" || payload.state === "timeout") && payload.errorMessage != null && projectCaseFailures != null) projectCaseFailures.push({
1323
1515
  caseId,
1324
1516
  caseName: payload.name,
1325
1517
  errorMessage: payload.errorMessage,
@@ -1342,8 +1534,10 @@ function createTaskReporterHooks(task, reporter, projectName, recordEvent, proje
1342
1534
  const caseId = createTaskCaseReporterId(payload);
1343
1535
  syncCaseTotal(payload.total);
1344
1536
  reporter.onCaseStart({
1537
+ autoRetry: payload.autoRetry,
1345
1538
  caseId,
1346
1539
  caseName: payload.name,
1540
+ retryIndex: payload.retryIndex,
1347
1541
  taskId: task.id
1348
1542
  });
1349
1543
  vitestCompatReporter?.onCaseStart({
@@ -1360,7 +1554,7 @@ function createTaskReporterHooks(task, reporter, projectName, recordEvent, proje
1360
1554
  }
1361
1555
  };
1362
1556
  }
1363
- function createCliTaskExecutionContext(task, models, cacheRootDirectory, cacheProjectName, workspaceId, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter) {
1557
+ function createCliTaskExecutionContext(task, models, cacheRootDirectory, cacheProjectName, workspaceId, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, runtimeConcurrency, vitestCompatReporter) {
1364
1558
  return {
1365
1559
  ...createTaskExecutionContext({
1366
1560
  cache: createFilesystemTaskCacheRuntime({
@@ -1371,7 +1565,8 @@ function createCliTaskExecutionContext(task, models, cacheRootDirectory, cachePr
1371
1565
  models,
1372
1566
  task
1373
1567
  }),
1374
- reporterHooks: createTaskReporterHooks(task, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter)
1568
+ reporterHooks: createTaskReporterHooks(task, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter),
1569
+ runtimeConcurrency
1375
1570
  };
1376
1571
  }
1377
1572
  function resolveTaskReporterHooks(task, context, reporter, projectName, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter) {
@@ -1492,13 +1687,14 @@ async function prepareProject(project) {
1492
1687
  };
1493
1688
  }
1494
1689
  }
1495
- async function executePreparedProject(prepared, identity, cacheProjectName, reporter, counters, recordEvent) {
1690
+ async function executePreparedProject(prepared, identity, cacheProjectName, reporter, counters, recordEvent, options) {
1496
1691
  const settledTaskIds = /* @__PURE__ */ new Set();
1497
1692
  const projectCaseCounters = {
1498
1693
  failed: 0,
1499
1694
  passed: 0,
1500
1695
  seenCaseIds: /* @__PURE__ */ new Set(),
1501
- skipped: 0
1696
+ skipped: 0,
1697
+ timeout: 0
1502
1698
  };
1503
1699
  const projectCaseFailures = [];
1504
1700
  const vitestCompatReporter = await createVievalVitestCompatReporterBridge({
@@ -1507,9 +1703,10 @@ async function executePreparedProject(prepared, identity, cacheProjectName, repo
1507
1703
  });
1508
1704
  const rawTaskExecutor = prepared.project.executor ?? createAutoTaskExecutor(reporter, prepared.name, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter);
1509
1705
  const taskExecutor = async (task, context) => {
1706
+ const runtimeTask = createScheduledTaskWithRuntimeConcurrency(task, prepared.project, options);
1510
1707
  return {
1511
- ...await rawTaskExecutor(task, context),
1512
- matrix: cloneScheduledTaskMatrix(task)
1708
+ ...await rawTaskExecutor(runtimeTask, context),
1709
+ matrix: cloneScheduledTaskMatrix(runtimeTask)
1513
1710
  };
1514
1711
  };
1515
1712
  for (const task of prepared.tasks) await vitestCompatReporter?.onTaskQueued({ taskId: task.id });
@@ -1517,7 +1714,7 @@ async function executePreparedProject(prepared, identity, cacheProjectName, repo
1517
1714
  try {
1518
1715
  const aggregated = await runScheduledTasks(prepared.tasks, taskExecutor, {
1519
1716
  createExecutionContext(task) {
1520
- return createCliTaskExecutionContext(task, prepared.project.models, resolve(prepared.project.root, ".vieval", "cache"), cacheProjectName ?? prepared.name, identity.workspaceId, reporter, prepared.name, recordEvent, projectCaseCounters, projectCaseFailures, vitestCompatReporter);
1717
+ return createCliTaskExecutionContext(task, prepared.project.models, resolve(prepared.project.root, ".vieval", "cache"), cacheProjectName ?? prepared.name, identity.workspaceId, reporter, prepared.name, recordEvent, projectCaseCounters, projectCaseFailures, resolveCliRuntimeConcurrency(options), vitestCompatReporter);
1521
1718
  },
1522
1719
  onTaskEnd(task, state) {
1523
1720
  settledTaskIds.add(task.id);
@@ -1538,7 +1735,8 @@ async function executePreparedProject(prepared, identity, cacheProjectName, repo
1538
1735
  onTaskStart(task) {
1539
1736
  reporter.onTaskStart({ taskId: task.id });
1540
1737
  vitestCompatReporter?.onTaskStart({ taskId: task.id });
1541
- }
1738
+ },
1739
+ maxConcurrency: resolveScheduledTaskConcurrency(prepared.project, options)
1542
1740
  });
1543
1741
  await vitestCompatReporter?.onRunEnd({ failed: false });
1544
1742
  return {
@@ -1546,6 +1744,7 @@ async function executePreparedProject(prepared, identity, cacheProjectName, repo
1546
1744
  failed: projectCaseCounters.failed,
1547
1745
  passed: projectCaseCounters.passed,
1548
1746
  skipped: projectCaseCounters.skipped,
1747
+ timeout: projectCaseCounters.timeout,
1549
1748
  total: projectCaseCounters.seenCaseIds.size
1550
1749
  },
1551
1750
  caseFailures: projectCaseFailures,
@@ -1592,6 +1791,7 @@ async function executePreparedProject(prepared, identity, cacheProjectName, repo
1592
1791
  failed: projectCaseCounters.failed,
1593
1792
  passed: projectCaseCounters.passed,
1594
1793
  skipped: projectCaseCounters.skipped,
1794
+ timeout: projectCaseCounters.timeout,
1595
1795
  total: projectCaseCounters.seenCaseIds.size
1596
1796
  },
1597
1797
  caseFailures: projectCaseFailures,
@@ -1642,6 +1842,7 @@ async function runVievalCli(options = {}) {
1642
1842
  const reporter = createReporterWithEventCapture(createRunReporter(options.reporter), eventRecorder.record);
1643
1843
  try {
1644
1844
  const selectedProjects = filterProjectsByName(loadedConfig.projects, options.project ?? []);
1845
+ const workspaceScheduler = createSchedulerRuntime({ concurrency: { workspace: resolveWorkspaceConcurrency(loadedConfig, options) } });
1645
1846
  const preparedProjects = await Promise.all(selectedProjects.map(async (project) => prepareProject(project)));
1646
1847
  const executableProjects = preparedProjects.filter((project) => project.kind === "prepared").map((project) => project.prepared);
1647
1848
  const totalTasks = preparedProjects.reduce((sum, project) => {
@@ -1659,14 +1860,21 @@ async function runVievalCli(options = {}) {
1659
1860
  };
1660
1861
  reporter.onRunStart({ totalTasks });
1661
1862
  for (const project of executableProjects) for (const task of project.tasks) reporter.onTaskQueued(createTaskQueuePayload(task, project.name));
1662
- const projectSummaries = [];
1663
- for (const preparedProject of preparedProjects) {
1664
- if (preparedProject.kind === "summary") {
1665
- projectSummaries.push(preparedProject.summary);
1666
- continue;
1667
- }
1668
- projectSummaries.push(await executePreparedProject(preparedProject.prepared, identity, options.cacheProjectName, reporter, reporterCounters, eventRecorder.record));
1669
- }
1863
+ const projectSummaries = (await Promise.all(preparedProjects.map(async (preparedProject, index) => {
1864
+ if (preparedProject.kind === "summary") return {
1865
+ index,
1866
+ summary: preparedProject.summary
1867
+ };
1868
+ return {
1869
+ index,
1870
+ summary: await workspaceScheduler.runCase({
1871
+ experimentId: identity.experimentId,
1872
+ projectName: preparedProject.prepared.name,
1873
+ scope: "workspace",
1874
+ workspaceId: identity.workspaceId
1875
+ }, async () => executePreparedProject(preparedProject.prepared, identity, options.cacheProjectName, reporter, reporterCounters, eventRecorder.record, options))
1876
+ };
1877
+ }))).sort((left, right) => left.index - right.index).map((item) => item.summary);
1670
1878
  reporter.onRunEnd({
1671
1879
  failedTasks: reporterCounters.failedTasks,
1672
1880
  passedTasks: reporterCounters.passedTasks,
@@ -1736,10 +1944,10 @@ function formatVievalCliRunOutput(output) {
1736
1944
  executedTasks += project.result?.overall.runCount ?? 0;
1737
1945
  const badge = createProjectBadge(project.name, colors, colorEnabled);
1738
1946
  const isFailed = project.errorMessage != null;
1739
- const hasFailedCases = (project.caseSummary?.failed ?? 0) > 0 || (project.caseFailures?.length ?? 0) > 0;
1947
+ const hasFailedCases = (project.caseSummary?.failed ?? 0) > 0 || (project.caseSummary?.timeout ?? 0) > 0 || (project.caseFailures?.length ?? 0) > 0;
1740
1948
  if (isFailed) {
1741
1949
  failedProjects += 1;
1742
- lines.push(` ${colors.red("❯")} ${badge}${formatDuration(project.durationMs, colors)}`);
1950
+ lines.push(` ${colors.red("❯")} ${badge}${formatDuration$1(project.durationMs, colors)}`);
1743
1951
  lines.push(` ${project.errorMessage}`);
1744
1952
  continue;
1745
1953
  }
@@ -1748,7 +1956,7 @@ function formatVievalCliRunOutput(output) {
1748
1956
  const countLabel = colors.dim(`(${project.taskCount} tasks)`);
1749
1957
  const detailsLabel = colors.dim(` ${project.discoveredEvalFileCount} files, ${project.entryCount} entries, 0 runs, hybrid n/a`);
1750
1958
  const matrixSummary = formatMatrixSummary(project.matrixSummary);
1751
- lines.push(` ${colors.dim("○")} ${badge}${countLabel}${detailsLabel}${formatDuration(project.durationMs, colors)}`);
1959
+ lines.push(` ${colors.dim("○")} ${badge}${countLabel}${detailsLabel}${formatDuration$1(project.durationMs, colors)}`);
1752
1960
  if (matrixSummary != null) lines.push(` ${colors.dim(matrixSummary)}`);
1753
1961
  const scheduleBreakdown = formatScheduleBreakdown(project);
1754
1962
  if (scheduleBreakdown != null) lines.push(` ${scheduleBreakdown}`);
@@ -1760,10 +1968,10 @@ function formatVievalCliRunOutput(output) {
1760
1968
  const hybridAverageLabel = hybridAverage == null ? "n/a" : String(hybridAverage);
1761
1969
  const runCount = project.result?.overall.runCount ?? 0;
1762
1970
  const countLabel = colors.dim(`(${project.taskCount} tasks)`);
1763
- const caseSummaryLabel = project.caseSummary == null ? "" : `, cases ${project.caseSummary.passed} passed | ${project.caseSummary.failed} failed`;
1971
+ const caseSummaryLabel = project.caseSummary == null ? "" : `, cases ${project.caseSummary.passed} passed | ${project.caseSummary.failed} failed | ${project.caseSummary.timeout} timeout`;
1764
1972
  const detailsLabel = colors.dim(` ${project.discoveredEvalFileCount} files, ${project.entryCount} entries, ${runCount} runs${caseSummaryLabel}, hybrid ${hybridAverageLabel}`);
1765
1973
  const matrixSummary = formatMatrixSummary(project.matrixSummary);
1766
- lines.push(` ${hasFailedCases ? colors.red("❯") : colors.green("✓")} ${badge}${countLabel}${detailsLabel}${formatDuration(project.durationMs, colors)}`);
1974
+ lines.push(` ${hasFailedCases ? colors.red("❯") : colors.green("✓")} ${badge}${countLabel}${detailsLabel}${formatDuration$1(project.durationMs, colors)}`);
1767
1975
  if (matrixSummary != null) lines.push(` ${colors.dim(matrixSummary)}`);
1768
1976
  const scheduleBreakdown = formatScheduleBreakdown(project);
1769
1977
  if (scheduleBreakdown != null) lines.push(` ${scheduleBreakdown}`);
@@ -1904,6 +2112,11 @@ const evalRunHelpText = `
1904
2112
  --workspace Workspace id used in report artifacts
1905
2113
  --experiment Experiment id used in report artifacts
1906
2114
  --attempt Attempt id used in report artifacts
2115
+ --workspace-concurrency Workspace scheduling cap
2116
+ --project-concurrency Project scheduling cap
2117
+ --task-concurrency Task scheduling cap
2118
+ --attempt-concurrency Attempt scheduling cap
2119
+ --case-concurrency Case scheduling cap
1907
2120
  --report-out Report output root directory
1908
2121
  --json Print machine-readable JSON output
1909
2122
  `;
@@ -1945,17 +2158,27 @@ function parseCliArguments(argv) {
1945
2158
  workspace: { type: "string" },
1946
2159
  experiment: { type: "string" },
1947
2160
  attempt: { type: "string" },
2161
+ workspaceConcurrency: { type: "number" },
2162
+ projectConcurrency: { type: "number" },
2163
+ taskConcurrency: { type: "number" },
2164
+ attemptConcurrency: { type: "number" },
2165
+ caseConcurrency: { type: "number" },
1948
2166
  reportOut: { type: "string" }
1949
2167
  }
1950
2168
  });
1951
2169
  return {
1952
2170
  attempt: cli.flags.attempt,
2171
+ attemptConcurrency: cli.flags.attemptConcurrency,
2172
+ caseConcurrency: cli.flags.caseConcurrency,
1953
2173
  configFilePath: cli.flags.config,
1954
2174
  experiment: cli.flags.experiment,
1955
2175
  json: cli.flags.json === true,
1956
2176
  project: normalizeProjectNames(cli.flags.project),
2177
+ projectConcurrency: cli.flags.projectConcurrency,
1957
2178
  reportOut: cli.flags.reportOut,
1958
- workspace: cli.flags.workspace
2179
+ taskConcurrency: cli.flags.taskConcurrency,
2180
+ workspace: cli.flags.workspace,
2181
+ workspaceConcurrency: cli.flags.workspaceConcurrency
1959
2182
  };
1960
2183
  }
1961
2184
  /**
@@ -1991,11 +2214,16 @@ async function runEvalRunCli(argv) {
1991
2214
  try {
1992
2215
  const output = await runVievalCli({
1993
2216
  attempt: parsed.attempt,
2217
+ attemptConcurrency: parsed.attemptConcurrency,
2218
+ caseConcurrency: parsed.caseConcurrency,
1994
2219
  configFilePath: parsed.configFilePath,
1995
2220
  experiment: parsed.experiment,
1996
2221
  project: parsed.project,
2222
+ projectConcurrency: parsed.projectConcurrency,
1997
2223
  reportOut: parsed.reportOut,
1998
- workspace: parsed.workspace
2224
+ taskConcurrency: parsed.taskConcurrency,
2225
+ workspace: parsed.workspace,
2226
+ workspaceConcurrency: parsed.workspaceConcurrency
1999
2227
  });
2000
2228
  if (parsed.json) {
2001
2229
  process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
@@ -2590,4 +2818,4 @@ async function runTopLevelCli(argv) {
2590
2818
  //#endregion
2591
2819
  export { runTopLevelCli as n, parseTopLevelCliArguments as t };
2592
2820
 
2593
- //# sourceMappingURL=cli-DayPXzHX.mjs.map
2821
+ //# sourceMappingURL=cli-sanbKtQq.mjs.map