vieval 0.0.4 → 0.0.6

package/dist/{index-BDMEAmf2.d.mts → index-DBZKkpBe.d.mts}

@@ -1,5 +1,5 @@
-import { ReadStream, WriteStream } from "node:fs";
 import { Buffer } from "node:buffer";
+import { ReadStream, WriteStream } from "node:fs";
 
 //#region src/core/cache/types.d.ts
 /**
@@ -477,6 +477,10 @@ interface ModelDefinition {
    * Alias names that can resolve this model.
    */
   aliases: string[];
+  /**
+   * Optional execution policy hints attached to this model.
+   */
+  executionPolicy?: TaskExecutionPolicy;
   /**
    * Optional model-level call parameters.
    */
@@ -858,6 +862,61 @@ interface TaskRunOutput {
    */
   scores: readonly RunScore[];
 }
+/**
+ * Execution policy applied to task and case callbacks.
+ *
+ * Use when:
+ * - one task or case should time out after a bounded duration
+ * - failures should retry within the current attempt or trigger a later full task attempt
+ *
+ * Expects:
+ * - `timeout` to be a positive integer when provided
+ * - `autoRetry` and `autoAttempt` to be non-negative integers when provided
+ *
+ * Returns:
+ * - one partial execution policy descriptor
+ */
+interface TaskExecutionPolicy {
+  /**
+   * Additional retries allowed within the current attempt.
+   *
+   * @default 0
+   */
+  autoRetry?: number;
+  /**
+   * Additional full task attempts allowed after the current attempt settles.
+   *
+   * @default 0
+   */
+  autoAttempt?: number;
+  /**
+   * Timeout in milliseconds for one case execution.
+   */
+  timeout?: number;
+}
+/**
+ * Task-local concurrency metadata.
+ *
+ * Use when:
+ * - task declarations need to preserve attempt and case caps for later runtime coordination
+ * - DSL execution needs to resolve the default task-level case concurrency for registered cases
+ *
+ * Expects:
+ * - each provided value to be a positive integer chosen by the caller
+ *
+ * Returns:
+ * - one partial task-local concurrency descriptor
+ */
+interface TaskConcurrencyConfig {
+  /**
+   * Attempt-level concurrency cap for this task.
+   */
+  attempt?: number;
+  /**
+   * Case-level concurrency cap for this task.
+   */
+  case?: number;
+}
 /**
  * Runtime context passed into eval task `run`.
  */
@@ -933,6 +992,23 @@ interface TaskRunContext {
    * - hooks are best-effort observers and should not affect task scoring
    */
   reporterHooks?: TaskReporterHooks;
+  /**
+   * Optional runtime scheduling overrides supplied by CLI or host execution.
+   *
+   * Use when:
+   * - run operators need to override task/case concurrency without editing eval code
+   * - DSL task runners need to distinguish runtime flags from code defaults
+   *
+   * Expects:
+   * - values are positive integers when provided
+   *
+   * @default undefined
+   */
+  runtimeConcurrency?: TaskConcurrencyConfig;
+  /**
+   * Cooperative abort signal for the current execution.
+   */
+  signal?: AbortSignal;
 }
 /**
  * Allowed terminal outcomes for one task case.
@@ -943,7 +1019,7 @@ interface TaskRunContext {
  * Expects:
  * - consumers treat the value as the final state for the case
  */
-type TaskCaseState = 'passed' | 'failed';
+type TaskCaseState = 'passed' | 'failed' | 'timeout';
 /**
  * Payload emitted when a task case starts.
  *
@@ -956,10 +1032,18 @@ type TaskCaseState = 'passed' | 'failed';
  * - `total` is the total number of registered cases
  */
 interface TaskCaseReporterPayload {
+  /**
+   * Maximum retry count configured for this case.
+   */
+  autoRetry?: number;
   /**
    * Declared case label.
    */
   name: string;
+  /**
+   * Current retry attempt index, where `0` is the first try.
+   */
+  retryIndex?: number;
   /**
    * Zero-based case position within the task.
    */
@@ -1052,6 +1136,24 @@ interface TaskDefinition {
    * Stable task id for diagnostics.
    */
   id: string;
+  /**
+   * Optional task-local concurrency metadata.
+   *
+   * Use when:
+   * - task declarations need to preserve task-scoped attempt/case caps for later scheduler wiring
+   * - higher-level orchestration wants to inspect task-local concurrency without executing the task
+   *
+   * Expects:
+   * - each provided value to be a positive integer chosen by the caller
+   *
+   * Returns:
+   * - one partial task-local concurrency descriptor
+   */
+  concurrency?: TaskConcurrencyConfig;
+  /**
+   * Optional task-local execution policy.
+   */
+  executionPolicy?: TaskExecutionPolicy;
   /**
    * Optional matrix layering for this task definition.
    *
@@ -1186,5 +1288,5 @@ interface ConfigHookPlugin<TConfig> {
   configVievalResolved?: (config: TConfig) => void | Promise<void>;
 }
 //#endregion
-export { ScheduledTaskMatrixMeta as $, TaskModelSelectionOptions as A, AggregatedRunResults as B, RunScheduledTasksOptions as C, runScheduledTasks as D, ScheduledTaskExecutor as E, collectEvalEntries as F, aggregateRunResults as G, RunResult as H, CreateVievalRunnerRuntimeContextOptions as I, RunnerMatrixDefinition as J, CreateRunnerScheduleOptions as K, RunnerRuntimeContext as L, ModelDefinition as M, resolveModelByName as N, CreateTaskExecutionContextOptions as O, asProjectRelativePath as P, ScheduledTaskMatrix as Q, createRunnerRuntimeContext as R, TaskRunOutput as S, RunnerTaskState as T, RunScore as U, AggregatedRunSummary as V, RunScoreKind as W, RunnerMatrixSelection as X, RunnerMatrixInput as Y, ScheduledTask as Z, TaskCaseState as _, EvalDefinition as a, CacheFileOptions as at, TaskReporterHooks as b, MatrixAxisValues as c, MatrixPrimitive as d, createRunnerSchedule as et, MatrixRow as f, TaskCaseReporterPayload as g, TaskCaseReporterEndPayload as h, CollectedEvalEntry as i, CacheFileHandle as it, createTaskExecutionContext as j, TaskExecutionContext as k, MatrixDefinition as l, ScopedMatrices as m, defineEval as n, createFilesystemTaskCacheRuntime as nt, EvalModule as o, CacheNamespace as ot, MatrixValue as p, InferenceExecutor as q, defineTask as r, normalizeCacheFilePathSegments as rt, EvalModuleMap as s, TaskCacheRuntime as st, ConfigHookPlugin as t, CreateFilesystemTaskCacheRuntimeOptions as tt, MatrixLayer as u, TaskDefinition as v, RunnerExecutionError as w, TaskRunContext as x, TaskReporterEventPayload as y, AggregatedProviderSummary as z };
-//# sourceMappingURL=index-BDMEAmf2.d.mts.map
+export { ScheduledTask as $, CreateTaskExecutionContextOptions as A, createRunnerRuntimeContext as B, TaskRunContext as C, RunnerTaskState as D, RunnerExecutionError as E, resolveModelByName as F, RunScore as G, AggregatedRunResults as H, asProjectRelativePath as I, CreateRunnerScheduleOptions as J, RunScoreKind as K, collectEvalEntries as L, TaskModelSelectionOptions as M, createTaskExecutionContext as N, ScheduledTaskExecutor as O, ModelDefinition as P, RunnerMatrixSelection as Q, CreateVievalRunnerRuntimeContextOptions as R, TaskReporterHooks as S, RunScheduledTasksOptions as T, AggregatedRunSummary as U, AggregatedProviderSummary as V, RunResult as W, RunnerMatrixDefinition as X, InferenceExecutor as Y, RunnerMatrixInput as Z, TaskCaseState as _, EvalDefinition as a, normalizeCacheFilePathSegments as at, TaskExecutionPolicy as b, MatrixAxisValues as c, CacheNamespace as ct, MatrixPrimitive as d, ScheduledTaskMatrix as et, MatrixRow as f, TaskCaseReporterPayload as g, TaskCaseReporterEndPayload as h, CollectedEvalEntry as i, createFilesystemTaskCacheRuntime as it, TaskExecutionContext as j, runScheduledTasks as k, MatrixDefinition as l, TaskCacheRuntime as lt, ScopedMatrices as m, defineEval as n, createRunnerSchedule as nt, EvalModule as o, CacheFileHandle as ot, MatrixValue as p, aggregateRunResults as q, defineTask as r, CreateFilesystemTaskCacheRuntimeOptions as rt, EvalModuleMap as s, CacheFileOptions as st, ConfigHookPlugin as t, ScheduledTaskMatrixMeta as tt, MatrixLayer as u, TaskConcurrencyConfig as v, TaskRunOutput as w, TaskReporterEventPayload as x, TaskDefinition as y, RunnerRuntimeContext as z };
+//# sourceMappingURL=index-DBZKkpBe.d.mts.map

package/dist/index-fakXoZEe.d.mts

@@ -0,0 +1,147 @@
+//#region src/core/scheduler/types.d.ts
+/**
+ * Hierarchical scheduler scopes used by the queue runtime.
+ *
+ * Use when:
+ * - selecting which concurrency cap applies to a unit of work
+ * - ordering middleware acquisition and release hooks
+ *
+ * Expects:
+ * - values move from broad to narrow scope in this order:
+ *   `workspace -> project -> task -> attempt -> case`
+ *
+ * Returns:
+ * - a string literal scope identifier
+ */
+type SchedulerScope = 'workspace' | 'project' | 'task' | 'attempt' | 'case';
+/**
+ * Context carried through queue acquisition, execution, and release.
+ *
+ * Use when:
+ * - middleware needs stable identifiers for logging or instrumentation
+ * - runtime helpers need to know which hierarchical scope is being executed
+ *
+ * Expects:
+ * - `workspaceId` and `experimentId` are always present
+ * - narrower ids are only provided when the selected scope requires them
+ *
+ * Returns:
+ * - a serializable scope context object
+ */
+interface SchedulerScopeContext {
+  scope: SchedulerScope;
+  workspaceId: string;
+  experimentId: string;
+  projectName?: string;
+  taskId?: string;
+  attemptIndex?: number;
+  caseId?: string;
+}
+/**
+ * Middleware hooks wrapped around scheduler execution.
+ *
+ * Use when:
+ * - recording queue lifecycle telemetry
+ * - attaching tracing or temporary resources around queued work
+ *
+ * Expects:
+ * - implementations call `next()` exactly once to continue the pipeline
+ *
+ * Returns:
+ * - optional async acquire and release hooks
+ */
+interface SchedulerMiddleware {
+  onAcquire?: (context: SchedulerScopeContext, next: () => Promise<void>) => Promise<void> | void;
+  onRelease?: (context: SchedulerScopeContext, next: () => Promise<void>) => Promise<void> | void;
+}
+/**
+ * Per-scope concurrency limits used by the scheduler runtime.
+ *
+ * Use when:
+ * - bounding parallel work for a specific scope
+ * - disabling a scope cap by omitting its entry
+ *
+ * Expects:
+ * - values are positive integers when provided
+ *
+ * Returns:
+ * - a partial map of scheduler scope to concurrency cap
+ */
+interface SchedulerConcurrencyConfig {
+  workspace?: number;
+  project?: number;
+  task?: number;
+  attempt?: number;
+  case?: number;
+}
+/**
+ * Options accepted by {@link createSchedulerRuntime}.
+ *
+ * Use when:
+ * - constructing a scheduler runtime with queue limits or middleware
+ *
+ * Expects:
+ * - omitted configuration falls back to unbounded execution for that concern
+ *
+ * Returns:
+ * - queue and middleware configuration for the runtime
+ */
+interface CreateSchedulerRuntimeOptions {
+  concurrency?: SchedulerConcurrencyConfig;
+  middleware?: SchedulerMiddleware[];
+}
+/**
+ * Runtime API used to execute case-level work through scheduler policies.
+ *
+ * Use when:
+ * - the runner needs to enqueue case execution under middleware and queue caps
+ *
+ * Expects:
+ * - `runCase` receives a case context and a callback that performs the work
+ *
+ * Returns:
+ * - a promise that resolves with the callback result once all guards release
+ */
+interface SchedulerRuntime {
+  runCase: <T>(context: SchedulerScopeContext, execute: () => Promise<T>) => Promise<T>;
+}
+//#endregion
+//#region src/core/scheduler/runtime.d.ts
+/**
+ * Creates the core scheduler runtime used to serialize work by scope.
+ *
+ * Call stack:
+ *
+ * {@link createSchedulerRuntime}
+ *   -> `createRuntimeQueues`
+ *     -> `runtime.runCase(context, execute)`
+ *       -> `runWithQueues`
+ *         -> `runAcquireMiddleware`
+ *           -> `execute`
+ *         -> `runReleaseMiddleware`
+ *
+ * Use when:
+ * - runner code needs concurrency caps for queued case execution
+ * - middleware should wrap work with acquire/release lifecycle hooks
+ *
+ * Expects:
+ * - middleware is ordered from outermost to innermost concern
+ * - concurrency caps are positive integers when provided
+ *
+ * Returns:
+ * - a scheduler runtime with case execution support
+ */
+declare function createSchedulerRuntime(options?: CreateSchedulerRuntimeOptions): SchedulerRuntime;
+/**
+ * Resolves the scheduler scopes that apply to a context.
+ *
+ * Before:
+ * - `{ scope: 'case', workspaceId: 'ws', experimentId: 'exp', caseId: 'case-1' }`
+ *
+ * After:
+ * - `['workspace', 'project', 'task', 'attempt', 'case']` up to the requested scope
+ */
+declare function getActiveScopes(context: SchedulerScopeContext): SchedulerScope[];
+//#endregion
+export { SchedulerMiddleware as a, SchedulerScopeContext as c, SchedulerConcurrencyConfig as i, getActiveScopes as n, SchedulerRuntime as o, CreateSchedulerRuntimeOptions as r, SchedulerScope as s, createSchedulerRuntime as t };
+//# sourceMappingURL=index-fakXoZEe.d.mts.map

package/dist/index.d.mts

@@ -1,6 +1,6 @@
-import { H as RunResult, M as ModelDefinition, S as TaskRunOutput, W as RunScoreKind, Z as ScheduledTask, k as TaskExecutionContext, l as MatrixDefinition, q as InferenceExecutor, t as ConfigHookPlugin, u as MatrixLayer, x as TaskRunContext } from "./index-BDMEAmf2.mjs";
-import { a as requiredEnvFrom } from "./env-DtpjACOW.mjs";
-import { t as expect } from "./expect-B2vaoRVZ.mjs";
+import { $ as ScheduledTask, C as TaskRunContext, K as RunScoreKind, P as ModelDefinition, W as RunResult, Y as InferenceExecutor, b as TaskExecutionPolicy, j as TaskExecutionContext, l as MatrixDefinition, t as ConfigHookPlugin, u as MatrixLayer, v as TaskConcurrencyConfig, w as TaskRunOutput } from "./index-DBZKkpBe.mjs";
+import { a as requiredEnvFrom } from "./env-BeHv_5mo.mjs";
+import { expect } from "./expect.mjs";
 import * as _$c12 from "c12";
 
 //#region src/cli/reporters/vitest-compat-reporter.d.ts
@@ -66,6 +66,41 @@ type VievalVitestCompatReporterReference = VievalVitestCompatReporterValue | rea
  * CLI plugin shape bound to the full CLI config object.
  */
 type CliConfigPlugin = ConfigHookPlugin<CliConfig>;
+/**
+ * Concurrency limits that can be declared in CLI-facing config.
+ *
+ * Use when:
+ * - the CLI needs independent caps for workspace, project, task, attempt, or case scheduling scopes
+ * - config authors want to define concurrency without wiring runtime execution yet
+ *
+ * Expects:
+ * - each provided value to be a positive integer chosen by the caller
+ *
+ * Returns:
+ * - one partial concurrency descriptor keyed by scheduling scope
+ */
+interface CliConcurrencyConfig {
+  /**
+   * Workspace-level concurrency cap.
+   */
+  workspace?: number;
+  /**
+   * Project-level concurrency cap.
+   */
+  project?: number;
+  /**
+   * Task-level concurrency cap.
+   */
+  task?: number;
+  /**
+   * Attempt-level concurrency cap.
+   */
+  attempt?: number;
+  /**
+   * Case-level concurrency cap.
+   */
+  case?: number;
+}
 /**
  * Defines one project block for `vieval run`.
  */
@@ -115,6 +150,12 @@ interface CliProjectConfig {
    * Optional eval-time matrix dimensions.
    */
   evalMatrix?: MatrixDefinition | MatrixLayer;
+  /**
+   * Optional project-scoped concurrency overrides.
+   *
+   * @default inherited from top-level or CLI execution settings
+   */
+  concurrency?: Omit<CliConcurrencyConfig, 'workspace'>;
   /**
    * Optional task executor.
    *
@@ -218,9 +259,11 @@ interface CliComparisonConfig {
  * Expects:
  * - `model` resolves configured models for the current task
  * - `reporterHooks` follows `TaskRunContext['reporterHooks']`
+ * - `runtimeConcurrency` follows `TaskRunContext['runtimeConcurrency']`
  */
 interface CliProjectExecutorContext extends TaskExecutionContext {
   reporterHooks?: TaskRunContext['reporterHooks'];
+  runtimeConcurrency?: TaskRunContext['runtimeConcurrency'];
 }
 /**
  * Top-level CLI config loaded from `vieval.config.*`.
@@ -232,6 +275,19 @@ interface CliConfigBase {
    * @default []
    */
   models?: ModelDefinition[];
+  /**
+   * Global concurrency defaults inherited by projects and tasks.
+   *
+   * Use when:
+   * - config authors want one shared concurrency policy across workspace, project, task, attempt, and case scopes
+   * - project-local overrides should start from a top-level baseline
+   *
+   * Expects:
+   * - each provided value to be a positive integer chosen by the caller
+   *
+   * @default undefined
+   */
+  concurrency?: CliConcurrencyConfig;
   /**
    * Global config plugins.
    *
@@ -345,11 +401,43 @@ interface CaseRunContext<TInput> extends TaskRunContext {
    * - `value` to be JSON-serializable
    */
   metric: (name: string, value: boolean | number | string | null) => void;
+  /**
+   * Cooperative abort signal for the current case execution.
+   */
+  signal: AbortSignal;
 }
 /**
  * Callback for one task case.
  */
 type CaseRunner<TInput> = (context: CaseRunContext<TInput>) => Promise<void> | void;
+/**
+ * Per-group options for `casesFromInputs`.
+ *
+ * Use when:
+ * - one generated case group should run with a lower case concurrency than the task default
+ * - a task should keep a broader task-level cap while one expensive case family stays bounded
+ *
+ * Expects:
+ * - `concurrency` to be a positive integer when provided
+ *
+ * Returns:
+ * - one partial case-group execution descriptor
+ */
+interface CasesFromInputsOptions extends TaskExecutionPolicy {
+  /**
+   * Case-level concurrency cap for cases registered by one `casesFromInputs(...)` call.
+   */
+  concurrency?: number;
+}
+/**
+ * Per-case registration options for `caseOf`.
+ */
+interface CaseRegistrationOptions<TInput> extends TaskExecutionPolicy {
+  /**
+   * Optional case input payload.
+   */
+  input: TInput;
+}
 /**
  * Builder callbacks passed into `describeTask`.
  */
@@ -359,35 +447,44 @@ interface DescribeTaskBuilder {
    */
   caseOf: {
     (name: string, run: CaseRunner<undefined>): void;
-    <TInput>(name: string, run: CaseRunner<TInput>, options: {
-      input: TInput;
-    }): void;
+    <TInput>(name: string, run: CaseRunner<TInput>, options: CaseRegistrationOptions<TInput>): void;
   };
   /**
    * Registers multiple cases from input list.
    */
-  casesFromInputs: <TInput>(namePrefix: string, inputs: readonly TInput[], run: CaseRunner<TInput>) => void;
+  casesFromInputs: <TInput>(namePrefix: string, inputs: readonly TInput[], run: CaseRunner<TInput>, options?: CasesFromInputsOptions) => void;
 }
 /**
  * Options for `describeTask`.
  */
-interface DescribeTaskOptions {
+interface DescribeTaskOptions extends TaskExecutionPolicy {
   /**
    * Optional description override.
    */
   description?: string;
+  /**
+   * Optional task-local concurrency overrides.
+   *
+   * Use when:
+   * - one task should cap attempt fan-out independently from the surrounding project
+   * - one task should cap case fan-out without changing global scheduling defaults
+   *
+   * Expects:
+   * - each provided value to be a positive integer
+   *
+   * @default inherited from project or CLI concurrency settings
+   */
+  concurrency?: TaskConcurrencyConfig;
 }
 /**
  * Registers one case in the currently active task scope.
  */
 declare function caseOf(name: string, run: CaseRunner<undefined>): void;
-declare function caseOf<TInput>(name: string, run: CaseRunner<TInput>, options: {
-  input: TInput;
-}): void;
+declare function caseOf<TInput>(name: string, run: CaseRunner<TInput>, options: CaseRegistrationOptions<TInput>): void;
 /**
  * Registers multiple cases in the currently active task scope.
  */
-declare function casesFromInputs<TInput>(namePrefix: string, inputs: readonly TInput[], run: CaseRunner<TInput>): void;
+declare function casesFromInputs<TInput>(namePrefix: string, inputs: readonly TInput[], run: CaseRunner<TInput>, options?: CasesFromInputsOptions): void;
 /**
  * Defines one eval task with task/case semantics similar to Vitest.
  *
@@ -399,6 +496,8 @@ declare function describeTask(name: string, build: ((builder: DescribeTaskBuilde
   readonly description: string;
   readonly name: string;
   readonly task: {
+    readonly concurrency: TaskConcurrencyConfig | undefined;
+    readonly executionPolicy: TaskExecutionPolicy | undefined;
     readonly id: string;
     readonly run: (context: TaskRunContext) => Promise<TaskRunOutput>;
   };