vieval 0.0.4 → 0.0.6

package/dist/index-C3gPFmcR.d.mts

@@ -1,314 +0,0 @@
-import { U as RunScore, W as RunScoreKind } from "./index-BDMEAmf2.mjs";
-
-//#region src/core/assertions/index.d.ts
-/**
- * Stores mutable evaluation state for stateful assertion flows.
- *
- * Use when:
- * - assertions need to share counters, rolling metrics, or memoized values
- * - a scenario evaluates multiple steps and expects state-aware checks
- */
-type AssertionState = Map<string, unknown>;
-/**
- * Represents one tool call emitted by a model response.
- */
-interface ToolCall {
-  /**
-   * Tool name used by the call.
-   */
-  name: string;
-  /**
-   * Tool arguments payload.
-   */
-  args: unknown;
-}
-/**
- * Normalized assertion context for one model output.
- */
-interface AssertionContext {
-  /**
-   * Plain text model output used by text assertions.
-   */
-  text: string;
-  /**
-   * Optional structured output parsed from the model response.
-   */
-  structuredOutput?: unknown;
-  /**
-   * Optional tool calls extracted from the model response.
-   */
-  toolCalls?: readonly ToolCall[];
-  /**
-   * Shared mutable state for stateful assertion measurement.
-   */
-  state: AssertionState;
-}
-/**
- * Result for one assertion evaluation.
- */
-interface AssertionOutcome {
-  /**
-   * Stable assertion id.
-   */
-  id: string;
-  /**
-   * Assertion family emitted as run score kind.
-   */
-  scoreKind: RunScoreKind;
-  /**
-   * Whether the assertion passed.
-   */
-  pass: boolean;
-  /**
-   * Normalized score in the `0..1` range.
-   */
-  score: number;
-  /**
-   * Human-readable reason for logs and reports.
-   */
-  reason: string;
-}
-/**
- * Async assertion function used by eval scenarios.
- */
-type Assertion = (context: AssertionContext) => Promise<AssertionOutcome>;
-/**
- * Normalizes text for matching.
- *
- * Before: `"  Hello\nWorld  "`
- * After: `"hello world"`
- */
-declare function normalizeMatchText(value: string, caseSensitive: boolean): string;
-/**
- * Options for include-keyword assertions.
- */
-interface MustIncludeAssertionOptions {
-  /**
-   * Stable assertion id.
-   */
-  id: string;
-  /**
-   * Keywords that must be present.
-   */
-  keywords: readonly string[];
-  /**
-   * Match mode for keywords.
-   *
-   * @default 'all'
-   */
-  mode?: 'all' | 'any';
-  /**
-   * Case-sensitive matching toggle.
-   *
-   * @default false
-   */
-  caseSensitive?: boolean;
-}
-/**
- * Creates an assertion that requires specific keywords in model text.
- *
- * Example:
- * `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`
- */
-declare function expectMustInclude(options: MustIncludeAssertionOptions): Assertion;
-/**
- * Options for exclude-keyword assertions.
- */
-interface MustExcludeAssertionOptions {
-  /**
-   * Stable assertion id.
-   */
-  id: string;
-  /**
-   * Keywords that must not appear.
-   */
-  keywords: readonly string[];
-  /**
-   * Case-sensitive matching toggle.
-   *
-   * @default false
-   */
-  caseSensitive?: boolean;
-}
-/**
- * Creates an assertion that forbids specific keywords.
- *
- * Example:
- * `expectMustExclude({ id: 'no-engine-dump', keywords: ['bestmove', 'ponder'] })`
- */
-declare function expectMustExclude(options: MustExcludeAssertionOptions): Assertion;
-/**
- * Options for regular-expression assertions.
- */
-interface RegexAssertionOptions {
-  /**
-   * Stable assertion id.
-   */
-  id: string;
-  /**
-   * Pattern to apply to model text.
-   */
-  pattern: RegExp;
-}
-/**
- * Creates an assertion based on a regular expression.
- *
- * Example:
- * `expectRegex({ id: 'starts-with-act', pattern: /^<\|ACT:/ })`
- */
-declare function expectRegex(options: RegexAssertionOptions): Assertion;
-/**
- * Options for structured-output assertions.
- */
-interface StructuredOutputAssertionOptions<TValue> {
-  /**
-   * Stable assertion id.
-   */
-  id: string;
-  /**
-   * Runtime validator for structured output.
-   */
-  validate: (value: unknown) => value is TValue;
-  /**
-   * Optional failure reason.
-   */
-  failureReason?: string;
-}
-/**
- * Creates an assertion for structured model output.
- *
- * Example:
- * `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`
- */
-declare function expectStructuredOutput<TValue>(options: StructuredOutputAssertionOptions<TValue>): Assertion;
-/**
- * Options for tool-call argument assertions.
- */
-interface ToolCallArgsAssertionOptions {
-  /**
-   * Stable assertion id.
-   */
-  id: string;
-  /**
-   * Tool name to inspect.
-   */
-  toolName: string;
-  /**
-   * Runtime validator for tool arguments.
-   */
-  validate: (args: unknown) => boolean;
-}
-/**
- * Creates an assertion for validating tool-call arguments.
- *
- * Example:
- * `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`
- */
-declare function expectToolCallArgs(options: ToolCallArgsAssertionOptions): Assertion;
-/**
- * Rubric judge result returned by teacher-model or rubric logic.
- */
-interface RubricJudgeResult {
-  /**
-   * Normalized score in the `0..1` range.
-   */
-  score: number;
-  /**
-   * Judge explanation text.
-   */
-  reason: string;
-  /**
-   * Optional judge model id.
-   */
-  judgeModel?: string;
-}
-/**
- * Options for rubric assertions.
- */
-interface RubricAssertionOptions {
-  /**
-   * Stable assertion id.
-   */
-  id: string;
-  /**
-   * Async rubric judge callback.
-   */
-  judge: (context: AssertionContext) => Promise<RubricJudgeResult>;
-  /**
-   * Minimum passing score.
-   *
-   * @default 0.7
-   */
-  minScore?: number;
-}
-/**
- * Creates a rubric assertion driven by teacher-model style scoring.
- *
- * Example:
- * `expectRubric({ id: 'human-like-tone', judge: judgeFn, minScore: 0.8 })`
- */
-declare function expectRubric(options: RubricAssertionOptions): Assertion;
-/**
- * Options for custom assertions.
- */
-interface CustomAssertionOptions {
-  /**
-   * Stable assertion id.
-   */
-  id: string;
-  /**
-   * Score family emitted by this custom assertion.
-   */
-  scoreKind: RunScoreKind;
-  /**
-   * Custom evaluator callback.
-   */
-  evaluate: (context: AssertionContext) => Promise<{
-    pass: boolean;
-    reason: string;
-    score: number;
-  }> | {
-    pass: boolean;
-    reason: string;
-    score: number;
-  };
-}
-/**
- * Creates a custom assertion with fully user-defined logic.
- *
- * Example:
- * `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`
- */
-declare function expectCustom(options: CustomAssertionOptions): Assertion;
-/**
- * Creates an inverse assertion.
- *
- * Example:
- * `expectNot(expectMustInclude({ id: 'contains-engine-word', keywords: ['bestmove'] }), { id: 'no-engine-word' })`
- */
-declare function expectNot(assertion: Assertion, options: {
-  id: string;
-}): Assertion;
-/**
- * Executes assertion list and returns all outcomes.
- *
- * Call stack:
- *
- * {@link evaluateAssertions}
- *   -> `assertion(context)`
- *     -> {@link AssertionOutcome}[]
- */
-declare function evaluateAssertions(assertions: readonly Assertion[], context: Omit<AssertionContext, 'state'> & {
-  state?: AssertionState;
-}): Promise<AssertionOutcome[]>;
-/**
- * Converts assertion outcomes to run-score tuples consumed by aggregation.
- */
-declare function toRunScores(outcomes: readonly AssertionOutcome[]): RunScore[];
-/**
- * Returns failing assertion outcomes in original order.
- */
-declare function collectFailedAssertions(outcomes: readonly AssertionOutcome[]): AssertionOutcome[];
-//#endregion
-export { expectToolCallArgs as C, expectStructuredOutput as S, toRunScores as T, expectMustExclude as _, CustomAssertionOptions as a, expectRegex as b, RegexAssertionOptions as c, StructuredOutputAssertionOptions as d, ToolCall as f, expectCustom as g, evaluateAssertions as h, AssertionState as i, RubricAssertionOptions as l, collectFailedAssertions as m, AssertionContext as n, MustExcludeAssertionOptions as o, ToolCallArgsAssertionOptions as p, AssertionOutcome as r, MustIncludeAssertionOptions as s, Assertion as t, RubricJudgeResult as u, expectMustInclude as v, normalizeMatchText as w, expectRubric as x, expectNot as y };
-//# sourceMappingURL=index-C3gPFmcR.d.mts.map

package/dist/registry-CHJcTN2W.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"registry-CHJcTN2W.mjs","names":["loadEnv","loadViteEnv"],"sources":["../src/cli/config.ts","../src/dsl/registry.ts"],"sourcesContent":["import type { ConfigHookPlugin, MatrixDefinition, MatrixLayer, TaskRunContext } from '../config'\nimport type { ModelDefinition } from '../config/models'\nimport type { RunResult, TaskExecutionContext } from '../core/runner'\nimport type { InferenceExecutor, ScheduledTask } from '../core/runner/schedule'\nimport type { VievalVitestCompatReporterReference } from './reporters/vitest-compat-reporter'\n\nimport process from 'node:process'\n\nimport { access, readFile } from 'node:fs/promises'\nimport { createRequire } from 'node:module'\nimport { dirname, extname, isAbsolute, join, resolve } from 'node:path'\nimport { pathToFileURL } from 'node:url'\n\nimport { errorMessageFrom } from '@moeru/std'\nimport { createDefineConfig, loadConfig } from 'c12'\nimport { loadEnv as loadViteEnv } from 'vite'\n\nconst matrixLayerKeys = new Set(['disable', 'extend', 'override'])\nconst ambiguousMatrixDefinitionErrorMessage = 'Ambiguous matrix definition: cannot mix reserved layer keys (disable, extend, override) with matrix axis keys.'\nconst require = createRequire(import.meta.url)\n\n/**\n * CLI plugin shape bound to the full CLI config object.\n */\nexport type CliConfigPlugin = ConfigHookPlugin<CliConfig>\n\n/**\n * Defines one project block for `vieval run`.\n */\nexport interface CliProjectConfig {\n  /**\n   * Project label used in summary output.\n   */\n  name: string\n  /**\n   * Project root used for include/exclude glob matching.\n   *\n   * @default process cwd\n   */\n  root?: string\n  /**\n   * Glob patterns for eval file discovery.\n   *\n   * @default Common eval file globs for TypeScript and JavaScript module formats.\n   */\n  include?: string[]\n  /**\n   * Glob patterns excluded from discovery.\n   *\n   * @default Common exclusion globs for dependencies, build output, and VCS directories.\n   */\n  exclude?: string[]\n  /**\n   * Providers expanded by scheduler.\n   *\n   * @default [{ id: 'default' }]\n   */\n  inferenceExecutors?: InferenceExecutor[]\n  /**\n   * Model definitions available to project runtime execution.\n   *\n   * Inference executors control schedule fan-out, while models provide\n   * runtime lookup metadata for `context.model(...)` during task execution.\n   *\n   * @default inherited from top-level config models\n   */\n  models?: ModelDefinition[]\n  /**\n   * Optional run-time matrix dimensions.\n   */\n  runMatrix?: MatrixDefinition | MatrixLayer\n  /**\n   * Optional eval-time matrix dimensions.\n   */\n  evalMatrix?: MatrixDefinition | MatrixLayer\n  /**\n   * Optional task executor.\n   *\n   * Use when this project should execute live inferenceExecutor requests.\n   * If omitted, `vieval run` performs collection + scheduling only.\n   */\n  executor?: (task: ScheduledTask, context: CliProjectExecutorContext) => Promise<RunResult>\n  /**\n   * Optional project-local plugins.\n   */\n  plugins?: CliConfigPlugin[]\n  /**\n   * Optional vitest-compatible reporter modules.\n   *\n   * Use when:\n   * - project runs should emit additional reporter callbacks using Vitest-style lifecycle names\n   *\n   * @default []\n   */\n  reporters?: VievalVitestCompatReporterReference[]\n}\n\n/**\n * One workspace descriptor for workspace-mode configs.\n */\nexport interface CliWorkspaceConfig {\n  /**\n   * Workspace identifier.\n   */\n  id: string\n  /**\n   * Workspace root path.\n   */\n  root: string\n}\n\n/**\n * One explicit comparison method descriptor.\n */\nexport interface CliComparisonMethodConfig {\n  /**\n   * Method identifier shown in compare reports.\n   */\n  id: string\n  /**\n   * Workspace path containing this method's `vieval.config.*`.\n   */\n  workspace: string\n  /**\n   * Project name to execute inside workspace config.\n   */\n  project: string\n  /**\n   * Optional explicit config file path for this workspace.\n   */\n  configFilePath?: string\n}\n\n/**\n * Benchmark identity and shared cache namespace.\n */\nexport interface CliComparisonBenchmarkConfig {\n  /**\n   * Benchmark identifier used in report artifacts.\n   */\n  id: string\n  /**\n   * Shared cache namespace reused across method runs.\n   */\n  sharedCaseNamespace: string\n}\n\n/**\n * One comparison entry loaded by `vieval compare`.\n */\nexport interface CliComparisonConfig {\n  /**\n   * Comparison id selected by `--comparison`.\n   */\n  id: string\n  /**\n   * Benchmark metadata for reporting and shared cache coordination.\n   */\n  benchmark: CliComparisonBenchmarkConfig\n  /**\n   * Optional explicit method list.\n   */\n  methods?: CliComparisonMethodConfig[]\n  /**\n   * Optional workspace glob(s) discovered relative to config directory.\n   */\n  includesWorkspaces?: string | string[]\n  /**\n   * Optional workspace exclude glob(s), also relative to config directory.\n   */\n  excludesWorkspaces?: string | string[]\n}\n\n/**\n * Execution context exposed to project-level `executor` implementations.\n *\n * Use when:\n * - a project executor needs the task-scoped model resolver plus case reporter hooks\n * - custom scheduling logic wants the same hook shape as `TaskRunContext`\n *\n * Expects:\n * - `model` resolves configured models for the current task\n * - `reporterHooks` follows `TaskRunContext['reporterHooks']`\n */\nexport interface CliProjectExecutorContext extends TaskExecutionContext {\n  reporterHooks?: TaskRunContext['reporterHooks']\n}\n\n/**\n * Top-level CLI config loaded from `vieval.config.*`.\n */\ninterface CliConfigBase {\n  /**\n   * Global model definitions inherited by projects.\n   *\n   * @default []\n   */\n  models?: ModelDefinition[]\n  /**\n   * Global config plugins.\n   *\n   * @default []\n   */\n  plugins?: CliConfigPlugin[]\n  /**\n   * Global vitest-compatible reporter modules inherited by projects.\n   *\n   * @default []\n   */\n  reporters?: VievalVitestCompatReporterReference[]\n  /**\n   * Environment variables injected into `process.env` during `vieval run`.\n   *\n   * Use when:\n   * - eval tasks depend on runtime env values (for example inferenceExecutor API keys)\n   * - config wants deterministic env values without shell-level exports\n   *\n   * @default {}\n   */\n  env?: NodeJS.ProcessEnv\n}\n\n/**\n * Project mode config for `vieval run`.\n */\nexport interface CliProjectModeConfig extends CliConfigBase {\n  /**\n   * Project list expanded by `vieval run`.\n   *\n   * @default [{ name: 'default' }]\n   */\n  projects?: CliProjectConfig[]\n  comparisons?: never\n  workspaces?: never\n}\n\n/**\n * Workspace mode config placeholder for future workspace orchestration.\n */\nexport interface CliWorkspaceModeConfig extends CliConfigBase {\n  workspaces: CliWorkspaceConfig[]\n  projects?: never\n  comparisons?: never\n}\n\n/**\n * Comparison mode config for `vieval compare`.\n */\nexport interface CliComparisonModeConfig extends CliConfigBase {\n  comparisons: CliComparisonConfig[]\n  projects?: never\n  workspaces?: never\n}\n\n/**\n * Top-level CLI config loaded from `vieval.config.*`.\n *\n * Exactly one top-level mode is allowed:\n * - `projects`\n * - `workspaces`\n * - `comparisons`\n */\nexport type CliConfig = CliProjectModeConfig | CliWorkspaceModeConfig | CliComparisonModeConfig\n\nexport type CliConfigMode = 'comparisons' | 'projects' | 'workspaces'\n\nexport interface LoadedRawCliConfig {\n  config: CliConfig | null\n  configFilePath: string | null\n}\n\n/**\n * Normalized CLI project used by runtime orchestration.\n */\nexport interface NormalizedCliProjectConfig {\n  exclude: string[]\n  executor?: (task: ScheduledTask, context: CliProjectExecutorContext) => Promise<RunResult>\n  include: string[]\n  runMatrix?: MatrixLayer\n  evalMatrix?: MatrixLayer\n  models: ModelDefinition[]\n  name: string\n  inferenceExecutors: InferenceExecutor[]\n  root: string\n  reporters: VievalVitestCompatReporterReference[]\n}\n\n/**\n * Result of loading and normalizing a config file.\n */\nexport interface LoadedCliConfig {\n  configFilePath: string | null\n  env: NodeJS.ProcessEnv\n  projects: NormalizedCliProjectConfig[]\n}\n\n/**\n * Runtime options for config loading.\n */\nexport interface LoadVievalCliConfigOptions {\n  /**\n   * Starting directory for config lookup.\n   *\n   * @default process.cwd()\n   */\n  cwd?: string\n  /**\n   * Explicit config file path.\n   */\n  configFilePath?: string\n}\n\n/**\n * Helper used by `vieval.config.*` for better type inference.\n */\nexport const defineConfig = createDefineConfig<CliConfig>()\n\n/**\n * Loads `.env*` files using Vite's env resolution behavior.\n *\n * Use when:\n * - `vieval.config.*` should mirror Vitest/Vite env loading semantics\n * - config wants to populate top-level `env` via file-based values\n *\n * Expects:\n * - `mode` to match the env file suffix (`.env.<mode>`)\n * - `envDir` to point at the directory containing `.env` files\n *\n * Returns:\n * - Key/value map compatible with `CliConfig['env']`\n */\nexport function loadEnv(mode: string, envDir: string, prefixes: string | string[] = ''): NodeJS.ProcessEnv {\n  return loadViteEnv(mode, envDir, prefixes)\n}\n\nasync function applyVievalPlugins(config: CliConfig): Promise<CliConfig> {\n  let currentConfig: CliConfig = config\n  const plugins = currentConfig.plugins ?? []\n\n  for (const plugin of plugins) {\n    if (plugin.configVieval == null) {\n      continue\n    }\n\n    const nextConfig = await plugin.configVieval(currentConfig)\n    if (nextConfig != null) {\n      currentConfig = {\n        ...currentConfig,\n        ...nextConfig,\n      } as CliConfig\n    }\n  }\n\n  for (const plugin of plugins) {\n    await plugin.configVievalResolved?.(currentConfig)\n  }\n\n  return currentConfig\n}\n\nasync function isReadableFile(filePath: string): Promise<boolean> {\n  try {\n    await access(filePath)\n    return true\n  }\n  catch {\n    return false\n  }\n}\n\nfunction isConfigFileExtensionUsingRequire(extension: string): boolean {\n  return extension === '.cjs' || extension === '.cts'\n}\n\nfunction isConfigFileExtensionUsingJsonParse(extension: string): boolean {\n  return extension === '.json'\n}\n\nasync function importVievalConfigModule(filePath: string): Promise<unknown> {\n  const extension = extname(filePath)\n\n  if (isConfigFileExtensionUsingJsonParse(extension)) {\n    const raw = await readFile(filePath, 'utf-8')\n    return JSON.parse(raw) as unknown\n  }\n\n  if (isConfigFileExtensionUsingRequire(extension)) {\n    return require(filePath) as unknown\n  }\n\n  return import(pathToFileURL(filePath).href)\n}\n\nfunction resolveConfigExport(moduleValue: unknown): unknown {\n  if (moduleValue == null) {\n    return null\n  }\n\n  if (typeof moduleValue !== 'object') {\n    return moduleValue\n  }\n\n  if ('default' in moduleValue) {\n    return (moduleValue as { default: unknown }).default\n  }\n\n  return moduleValue\n}\n\nasync function findNearestConfigFile(startDirectory: string): Promise<string | null> {\n  const supportedFileNames = [\n    'vieval.config.ts',\n    'vieval.config.mts',\n    'vieval.config.cts',\n    'vieval.config.js',\n    'vieval.config.mjs',\n    'vieval.config.cjs',\n    'vieval.config.json',\n  ]\n\n  let currentDirectory = resolve(startDirectory)\n\n  while (true) {\n    for (const fileName of supportedFileNames) {\n      const candidatePath = join(currentDirectory, fileName)\n      if (await isReadableFile(candidatePath)) {\n        return candidatePath\n      }\n    }\n\n    const parentDirectory = dirname(currentDirectory)\n    if (parentDirectory === currentDirectory) {\n      return null\n    }\n    currentDirectory = parentDirectory\n  }\n}\n\nasync function resolveVievalConfig(\n  cwd: string,\n  explicitConfigFilePath: string | undefined,\n): Promise<{\n  config: CliConfig | null\n  configFilePath: string | null\n}> {\n  const resolvedConfigFilePath = explicitConfigFilePath == null\n    ? await findNearestConfigFile(cwd)\n    : (isAbsolute(explicitConfigFilePath) ? explicitConfigFilePath : resolve(cwd, explicitConfigFilePath))\n\n  if (explicitConfigFilePath != null && resolvedConfigFilePath != null && !await isReadableFile(resolvedConfigFilePath)) {\n    throw new Error(`Config file does not exist or is not readable: ${resolvedConfigFilePath}`)\n  }\n\n  if (resolvedConfigFilePath == null) {\n    return {\n      config: null,\n      configFilePath: null,\n    }\n  }\n\n  const loaded = await loadConfig<CliConfig>({\n    configFile: resolvedConfigFilePath,\n    cwd,\n    dotenv: false,\n    envName: false,\n    extend: false,\n    import: importVievalConfigModule,\n    packageJson: false,\n    rcFile: false,\n    resolveModule: resolveConfigExport,\n  })\n  return {\n    config: loaded.config,\n    configFilePath: resolvedConfigFilePath,\n  }\n}\n\nfunction isLayerMatrixDefinition(matrix: MatrixDefinition | MatrixLayer): matrix is MatrixLayer {\n  const matrixKeys = Object.keys(matrix)\n  return (\n    matrixKeys.length > 0\n    && matrixKeys.every(key => matrixLayerKeys.has(key))\n  )\n}\n\nfunction assertNonAmbiguousMatrixDefinition(matrix: MatrixDefinition | MatrixLayer): void {\n  const matrixKeys = Object.keys(matrix)\n  const hasReservedKeys = matrixKeys.some(key => matrixLayerKeys.has(key))\n  const hasAxisKeys = matrixKeys.some(key => !matrixLayerKeys.has(key))\n\n  if (hasReservedKeys && hasAxisKeys) {\n    throw new TypeError(ambiguousMatrixDefinitionErrorMessage)\n  }\n}\n\nfunction normalizeMatrixLayerInput(matrix: MatrixDefinition | MatrixLayer | undefined): MatrixLayer | undefined {\n  if (matrix == null) {\n    return undefined\n  }\n\n  assertNonAmbiguousMatrixDefinition(matrix)\n\n  if (isLayerMatrixDefinition(matrix)) {\n    return matrix\n  }\n\n  return {\n    extend: matrix,\n  }\n}\n\nfunction normalizeProjectConfig(\n  project: CliProjectConfig,\n  cwd: string,\n  inheritedModels: readonly ModelDefinition[],\n  inheritedReporterReferences: readonly VievalVitestCompatReporterReference[],\n): NormalizedCliProjectConfig {\n  const include = project.include ?? [\n    '**/*.eval.ts',\n    '**/*.eval.mts',\n    '**/*.eval.cts',\n    '**/*.eval.js',\n    '**/*.eval.mjs',\n    '**/*.eval.cjs',\n  ]\n  const exclude = project.exclude ?? [\n    '**/node_modules/**',\n    '**/dist/**',\n    '**/.git/**',\n  ]\n  const models = project.models ?? [...inheritedModels]\n  const inferenceExecutors = project.inferenceExecutors ?? [{ id: 'default' }]\n  const root = project.root == null\n    ? cwd\n    : (isAbsolute(project.root) ? project.root : resolve(cwd, project.root))\n  const reporters = project.reporters ?? [...inheritedReporterReferences]\n\n  return {\n    exclude,\n    executor: project.executor,\n    include,\n    evalMatrix: normalizeMatrixLayerInput(project.evalMatrix),\n    models,\n    name: project.name,\n    inferenceExecutors,\n    reporters,\n    runMatrix: normalizeMatrixLayerInput(project.runMatrix),\n    root,\n  }\n}\n\nfunction normalizeConfig(config: CliConfig | null | undefined, cwd: string): NormalizedCliProjectConfig[] {\n  if (config != null) {\n    const mode = detectCliConfigMode(config)\n    if (mode === 'comparisons') {\n      throw new Error('vieval run requires project-mode config. Received comparison-mode config.')\n    }\n    if (mode === 'workspaces') {\n      throw new Error('vieval run requires project-mode config. Received workspace-mode config.')\n    }\n  }\n\n  const projects = config?.projects ?? [{ name: 'default' }]\n  const inheritedModels = config?.models ?? []\n  const inheritedReporterReferences = config?.reporters ?? []\n\n  return projects.map(project => normalizeProjectConfig(project, cwd, inheritedModels, inheritedReporterReferences))\n}\n\n/**\n * Detects which top-level config mode is active.\n *\n * Expects:\n * - exactly one of `projects`, `workspaces`, or `comparisons`\n *\n * Returns:\n * - active top-level mode key\n */\nexport function detectCliConfigMode(config: CliConfig): CliConfigMode {\n  const declaredModes: CliConfigMode[] = []\n  if (config.projects != null) {\n    declaredModes.push('projects')\n  }\n  if (config.workspaces != null) {\n    declaredModes.push('workspaces')\n  }\n  if (config.comparisons != null) {\n    declaredModes.push('comparisons')\n  }\n\n  if (declaredModes.length > 1) {\n    throw new Error(`Invalid vieval config: top-level keys are mutually exclusive. Found ${declaredModes.join(', ')}.`)\n  }\n\n  return declaredModes[0] ?? 'projects'\n}\n\n/**\n * Loads nearest `vieval.config.*` without project normalization.\n */\nexport async function loadRawVievalConfig(options: LoadVievalCliConfigOptions = {}): Promise<LoadedRawCliConfig> {\n  const cwd = options.cwd ?? process.cwd()\n\n  try {\n    const loadedConfig = await resolveVievalConfig(cwd, options.configFilePath)\n    if (loadedConfig.configFilePath == null || loadedConfig.config == null) {\n      return {\n        config: null,\n        configFilePath: null,\n      }\n    }\n\n    const config = await applyVievalPlugins(loadedConfig.config)\n    detectCliConfigMode(config)\n\n    return {\n      config,\n      configFilePath: loadedConfig.configFilePath,\n    }\n  }\n  catch (error) {\n    const errorMessage = errorMessageFrom(error) ?? 'Unknown config loading error.'\n    const configFilePath = options.configFilePath == null\n      ? 'vieval.config'\n      : (isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath))\n    throw new Error(`Failed to load vieval config \"${configFilePath}\": ${errorMessage}`, { cause: error })\n  }\n}\n\n/**\n * Loads nearest `vieval.config.*` and returns normalized project definitions.\n *\n * Call stack:\n *\n * {@link loadVievalCliConfig}\n *   -> {@link resolveVievalConfig}\n *   -> {@link normalizeConfig}\n *     -> {@link NormalizedCliProjectConfig}[]\n *\n * Use when:\n * - CLI orchestration needs project includes/excludes similar to Vitest\n * - callers want config auto-discovery without manual imports in eval files\n */\nexport async function loadVievalCliConfig(options: LoadVievalCliConfigOptions = {}): Promise<LoadedCliConfig> {\n  const cwd = options.cwd ?? process.cwd()\n  try {\n    const loadedConfig = await loadRawVievalConfig(options)\n    if (loadedConfig.configFilePath == null || loadedConfig.config == null) {\n      return {\n        configFilePath: null,\n        env: {},\n        projects: normalizeConfig(null, cwd),\n      }\n    }\n\n    const config = loadedConfig.config\n\n    return {\n      configFilePath: loadedConfig.configFilePath,\n      env: config.env ?? {},\n      projects: normalizeConfig(config, dirname(loadedConfig.configFilePath)),\n    }\n  }\n  catch (error) {\n    const errorMessage = errorMessageFrom(error) ?? 'Unknown config loading error.'\n    const configFilePath = options.configFilePath == null\n      ? 'vieval.config'\n      : (isAbsolute(options.configFilePath) ? options.configFilePath : resolve(cwd, options.configFilePath))\n    throw new Error(`Failed to load vieval config \"${configFilePath}\": ${errorMessage}`, { cause: error })\n  }\n}\n","import type { EvalDefinition } from '../config'\n\nimport process from 'node:process'\n\ninterface EvalDefinitionRegistryStore {\n  activeModuleHref: string | null\n  registeredDefinitionsByModule: Map<string, EvalDefinition[]>\n}\n\nconst registryStoreSymbol = Symbol.for('vieval.dsl.registry.store')\n\nfunction getRegistryStore(): EvalDefinitionRegistryStore {\n  const processWithStore = process as NodeJS.Process & {\n    [registryStoreSymbol]?: EvalDefinitionRegistryStore\n  }\n\n  processWithStore[registryStoreSymbol] ??= {\n    activeModuleHref: null,\n    registeredDefinitionsByModule: new Map<string, EvalDefinition[]>(),\n  }\n\n  return processWithStore[registryStoreSymbol]\n}\n\n/**\n * Starts module-scoped eval registration collection.\n */\nexport function beginModuleRegistration(moduleHref: string): void {\n  const store = getRegistryStore()\n  store.activeModuleHref = moduleHref\n}\n\n/**\n * Ends module-scoped eval registration collection.\n */\nexport function endModuleRegistration(): void {\n  const store = getRegistryStore()\n  store.activeModuleHref = null\n}\n\n/**\n * Registers one eval definition against the currently active module.\n */\nexport function registerEvalDefinition(definition: EvalDefinition): void {\n  const store = getRegistryStore()\n\n  if (store.activeModuleHref == null) {\n    return\n  }\n\n  const existing = store.registeredDefinitionsByModule.get(store.activeModuleHref) ?? []\n  existing.push(definition)\n  store.registeredDefinitionsByModule.set(store.activeModuleHref, existing)\n}\n\n/**\n * Consumes registered definitions for one module and clears stored state.\n */\nexport function consumeModuleRegistrations(moduleHref: string): EvalDefinition[] {\n  const store = getRegistryStore()\n  const definitions = store.registeredDefinitionsByModule.get(moduleHref) ?? []\n  store.registeredDefinitionsByModule.delete(moduleHref)\n  return definitions\n}\n"],"mappings":";;;;;;;;;AAiBA,MAAM,kBAAkB,IAAI,IAAI;CAAC;CAAW;CAAU;CAAW,CAAC;AAClE,MAAM,wCAAwC;AAC9C,MAAM,UAAU,cAAc,OAAO,KAAK,IAAI;;;;AAwS9C,MAAa,eAAe,oBAA+B;;;;;;;;;;;;;;;AAgB3D,SAAgBA,UAAQ,MAAc,QAAgB,WAA8B,IAAuB;AACzG,QAAOC,QAAY,MAAM,QAAQ,SAAS;;AAG5C,eAAe,mBAAmB,QAAuC;CACvE,IAAI,gBAA2B;CAC/B,MAAM,UAAU,cAAc,WAAW,EAAE;AAE3C,MAAK,MAAM,UAAU,SAAS;AAC5B,MAAI,OAAO,gBAAgB,KACzB;EAGF,MAAM,aAAa,MAAM,OAAO,aAAa,cAAc;AAC3D,MAAI,cAAc,KAChB,iBAAgB;GACd,GAAG;GACH,GAAG;GACJ;;AAIL,MAAK,MAAM,UAAU,QACnB,OAAM,OAAO,uBAAuB,cAAc;AAGpD,QAAO;;AAGT,eAAe,eAAe,UAAoC;AAChE,KAAI;AACF,QAAM,OAAO,SAAS;AACtB,SAAO;SAEH;AACJ,SAAO;;;AAIX,SAAS,kCAAkC,WAA4B;AACrE,QAAO,cAAc,UAAU,cAAc;;AAG/C,SAAS,oCAAoC,WAA4B;AACvE,QAAO,cAAc;;AAGvB,eAAe,yBAAyB,UAAoC;CAC1E,MAAM,YAAY,QAAQ,SAAS;AAEnC,KAAI,oCAAoC,UAAU,EAAE;EAClD,MAAM,MAAM,MAAM,SAAS,UAAU,QAAQ;AAC7C,SAAO,KAAK,MAAM,IAAI;;AAGxB,KAAI,kCAAkC,UAAU,CAC9C,QAAO,QAAQ,SAAS;AAG1B,QAAO,OAAO,cAAc,SAAS,CAAC;;AAGxC,SAAS,oBAAoB,aAA+B;AAC1D,KAAI,eAAe,KACjB,QAAO;AAGT,KAAI,OAAO,gBAAgB,SACzB,QAAO;AAGT,KAAI,aAAa,YACf,QAAQ,YAAqC;AAG/C,QAAO;;AAGT,eAAe,sBAAsB,gBAAgD;CACnF,MAAM,qBAAqB;EACzB;EACA;EACA;EACA;EACA;EACA;EACA;EACD;CAED,IAAI,mBAAmB,QAAQ,eAAe;AAE9C,QAAO,MAAM;AACX,OAAK,MAAM,YAAY,oBAAoB;GACzC,MAAM,gBAAgB,KAAK,kBAAkB,SAAS;AACtD,OAAI,MAAM,eAAe,cAAc,CACrC,QAAO;;EAIX,MAAM,kBAAkB,QAAQ,iBAAiB;AACjD,MAAI,oBAAoB,iBACtB,QAAO;AAET,qBAAmB;;;AAIvB,eAAe,oBACb,KACA,wBAIC;CACD,MAAM,yBAAyB,0BAA0B,OACrD,MAAM,sBAAsB,IAAI,GAC/B,WAAW,uBAAuB,GAAG,yBAAyB,QAAQ,KAAK,uBAAuB;AAEvG,KAAI,0BAA0B,QAAQ,0BAA0B,QAAQ,CAAC,MAAM,eAAe,uBAAuB,CACnH,OAAM,IAAI,MAAM,kDAAkD,yBAAyB;AAG7F,KAAI,0BAA0B,KAC5B,QAAO;EACL,QAAQ;EACR,gBAAgB;EACjB;AAcH,QAAO;EACL,SAZa,MAAM,WAAsB;GACzC,YAAY;GACZ;GACA,QAAQ;GACR,SAAS;GACT,QAAQ;GACR,QAAQ;GACR,aAAa;GACb,QAAQ;GACR,eAAe;GAChB,CAAC,EAEe;EACf,gBAAgB;EACjB;;AAGH,SAAS,wBAAwB,QAA+D;CAC9F,MAAM,aAAa,OAAO,KAAK,OAAO;AACtC,QACE,WAAW,SAAS,KACjB,WAAW,OAAM,QAAO,gBAAgB,IAAI,IAAI,CAAC;;AAIxD,SAAS,mCAAmC,QAA8C;CACxF,MAAM,aAAa,OAAO,KAAK,OAAO;CACtC,MAAM,kBAAkB,WAAW,MAAK,QAAO,gBAAgB,IAAI,IAAI,CAAC;CACxE,MAAM,cAAc,WAAW,MAAK,QAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC;AAErE,KAAI,mBAAmB,YACrB,OAAM,IAAI,UAAU,sCAAsC;;AAI9D,SAAS,0BAA0B,QAA6E;AAC9G,KAAI,UAAU,KACZ;AAGF,oCAAmC,OAAO;AAE1C,KAAI,wBAAwB,OAAO,CACjC,QAAO;AAGT,QAAO,EACL,QAAQ,QACT;;AAGH,SAAS,uBACP,SACA,KACA,iBACA,6BAC4B;CAC5B,MAAM,UAAU,QAAQ,WAAW;EACjC;EACA;EACA;EACA;EACA;EACA;EACD;CACD,MAAM,UAAU,QAAQ,WAAW;EACjC;EACA;EACA;EACD;CACD,MAAM,SAAS,QAAQ,UAAU,CAAC,GAAG,gBAAgB;CACrD,MAAM,qBAAqB,QAAQ,sBAAsB,CAAC,EAAE,IAAI,WAAW,CAAC;CAC5E,MAAM,OAAO,QAAQ,QAAQ,OACzB,MACC,WAAW,QAAQ,KAAK,GAAG,QAAQ,OAAO,QAAQ,KAAK,QAAQ,KAAK;CACzE,MAAM,YAAY,QAAQ,aAAa,CAAC,GAAG,4BAA4B;AAEvE,QAAO;EACL;EACA,UAAU,QAAQ;EAClB;EACA,YAAY,0BAA0B,QAAQ,WAAW;EACzD;EACA,MAAM,QAAQ;EACd;EACA;EACA,WAAW,0BAA0B,QAAQ,UAAU;EACvD;EACD;;AAGH,SAAS,gBAAgB,QAAsC,KAA2C;AACxG,KAAI,UAAU,MAAM;EAClB,MAAM,OAAO,oBAAoB,OAAO;AACxC,MAAI,SAAS,cACX,OAAM,IAAI,MAAM,4EAA4E;AAE9F,MAAI,SAAS,aACX,OAAM,IAAI,MAAM,2EAA2E;;CAI/F,MAAM,WAAW,QAAQ,YAAY,CAAC,EAAE,MAAM,WAAW,CAAC;CAC1D,MAAM,kBAAkB,QAAQ,UAAU,EAAE;CAC5C,MAAM,8BAA8B,QAAQ,aAAa,EAAE;AAE3D,QAAO,SAAS,KAAI,YAAW,uBAAuB,SAAS,KAAK,iBAAiB,4BAA4B,CAAC;;;;;;;;;;;AAYpH,SAAgB,oBAAoB,QAAkC;CACpE,MAAM,gBAAiC,EAAE;AACzC,KAAI,OAAO,YAAY,KACrB,eAAc,KAAK,WAAW;AAEhC,KAAI,OAAO,cAAc,KACvB,eAAc,KAAK,aAAa;AAElC,KAAI,OAAO,eAAe,KACxB,eAAc,KAAK,cAAc;AAGnC,KAAI,cAAc,SAAS,EACzB,OAAM,IAAI,MAAM,uEAAuE,cAAc,KAAK,KAAK,CAAC,GAAG;AAGrH,QAAO,cAAc,MAAM;;;;;AAM7B,eAAsB,oBAAoB,UAAsC,EAAE,EAA+B;CAC/G,MAAM,MAAM,QAAQ,OAAO,QAAQ,KAAK;AAExC,KAAI;EACF,MAAM,eAAe,MAAM,oBAAoB,KAAK,QAAQ,eAAe;AAC3E,MAAI,aAAa,kBAAkB,QAAQ,aAAa,UAAU,KAChE,QAAO;GACL,QAAQ;GACR,gBAAgB;GACjB;EAGH,MAAM,SAAS,MAAM,mBAAmB,aAAa,OAAO;AAC5D,sBAAoB,OAAO;AAE3B,SAAO;GACL;GACA,gBAAgB,aAAa;GAC9B;UAEI,OAAO;EACZ,MAAM,eAAe,iBAAiB,MAAM,IAAI;EAChD,MAAM,iBAAiB,QAAQ,kBAAkB,OAC7C,kBACC,WAAW,QAAQ,eAAe,GAAG,QAAQ,iBAAiB,QAAQ,KAAK,QAAQ,eAAe;AACvG,QAAM,IAAI,MAAM,iCAAiC,eAAe,KAAK,gBAAgB,EAAE,OAAO,OAAO,CAAC;;;;;;;;;;;;;;;;;AAkB1G,eAAsB,oBAAoB,UAAsC,EAAE,EAA4B;CAC5G,MAAM,MAAM,QAAQ,OAAO,QAAQ,KAAK;AACxC,KAAI;EACF,MAAM,eAAe,MAAM,oBAAoB,QAAQ;AACvD,MAAI,aAAa,kBAAkB,QAAQ,aAAa,UAAU,KAChE,QAAO;GACL,gBAAgB;GAChB,KAAK,EAAE;GACP,UAAU,gBAAgB,MAAM,IAAI;GACrC;EAGH,MAAM,SAAS,aAAa;AAE5B,SAAO;GACL,gBAAgB,aAAa;GAC7B,KAAK,OAAO,OAAO,EAAE;GACrB,UAAU,gBAAgB,QAAQ,QAAQ,aAAa,eAAe,CAAC;GACxE;UAEI,OAAO;EACZ,MAAM,eAAe,iBAAiB,MAAM,IAAI;EAChD,MAAM,iBAAiB,QAAQ,kBAAkB,OAC7C,kBACC,WAAW,QAAQ,eAAe,GAAG,QAAQ,iBAAiB,QAAQ,KAAK,QAAQ,eAAe;AACvG,QAAM,IAAI,MAAM,iCAAiC,eAAe,KAAK,gBAAgB,EAAE,OAAO,OAAO,CAAC;;;;;ACnpB1G,MAAM,sBAAsB,OAAO,IAAI,4BAA4B;AAEnE,SAAS,mBAAgD;CACvD,MAAM,mBAAmB;AAIzB,kBAAiB,yBAAyB;EACxC,kBAAkB;EAClB,+CAA+B,IAAI,KAA+B;EACnE;AAED,QAAO,iBAAiB;;;;;AAM1B,SAAgB,wBAAwB,YAA0B;CAChE,MAAM,QAAQ,kBAAkB;AAChC,OAAM,mBAAmB;;;;;AAM3B,SAAgB,wBAA8B;CAC5C,MAAM,QAAQ,kBAAkB;AAChC,OAAM,mBAAmB;;;;;AAM3B,SAAgB,uBAAuB,YAAkC;CACvE,MAAM,QAAQ,kBAAkB;AAEhC,KAAI,MAAM,oBAAoB,KAC5B;CAGF,MAAM,WAAW,MAAM,8BAA8B,IAAI,MAAM,iBAAiB,IAAI,EAAE;AACtF,UAAS,KAAK,WAAW;AACzB,OAAM,8BAA8B,IAAI,MAAM,kBAAkB,SAAS;;;;;AAM3E,SAAgB,2BAA2B,YAAsC;CAC/E,MAAM,QAAQ,kBAAkB;CAChC,MAAM,cAAc,MAAM,8BAA8B,IAAI,WAAW,IAAI,EAAE;AAC7E,OAAM,8BAA8B,OAAO,WAAW;AACtD,QAAO"}