vieval 0.0.11 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +31 -31
  2. package/dist/bin/vieval.mjs +1 -1
  3. package/dist/cli/index.d.mts +1 -1
  4. package/dist/cli/index.mjs +1 -1
  5. package/dist/{cli-CHFCF8UR.mjs → cli-uzS81IPd.mjs} +1529 -1529
  6. package/dist/cli-uzS81IPd.mjs.map +1 -0
  7. package/dist/config.d.mts +1 -1
  8. package/dist/core/assertions/index.d.mts +156 -156
  9. package/dist/core/assertions/index.mjs +82 -82
  10. package/dist/core/assertions/index.mjs.map +1 -1
  11. package/dist/core/inference-executors/index.d.mts +37 -37
  12. package/dist/core/inference-executors/index.mjs +53 -52
  13. package/dist/core/inference-executors/index.mjs.map +1 -1
  14. package/dist/core/processors/results/index.d.mts +18 -18
  15. package/dist/core/processors/results/index.mjs.map +1 -1
  16. package/dist/core/runner/index.d.mts +2 -2
  17. package/dist/core/runner/index.mjs +258 -258
  18. package/dist/core/runner/index.mjs.map +1 -1
  19. package/dist/core/scheduler/index.d.mts +1 -1
  20. package/dist/core/scheduler/index.mjs +64 -64
  21. package/dist/core/scheduler/index.mjs.map +1 -1
  22. package/dist/{env-bRH0K6fU.d.mts → env-Br6jaWGL.d.mts} +9 -9
  23. package/dist/{env-BVYeJhGA.mjs → env-egxaJtNn.mjs} +8 -8
  24. package/dist/env-egxaJtNn.mjs.map +1 -0
  25. package/dist/{expect-extensions-Mf1sMNBv.mjs → expect-extensions-BKdEPt3h.mjs} +46 -46
  26. package/dist/expect-extensions-BKdEPt3h.mjs.map +1 -0
  27. package/dist/expect.mjs +1 -1
  28. package/dist/{index-CwKBlCG9.d.mts → index-BLIlhiWT.d.mts} +565 -565
  29. package/dist/{index-Be5I1ZJL.d.mts → index-CIaJClcC.d.mts} +48 -48
  30. package/dist/index.d.mts +207 -195
  31. package/dist/index.mjs +147 -147
  32. package/dist/index.mjs.map +1 -1
  33. package/dist/models-CaCOUPZw.mjs.map +1 -1
  34. package/dist/plugins/chat-models/index.d.mts +279 -279
  35. package/dist/plugins/chat-models/index.mjs +359 -359
  36. package/dist/plugins/chat-models/index.mjs.map +1 -1
  37. package/dist/{registry-BSyjwZFx.mjs → registry-BK7k6X81.mjs} +293 -293
  38. package/dist/registry-BK7k6X81.mjs.map +1 -0
  39. package/dist/testing/expect-extensions.d.mts +27 -27
  40. package/dist/testing/expect-extensions.mjs +1 -1
  41. package/package.json +3 -3
  42. package/dist/cli-CHFCF8UR.mjs.map +0 -1
  43. package/dist/env-BVYeJhGA.mjs.map +0 -1
  44. package/dist/expect-extensions-Mf1sMNBv.mjs.map +0 -1
  45. package/dist/registry-BSyjwZFx.mjs.map +0 -1
@@ -1,22 +1,14 @@
1
- import { D as TaskRunContext, R as ModelDefinition, f as MatrixDefinition, t as ConfigHookPlugin, w as TaskExecutionPolicy } from "../../index-CwKBlCG9.mjs";
1
+ import { D as TaskRunContext, R as ModelDefinition, f as MatrixDefinition, t as ConfigHookPlugin, w as TaskExecutionPolicy } from "../../index-BLIlhiWT.mjs";
2
2
 
3
3
  //#region src/plugins/chat-models/runtime-config.d.ts
4
4
  /**
5
- * Runtime config consumed by OpenAI-compatible provider constructors.
5
+ * Union of normalized runtime configs for supported chat-model executors.
6
6
  */
7
- interface OpenAIChatModelRuntimeConfig {
8
- /**
9
- * Resolved inference executor kind.
10
- */
11
- inferenceExecutor: 'openai';
12
- /**
13
- * Concrete model name.
14
- */
15
- model: string;
16
- /**
17
- * Required API key.
18
- */
19
- apiKey: string;
7
+ type ChatModelRuntimeConfig = OllamaChatModelRuntimeConfig | OpenAIChatModelRuntimeConfig | OpenRouterChatModelRuntimeConfig;
8
+ /**
9
+ * Runtime config consumed by Ollama provider constructors.
10
+ */
11
+ interface OllamaChatModelRuntimeConfig {
20
12
  /**
21
13
  * Optional base URL override.
22
14
  */
@@ -25,11 +17,6 @@ interface OpenAIChatModelRuntimeConfig {
25
17
  * Optional request headers.
26
18
  */
27
19
  headers?: ChatModelHeaders;
28
- }
29
- /**
30
- * Runtime config consumed by Ollama provider constructors.
31
- */
32
- interface OllamaChatModelRuntimeConfig {
33
20
  /**
34
21
  * Resolved inference executor kind.
35
22
  */
@@ -38,6 +25,15 @@ interface OllamaChatModelRuntimeConfig {
38
25
  * Concrete model name.
39
26
  */
40
27
  model: string;
28
+ }
29
+ /**
30
+ * Runtime config consumed by OpenAI-compatible provider constructors.
31
+ */
32
+ interface OpenAIChatModelRuntimeConfig {
33
+ /**
34
+ * Required API key.
35
+ */
36
+ apiKey: string;
41
37
  /**
42
38
  * Optional base URL override.
43
39
  */
@@ -46,19 +42,19 @@ interface OllamaChatModelRuntimeConfig {
46
42
  * Optional request headers.
47
43
  */
48
44
  headers?: ChatModelHeaders;
49
- }
50
- /**
51
- * Runtime config consumed by OpenRouter provider constructors.
52
- */
53
- interface OpenRouterChatModelRuntimeConfig {
54
45
  /**
55
46
  * Resolved inference executor kind.
56
47
  */
57
- inferenceExecutor: 'openrouter';
48
+ inferenceExecutor: 'openai';
58
49
  /**
59
50
  * Concrete model name.
60
51
  */
61
52
  model: string;
53
+ }
54
+ /**
55
+ * Runtime config consumed by OpenRouter provider constructors.
56
+ */
57
+ interface OpenRouterChatModelRuntimeConfig {
62
58
  /**
63
59
  * Required API key.
64
60
  */
@@ -71,26 +67,29 @@ interface OpenRouterChatModelRuntimeConfig {
71
67
  * Optional request headers.
72
68
  */
73
69
  headers?: ChatModelHeaders;
70
+ /**
71
+ * Resolved inference executor kind.
72
+ */
73
+ inferenceExecutor: 'openrouter';
74
+ /**
75
+ * Concrete model name.
76
+ */
77
+ model: string;
74
78
  }
75
79
  /**
76
- * Union of normalized runtime configs for supported chat-model executors.
77
- */
78
- type ChatModelRuntimeConfig = OpenAIChatModelRuntimeConfig | OllamaChatModelRuntimeConfig | OpenRouterChatModelRuntimeConfig;
79
- /**
80
- * Normalizes one configured chat model into runtime executor config.
80
+ * Resolves Ollama runtime config from one resolved run-context model.
81
81
  *
82
82
  * Use when:
83
- * - eval code needs typed provider constructor options from a resolved model
84
- * - model parameters should be validated once with clear error messages
83
+ * - task execution already has a model resolved through chat-model helpers
84
+ * - eval code wants typed Ollama provider options with a concise helper name
85
85
  *
86
86
  * Expects:
87
- * - `model.inferenceExecutorId` to be one of the supported executor ids
88
- * - required OpenAI fields (apiKey) to exist in `model.parameters`
87
+ * - `model` to resolve to an Ollama-backed chat model
89
88
  *
90
89
  * Returns:
91
- * - validated runtime config union for OpenAI or Ollama
90
+ * - validated Ollama runtime config
92
91
  */
93
- declare function toChatModelRuntimeConfig(model: ModelDefinition): ChatModelRuntimeConfig;
92
+ declare function ollamaFromRunContext(model: ModelDefinition): OllamaChatModelRuntimeConfig;
94
93
  /**
95
94
  * Resolves OpenAI runtime config from one resolved run-context model.
96
95
  *
@@ -106,35 +105,49 @@ declare function toChatModelRuntimeConfig(model: ModelDefinition): ChatModelRunt
106
105
  */
107
106
  declare function openaiFromRunContext(model: ModelDefinition): OpenAIChatModelRuntimeConfig;
108
107
  /**
109
- * Resolves Ollama runtime config from one resolved run-context model.
108
+ * Resolves OpenRouter runtime config from one resolved run-context model.
110
109
  *
111
110
  * Use when:
112
111
  * - task execution already has a model resolved through chat-model helpers
113
- * - eval code wants typed Ollama provider options with a concise helper name
112
+ * - eval code wants typed OpenRouter provider options with a concise helper name
114
113
  *
115
114
  * Expects:
116
- * - `model` to resolve to an Ollama-backed chat model
115
+ * - `model` to resolve to an OpenRouter-backed chat model
117
116
  *
118
117
  * Returns:
119
- * - validated Ollama runtime config
118
+ * - validated OpenRouter runtime config
120
119
  */
121
- declare function ollamaFromRunContext(model: ModelDefinition): OllamaChatModelRuntimeConfig;
120
+ declare function openrouterFromRunContext(model: ModelDefinition): OpenRouterChatModelRuntimeConfig;
122
121
  /**
123
- * Resolves OpenRouter runtime config from one resolved run-context model.
122
+ * Normalizes one configured chat model into runtime executor config.
124
123
  *
125
124
  * Use when:
126
- * - task execution already has a model resolved through chat-model helpers
127
- * - eval code wants typed OpenRouter provider options with a concise helper name
125
+ * - eval code needs typed provider constructor options from a resolved model
126
+ * - model parameters should be validated once with clear error messages
128
127
  *
129
128
  * Expects:
130
- * - `model` to resolve to an OpenRouter-backed chat model
129
+ * - `model.inferenceExecutorId` to be one of the supported executor ids
130
+ * - required OpenAI fields (apiKey) to exist in `model.parameters`
131
131
  *
132
132
  * Returns:
133
- * - validated OpenRouter runtime config
133
+ * - validated runtime config union for OpenAI or Ollama
134
134
  */
135
- declare function openrouterFromRunContext(model: ModelDefinition): OpenRouterChatModelRuntimeConfig;
135
+ declare function toChatModelRuntimeConfig(model: ModelDefinition): ChatModelRuntimeConfig;
136
136
  //#endregion
137
137
  //#region src/plugins/chat-models/telemetry.d.ts
138
+ /**
139
+ * Provider identity attached to chat-model telemetry events.
140
+ */
141
+ interface ChatModelTelemetryProvider {
142
+ /**
143
+ * Provider id, for example `openai`.
144
+ */
145
+ id: string;
146
+ /**
147
+ * Optional concrete model id/name.
148
+ */
149
+ model?: string;
150
+ }
138
151
  /**
139
152
  * Represents one normalized chat-model tool call.
140
153
  *
@@ -146,52 +159,35 @@ declare function openrouterFromRunContext(model: ModelDefinition): OpenRouterCha
146
159
  * - `args` to be JSON-serializable
147
160
  */
148
161
  interface ChatModelToolCall {
149
- /**
150
- * Optional provider-assigned tool-call identifier.
151
- */
152
- id?: string;
153
- /**
154
- * Tool name.
155
- */
156
- name: string;
157
162
  /**
158
163
  * Parsed tool arguments object/value.
159
164
  */
160
165
  args: unknown;
161
- }
162
- /**
163
- * Provider identity attached to chat-model telemetry events.
164
- */
165
- interface ChatModelTelemetryProvider {
166
166
  /**
167
- * Provider id, for example `openai`.
167
+ * Optional provider-assigned tool-call identifier.
168
168
  */
169
- id: string;
169
+ id?: string;
170
170
  /**
171
- * Optional concrete model id/name.
171
+ * Tool name.
172
172
  */
173
- model?: string;
173
+ name: string;
174
174
  }
175
175
  /**
176
- * Input options for response telemetry emission.
176
+ * Input options for error telemetry emission.
177
177
  */
178
- interface EmitChatModelResponseTelemetryOptions {
178
+ interface EmitChatModelErrorTelemetryOptions {
179
179
  /**
180
180
  * Optional case id for case-scoped telemetry events.
181
181
  */
182
182
  caseId?: string;
183
183
  /**
184
- * Optional response latency in milliseconds.
184
+ * Error payload emitted by the inference client/runtime.
185
185
  */
186
- latencyMs?: number;
186
+ error: unknown;
187
187
  /**
188
188
  * Optional provider identity payload.
189
189
  */
190
190
  provider?: ChatModelTelemetryProvider;
191
- /**
192
- * Raw chat-model response object from the inference library/provider.
193
- */
194
- response: unknown;
195
191
  }
196
192
  /**
197
193
  * Input options for request telemetry emission.
@@ -211,43 +207,46 @@ interface EmitChatModelRequestTelemetryOptions {
211
207
  provider?: ChatModelTelemetryProvider;
212
208
  }
213
209
  /**
214
- * Input options for error telemetry emission.
210
+ * Input options for response telemetry emission.
215
211
  */
216
- interface EmitChatModelErrorTelemetryOptions {
212
+ interface EmitChatModelResponseTelemetryOptions {
217
213
  /**
218
214
  * Optional case id for case-scoped telemetry events.
219
215
  */
220
216
  caseId?: string;
221
217
  /**
222
- * Error payload emitted by the inference client/runtime.
218
+ * Optional response latency in milliseconds.
223
219
  */
224
- error: unknown;
220
+ latencyMs?: number;
225
221
  /**
226
222
  * Optional provider identity payload.
227
223
  */
228
224
  provider?: ChatModelTelemetryProvider;
225
+ /**
226
+ * Raw chat-model response object from the inference library/provider.
227
+ */
228
+ response: unknown;
229
229
  }
230
230
  /**
231
- * Extracts normalized tool calls from one chat-model response shape.
231
+ * Emits chat-model failure telemetry as a reportable task event.
232
232
  *
233
233
  * Use when:
234
- * - downstream scoring, reporting, or analysis should inspect tool call usage
235
- * - provider payload differences should stay hidden behind one stable shape
234
+ * - one inference call fails and report artifacts should include normalized error context
236
235
  *
237
- * Returns:
238
- * - normalized list of `{ id?, name, args }` tool calls
236
+ * Expects:
237
+ * - `context.reporterHooks?.onEvent` to be available in CLI execution paths
239
238
  */
240
- declare function extractChatModelToolCalls(response: unknown): ChatModelToolCall[];
239
+ declare function emitChatModelErrorTelemetry(context: TaskRunContext, options: EmitChatModelErrorTelemetryOptions): void;
241
240
  /**
242
- * Extracts numeric metering dimensions from one chat-model response usage block.
241
+ * Emits chat-model request telemetry as a reportable task event.
243
242
  *
244
243
  * Use when:
245
- * - report events should capture usage dimensions in a modality-neutral map
244
+ * - task code submits one model request and wants request-side traceability
246
245
  *
247
- * Returns:
248
- * - numeric dimensions keyed by provider usage field names
246
+ * Expects:
247
+ * - `context.reporterHooks?.onEvent` to be available in CLI execution paths
249
248
  */
250
- declare function extractMeteringDimensions(response: unknown): Record<string, number>;
249
+ declare function emitChatModelRequestTelemetry(context: TaskRunContext, options: EmitChatModelRequestTelemetryOptions): void;
251
250
  /**
252
251
  * Emits chat-model response telemetry as reportable task events.
253
252
  *
@@ -263,115 +262,80 @@ declare function extractMeteringDimensions(response: unknown): Record<string, nu
263
262
  */
264
263
  declare function emitChatModelResponseTelemetry(context: TaskRunContext, options: EmitChatModelResponseTelemetryOptions): void;
265
264
  /**
266
- * Emits chat-model request telemetry as a reportable task event.
265
+ * Extracts normalized tool calls from one chat-model response shape.
267
266
  *
268
267
  * Use when:
269
- * - task code submits one model request and wants request-side traceability
268
+ * - downstream scoring, reporting, or analysis should inspect tool call usage
269
+ * - provider payload differences should stay hidden behind one stable shape
270
270
  *
271
- * Expects:
272
- * - `context.reporterHooks?.onEvent` to be available in CLI execution paths
271
+ * Returns:
272
+ * - normalized list of `{ id?, name, args }` tool calls
273
273
  */
274
- declare function emitChatModelRequestTelemetry(context: TaskRunContext, options: EmitChatModelRequestTelemetryOptions): void;
274
+ declare function extractChatModelToolCalls(response: unknown): ChatModelToolCall[];
275
275
  /**
276
- * Emits chat-model failure telemetry as a reportable task event.
276
+ * Extracts numeric metering dimensions from one chat-model response usage block.
277
277
  *
278
278
  * Use when:
279
- * - one inference call fails and report artifacts should include normalized error context
279
+ * - report events should capture usage dimensions in a modality-neutral map
280
280
  *
281
- * Expects:
282
- * - `context.reporterHooks?.onEvent` to be available in CLI execution paths
281
+ * Returns:
282
+ * - numeric dimensions keyed by provider usage field names
283
283
  */
284
- declare function emitChatModelErrorTelemetry(context: TaskRunContext, options: EmitChatModelErrorTelemetryOptions): void;
284
+ declare function extractMeteringDimensions(response: unknown): Record<string, number>;
285
285
  //#endregion
286
286
  //#region src/plugins/chat-models/index.d.ts
287
287
  /**
288
- * Minimal inference-executor shape expected by chat model runtime callers.
288
+ * Chat-model specific specialization of the canonical `ModelDefinition`.
289
289
  */
290
- interface ChatModelExecutorLike {
291
- chat: (model: string) => Record<string, unknown>;
292
- }
290
+ type ChatModelDefinition = Omit<ModelDefinition, 'inferenceExecutor'> & {
291
+ inferenceExecutor: ChatModelExecutorInput;
292
+ provider?: string;
293
+ runtimeResolvers?: {
294
+ apiKey?: ChatModelResolverValue<string>;
295
+ baseURL?: ChatModelResolverValue<string>;
296
+ headers?: ChatModelResolverValue<ChatModelHeaders>;
297
+ };
298
+ };
293
299
  /**
294
300
  * Inference-executor input accepted by `chatModelFrom`.
295
301
  */
296
- type ChatModelExecutorInput = string | ChatModelExecutorLike;
297
- /**
298
- * Chat-model header payload accepted by executor parameters.
299
- */
300
- type ChatModelHeaders = Record<string, string | string[]>;
301
- /**
302
- * Runtime env context passed to model callback resolvers.
303
- */
304
- interface ChatModelResolverContext {
305
- env: Record<string, string>;
306
- }
307
- /**
308
- * Value-or-callback resolver used by model runtime fields.
309
- */
310
- type ChatModelResolverValue<TValue> = TValue | ((config: ChatModelResolverContext) => Promise<TValue> | TValue);
311
- /**
312
- * OpenAI-specific inference executor config shape.
313
- */
314
- interface OpenAIChatModelInferenceExecutor {
315
- inferenceExecutor: 'openai';
316
- apiKey?: ChatModelResolverValue<string>;
317
- baseURL?: ChatModelResolverValue<string>;
318
- headers?: ChatModelResolverValue<ChatModelHeaders>;
319
- }
320
- /**
321
- * Ollama-specific inference executor config shape.
322
- */
323
- interface OllamaChatModelInferenceExecutor {
324
- inferenceExecutor: 'ollama';
325
- baseURL?: ChatModelResolverValue<string>;
326
- headers?: ChatModelResolverValue<ChatModelHeaders>;
327
- }
328
- /**
329
- * OpenRouter-specific inference executor config shape.
330
- */
331
- interface OpenRouterChatModelInferenceExecutor {
332
- inferenceExecutor: 'openrouter';
333
- apiKey?: ChatModelResolverValue<string>;
334
- baseURL?: ChatModelResolverValue<string>;
335
- headers?: ChatModelResolverValue<ChatModelHeaders>;
336
- }
302
+ type ChatModelExecutorInput = ChatModelExecutorLike | string;
337
303
  /**
338
- * Generic inference executor config shape.
304
+ * Minimal inference-executor shape expected by chat model runtime callers.
339
305
  */
340
- interface GenericChatModelInferenceExecutor {
341
- inferenceExecutor?: ChatModelExecutorInput;
306
+ interface ChatModelExecutorLike {
307
+ chat: (model: string) => Record<string, unknown>;
342
308
  }
343
- /**
344
- * Union of supported inference executor config shapes for `chatModelFrom`.
345
- */
346
- type ChatModelInferenceExecutor = OpenAIChatModelInferenceExecutor | OllamaChatModelInferenceExecutor | OpenRouterChatModelInferenceExecutor | GenericChatModelInferenceExecutor;
347
309
  /**
348
310
  * Common builder input fields for `chatModelFrom`.
349
311
  */
350
312
  interface ChatModelFromBaseOptions {
351
313
  /**
352
- * Provider id registered through `ChatProviders`.
353
- *
354
- * Use when:
355
- * - model runtime transport and credentials should be delegated to a named provider preset
314
+ * Alias names used by `resolveModelByName`.
315
+ */
316
+ aliases?: string[];
317
+ /**
318
+ * Additional full task attempts allowed after the current attempt settles.
356
319
  *
357
- * Expects:
358
- * - one `ChatProviders` plugin entry to expose the same id
320
+ * @default 0
359
321
  */
360
- provider?: string;
322
+ autoAttempt?: number;
361
323
  /**
362
- * Inference-executor id or inference-executor instance.
324
+ * Additional retries allowed within the current attempt.
325
+ *
326
+ * @default 0
363
327
  */
364
- inferenceExecutor?: ChatModelExecutorInput;
328
+ autoRetry?: number;
365
329
  /**
366
- * Optional explicit inference-executor id for inference-executor instances.
330
+ * Delay in milliseconds before a retry starts.
367
331
  *
368
- * @default 'custom'
332
+ * @default retryIndex => 500 * 2 ** (retryIndex - 1)
369
333
  */
370
- inferenceExecutorId?: string;
334
+ autoRetryDelay?: TaskExecutionPolicy['autoRetryDelay'];
371
335
  /**
372
- * Concrete model name.
336
+ * Optional execution policy hints attached to this model.
373
337
  */
374
- model: string;
338
+ executionPolicy?: TaskExecutionPolicy;
375
339
  /**
376
340
  * Optional stable model id.
377
341
  *
@@ -379,72 +343,69 @@ interface ChatModelFromBaseOptions {
379
343
  */
380
344
  id?: string;
381
345
  /**
382
- * Alias names used by `resolveModelByName`.
346
+ * Inference-executor id or inference-executor instance.
383
347
  */
384
- aliases?: string[];
348
+ inferenceExecutor?: ChatModelExecutorInput;
385
349
  /**
386
- * Optional execution policy hints attached to this model.
350
+ * Optional explicit inference-executor id for inference-executor instances.
351
+ *
352
+ * @default 'custom'
387
353
  */
388
- executionPolicy?: TaskExecutionPolicy;
354
+ inferenceExecutorId?: string;
389
355
  /**
390
- * Additional retries allowed within the current attempt.
391
- *
392
- * @default 0
356
+ * Concrete model name.
393
357
  */
394
- autoRetry?: number;
358
+ model: string;
395
359
  /**
396
- * Delay in milliseconds before a retry starts.
397
- *
398
- * @default retryIndex => 500 * 2 ** (retryIndex - 1)
360
+ * Optional model-level call parameters.
399
361
  */
400
- autoRetryDelay?: TaskExecutionPolicy['autoRetryDelay'];
362
+ parameters?: Record<string, unknown>;
401
363
  /**
402
- * Additional full task attempts allowed after the current attempt settles.
364
+ * Provider id registered through `ChatProviders`.
403
365
  *
404
- * @default 0
366
+ * Use when:
367
+ * - model runtime transport and credentials should be delegated to a named provider preset
368
+ *
369
+ * Expects:
370
+ * - one `ChatProviders` plugin entry to expose the same id
405
371
  */
406
- autoAttempt?: number;
372
+ provider?: string;
407
373
  /**
408
374
  * Timeout in milliseconds for model-backed work.
409
375
  */
410
376
  timeout?: number;
411
- /**
412
- * Optional model-level call parameters.
413
- */
414
- parameters?: Record<string, unknown>;
415
377
  }
416
378
  /**
417
379
  * Builder input for `chatModelFrom`.
418
380
  */
419
- type ChatModelFromOptions = ChatModelInferenceExecutor & ChatModelFromBaseOptions;
381
+ type ChatModelFromOptions = ChatModelFromBaseOptions & ChatModelInferenceExecutor;
420
382
  /**
421
- * Chat-model specific specialization of the canonical `ModelDefinition`.
383
+ * Chat-model header payload accepted by executor parameters.
422
384
  */
423
- type ChatModelDefinition = Omit<ModelDefinition, 'inferenceExecutor'> & {
424
- inferenceExecutor: ChatModelExecutorInput;
425
- provider?: string;
426
- runtimeResolvers?: {
427
- apiKey?: ChatModelResolverValue<string>;
428
- baseURL?: ChatModelResolverValue<string>;
429
- headers?: ChatModelResolverValue<ChatModelHeaders>;
430
- };
431
- };
385
+ type ChatModelHeaders = Record<string, string | string[]>;
432
386
  /**
433
- * Env-key map for optional provider parameters.
434
- *
435
- * Use when:
436
- * - provider parameter values should be read from env keys
437
- * - missing keys should resolve to `undefined`
387
+ * Union of supported inference executor config shapes for `chatModelFrom`.
438
388
  */
439
- type OptionalProviderEnvMap = Record<string, string>;
389
+ type ChatModelInferenceExecutor = GenericChatModelInferenceExecutor | OllamaChatModelInferenceExecutor | OpenAIChatModelInferenceExecutor | OpenRouterChatModelInferenceExecutor;
440
390
  /**
441
- * Env-key map for required provider parameters.
442
- *
443
- * Use when:
444
- * - provider parameter values must exist before model execution
445
- * - missing keys should throw with key-aware error messages
391
+ * Runtime env context passed to model callback resolvers.
446
392
  */
447
- type RequiredProviderEnvMap = Record<string, string>;
393
+ interface ChatModelResolverContext {
394
+ env: Record<string, string>;
395
+ }
396
+ /**
397
+ * Value-or-callback resolver used by model runtime fields.
398
+ */
399
+ type ChatModelResolverValue<TValue> = ((config: ChatModelResolverContext) => Promise<TValue> | TValue) | TValue;
400
+ /**
401
+ * Options for the built-in `ChatModels` plugin.
402
+ */
403
+ interface ChatModelsPluginOptions {
404
+ /**
405
+ * Chat model definitions to append to config.
406
+ */
407
+ models: readonly ChatModelDefinition[];
408
+ }
448
409
  /**
449
410
  * One provider definition consumed by chat model presets.
450
411
  */
@@ -463,10 +424,6 @@ interface ChatProviderDefinition {
463
424
  * @default 'custom'
464
425
  */
465
426
  inferenceExecutorId?: string;
466
- /**
467
- * Optional literal provider-level parameters.
468
- */
469
- parameters?: Record<string, unknown>;
470
427
  /**
471
428
  * Optional provider parameters resolved via `envFrom`.
472
429
  *
@@ -475,6 +432,10 @@ interface ChatProviderDefinition {
475
432
  * - map value is the env key name
476
433
  */
477
434
  optionalEnv?: OptionalProviderEnvMap;
435
+ /**
436
+ * Optional literal provider-level parameters.
437
+ */
438
+ parameters?: Record<string, unknown>;
478
439
  /**
479
440
  * Required provider parameters resolved via `requiredEnvFrom`.
480
441
  *
@@ -492,97 +453,137 @@ interface ChatProviderFromOptions extends ChatProviderDefinition {}
492
453
  * Options for the built-in `ChatProviders` plugin.
493
454
  */
494
455
  interface ChatProvidersPluginOptions {
495
- /**
496
- * Provider definitions to append to config.
497
- */
498
- providers: readonly ChatProviderDefinition[];
499
456
  /**
500
457
  * Optional explicit env source used for env-backed provider parameters.
501
458
  *
502
459
  * @default process.env
503
460
  */
504
461
  env?: NodeJS.ProcessEnv;
462
+ /**
463
+ * Provider definitions to append to config.
464
+ */
465
+ providers: readonly ChatProviderDefinition[];
505
466
  }
506
467
  /**
507
- * Partial config shape needed by the chat models plugin.
468
+ * Generic inference executor config shape.
508
469
  */
509
- interface PluginConfig {
510
- env?: NodeJS.ProcessEnv;
511
- chatProviders?: ChatProviderDefinition[];
512
- models?: ModelDefinition[];
470
+ interface GenericChatModelInferenceExecutor {
471
+ inferenceExecutor?: ChatModelExecutorInput;
513
472
  }
514
473
  /**
515
- * Plugin type bound to the minimal config shape used by model plugins.
474
+ * Matrix scope that can carry a chat model selector.
516
475
  */
517
- type Plugin = ConfigHookPlugin<PluginConfig>;
476
+ type MatrixModelScope = 'eval' | 'run';
518
477
  /**
519
- * Builds one normalized chat model definition.
520
- *
521
- * Use when:
522
- * - registering chat models through config plugins
523
- * - a single model needs aliases for matrix selection or judge lookup
478
+ * Options for resolving a chat model from a matrix axis.
524
479
  */
525
- declare function chatModelFrom(options: ChatModelFromOptions): ChatModelDefinition;
480
+ interface ModelFromMatrixOptions {
481
+ /**
482
+ * Matrix axis whose selected value is a model id, model name, or alias.
483
+ */
484
+ axis: string;
485
+ }
526
486
  /**
527
- * Builds one normalized chat provider definition.
528
- *
529
- * Use when:
530
- * - one provider preset should be reused across multiple chat models
531
- * - provider configuration should support required/optional env-backed parameters
487
+ * Ollama-specific inference executor config shape.
532
488
  */
533
- declare function chatProviderFrom(options: ChatProviderFromOptions): ChatProviderDefinition;
489
+ interface OllamaChatModelInferenceExecutor {
490
+ baseURL?: ChatModelResolverValue<string>;
491
+ headers?: ChatModelResolverValue<ChatModelHeaders>;
492
+ inferenceExecutor: 'ollama';
493
+ }
534
494
  /**
535
- * Options for the built-in `ChatModels` plugin.
495
+ * OpenAI-specific inference executor config shape.
536
496
  */
537
- interface ChatModelsPluginOptions {
538
- /**
539
- * Chat model definitions to append to config.
540
- */
541
- models: readonly ChatModelDefinition[];
497
+ interface OpenAIChatModelInferenceExecutor {
498
+ apiKey?: ChatModelResolverValue<string>;
499
+ baseURL?: ChatModelResolverValue<string>;
500
+ headers?: ChatModelResolverValue<ChatModelHeaders>;
501
+ inferenceExecutor: 'openai';
542
502
  }
543
503
  /**
544
- * Matrix scope that can carry a chat model selector.
504
+ * OpenRouter-specific inference executor config shape.
545
505
  */
546
- type MatrixModelScope = 'eval' | 'run';
506
+ interface OpenRouterChatModelInferenceExecutor {
507
+ apiKey?: ChatModelResolverValue<string>;
508
+ baseURL?: ChatModelResolverValue<string>;
509
+ headers?: ChatModelResolverValue<ChatModelHeaders>;
510
+ inferenceExecutor: 'openrouter';
511
+ }
547
512
  /**
548
- * Options for resolving a chat model from a matrix axis.
513
+ * Env-key map for optional provider parameters.
514
+ *
515
+ * Use when:
516
+ * - provider parameter values should be read from env keys
517
+ * - missing keys should resolve to `undefined`
549
518
  */
550
- interface ModelFromMatrixOptions {
551
- /**
552
- * Matrix axis whose selected value is a model id, model name, or alias.
553
- */
554
- axis: string;
519
+ type OptionalProviderEnvMap = Record<string, string>;
520
+ /**
521
+ * Plugin type bound to the minimal config shape used by model plugins.
522
+ */
523
+ type Plugin = ConfigHookPlugin<PluginConfig>;
524
+ /**
525
+ * Partial config shape needed by the chat models plugin.
526
+ */
527
+ interface PluginConfig {
528
+ chatProviders?: ChatProviderDefinition[];
529
+ env?: NodeJS.ProcessEnv;
530
+ models?: ModelDefinition[];
555
531
  }
532
+ /**
533
+ * Env-key map for required provider parameters.
534
+ *
535
+ * Use when:
536
+ * - provider parameter values must exist before model execution
537
+ * - missing keys should throw with key-aware error messages
538
+ */
539
+ type RequiredProviderEnvMap = Record<string, string>;
556
540
  type MatrixModelContext = Pick<TaskRunContext, 'models' | 'task'>;
557
541
  /**
558
- * Resolves a configured chat model from one scoped matrix axis.
542
+ * Builds one normalized chat model definition.
559
543
  *
560
544
  * Use when:
561
- * - a matrix axis selects the agent, judge, or another chat model role
562
- * - eval code should keep model lookup semantics inside the chat-models plugin
545
+ * - registering chat models through config plugins
546
+ * - a single model needs aliases for matrix selection or judge lookup
547
+ */
548
+ declare function chatModelFrom(options: ChatModelFromOptions): ChatModelDefinition;
549
+ /**
550
+ * Creates a run-matrix `model` axis from configured chat model names.
551
+ *
552
+ * Use when:
553
+ * - run matrix should iterate over explicit chat model ids/aliases
554
+ * - project configs want a concise model-axis helper
563
555
  *
564
556
  * Expects:
565
- * - `scope` to identify `context.task.matrix.run` or `context.task.matrix.eval`
566
- * - `options.axis` to exist and contain a model id, model name, or alias
557
+ * - each provided name to match a configured model id or alias at runtime
567
558
  *
568
559
  * Returns:
569
- * - the configured model matching the selected matrix value
560
+ * - matrix axis object compatible with `runMatrix.extend/override`
570
561
  */
571
- declare function modelFromMatrix(context: MatrixModelContext, scope: MatrixModelScope, options: ModelFromMatrixOptions): ModelDefinition;
562
+ declare function chatModelMatrix(...names: string[]): MatrixDefinition;
572
563
  /**
573
- * Resolves a configured chat model from one run-matrix axis.
564
+ * Built-in chat models plugin that contributes model definitions to vieval config.
574
565
  *
575
566
  * Use when:
576
- * - run matrix selects the model used by the system under evaluation
577
- * - callers want the scoped helper instead of passing `scope: 'run'`
567
+ * - chat-model registration should stay in config-level plugin setup
568
+ * - tasks and assertions resolve models by name or alias at runtime
569
+ */
570
+ declare function ChatModels(options: ChatModelsPluginOptions): Plugin;
571
+ /**
572
+ * Builds one normalized chat provider definition.
578
573
  *
579
- * Expects:
580
- * - `options.axis` to exist in `context.task.matrix.run`
574
+ * Use when:
575
+ * - one provider preset should be reused across multiple chat models
576
+ * - provider configuration should support required/optional env-backed parameters
577
+ */
578
+ declare function chatProviderFrom(options: ChatProviderFromOptions): ChatProviderDefinition;
579
+ /**
580
+ * Built-in chat providers plugin that contributes provider presets to config.
581
581
  *
582
- * Returns:
583
- * - the configured model matching the selected run-matrix value
582
+ * Use when:
583
+ * - provider runtime config should be centralized and reusable
584
+ * - provider parameters should be resolved from env via `envFrom`/`requiredEnvFrom`
584
585
  */
585
- declare function modelFromRun(context: MatrixModelContext, options: ModelFromMatrixOptions): ModelDefinition;
586
+ declare function ChatProviders(options: ChatProvidersPluginOptions): Plugin;
586
587
  /**
587
588
  * Resolves a configured chat model from one eval-matrix axis.
588
589
  *
@@ -598,35 +599,34 @@ declare function modelFromRun(context: MatrixModelContext, options: ModelFromMat
598
599
  */
599
600
  declare function modelFromEval(context: MatrixModelContext, options: ModelFromMatrixOptions): ModelDefinition;
600
601
  /**
601
- * Creates a run-matrix `model` axis from configured chat model names.
602
+ * Resolves a configured chat model from one scoped matrix axis.
602
603
  *
603
604
  * Use when:
604
- * - run matrix should iterate over explicit chat model ids/aliases
605
- * - project configs want a concise model-axis helper
605
+ * - a matrix axis selects the agent, judge, or another chat model role
606
+ * - eval code should keep model lookup semantics inside the chat-models plugin
606
607
  *
607
608
  * Expects:
608
- * - each provided name to match a configured model id or alias at runtime
609
+ * - `scope` to identify `context.task.matrix.run` or `context.task.matrix.eval`
610
+ * - `options.axis` to exist and contain a model id, model name, or alias
609
611
  *
610
612
  * Returns:
611
- * - matrix axis object compatible with `runMatrix.extend/override`
613
+ * - the configured model matching the selected matrix value
612
614
  */
613
- declare function chatModelMatrix(...names: string[]): MatrixDefinition;
615
+ declare function modelFromMatrix(context: MatrixModelContext, scope: MatrixModelScope, options: ModelFromMatrixOptions): ModelDefinition;
614
616
  /**
615
- * Built-in chat providers plugin that contributes provider presets to config.
617
+ * Resolves a configured chat model from one run-matrix axis.
616
618
  *
617
619
  * Use when:
618
- * - provider runtime config should be centralized and reusable
619
- * - provider parameters should be resolved from env via `envFrom`/`requiredEnvFrom`
620
- */
621
- declare function ChatProviders(options: ChatProvidersPluginOptions): Plugin;
622
- /**
623
- * Built-in chat models plugin that contributes model definitions to vieval config.
620
+ * - run matrix selects the model used by the system under evaluation
621
+ * - callers want the scoped helper instead of passing `scope: 'run'`
624
622
  *
625
- * Use when:
626
- * - chat-model registration should stay in config-level plugin setup
627
- * - tasks and assertions resolve models by name or alias at runtime
623
+ * Expects:
624
+ * - `options.axis` to exist in `context.task.matrix.run`
625
+ *
626
+ * Returns:
627
+ * - the configured model matching the selected run-matrix value
628
628
  */
629
- declare function ChatModels(options: ChatModelsPluginOptions): Plugin;
629
+ declare function modelFromRun(context: MatrixModelContext, options: ModelFromMatrixOptions): ModelDefinition;
630
630
  //#endregion
631
631
  export { ChatModelDefinition, ChatModelExecutorInput, ChatModelExecutorLike, ChatModelFromBaseOptions, ChatModelFromOptions, ChatModelHeaders, ChatModelInferenceExecutor, ChatModelResolverContext, ChatModelResolverValue, ChatModelRuntimeConfig, ChatModelTelemetryProvider, ChatModelToolCall, ChatModels, ChatModelsPluginOptions, ChatProviderDefinition, ChatProviderFromOptions, ChatProviders, ChatProvidersPluginOptions, EmitChatModelErrorTelemetryOptions, EmitChatModelRequestTelemetryOptions, EmitChatModelResponseTelemetryOptions, GenericChatModelInferenceExecutor, MatrixModelScope, ModelFromMatrixOptions, OllamaChatModelInferenceExecutor, OllamaChatModelRuntimeConfig, OpenAIChatModelInferenceExecutor, OpenAIChatModelRuntimeConfig, OpenRouterChatModelInferenceExecutor, OpenRouterChatModelRuntimeConfig, OptionalProviderEnvMap, Plugin, PluginConfig, RequiredProviderEnvMap, chatModelFrom, chatModelMatrix, chatProviderFrom, emitChatModelErrorTelemetry, emitChatModelRequestTelemetry, emitChatModelResponseTelemetry, extractChatModelToolCalls, extractMeteringDimensions, modelFromEval, modelFromMatrix, modelFromRun, ollamaFromRunContext, openaiFromRunContext, openrouterFromRunContext, toChatModelRuntimeConfig };
632
632
  //# sourceMappingURL=index.d.mts.map