vieval 0.0.10 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -31
- package/dist/bin/vieval.mjs +1 -1
- package/dist/bin/vieval.mjs.map +1 -1
- package/dist/cli/index.d.mts +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/{cli-DTDgaqeI.mjs → cli-uzS81IPd.mjs} +1483 -1483
- package/dist/cli-uzS81IPd.mjs.map +1 -0
- package/dist/config.d.mts +1 -1
- package/dist/config.mjs +1 -1
- package/dist/config.mjs.map +1 -1
- package/dist/core/assertions/index.d.mts +156 -156
- package/dist/core/assertions/index.mjs +82 -82
- package/dist/core/assertions/index.mjs.map +1 -1
- package/dist/core/inference-executors/index.d.mts +37 -37
- package/dist/core/inference-executors/index.mjs +54 -53
- package/dist/core/inference-executors/index.mjs.map +1 -1
- package/dist/core/processors/results/index.d.mts +18 -18
- package/dist/core/processors/results/index.mjs.map +1 -1
- package/dist/core/runner/index.d.mts +2 -2
- package/dist/core/runner/index.mjs +259 -259
- package/dist/core/runner/index.mjs.map +1 -1
- package/dist/core/scheduler/index.d.mts +1 -1
- package/dist/core/scheduler/index.mjs +65 -65
- package/dist/core/scheduler/index.mjs.map +1 -1
- package/dist/{env-DfWZy_n4.d.mts → env-Br6jaWGL.d.mts} +9 -9
- package/dist/{env-nV5rVErX.mjs → env-egxaJtNn.mjs} +8 -8
- package/dist/env-egxaJtNn.mjs.map +1 -0
- package/dist/{expect-extensions-DCSqlneN.mjs → expect-extensions-BKdEPt3h.mjs} +46 -46
- package/dist/expect-extensions-BKdEPt3h.mjs.map +1 -0
- package/dist/expect.d.mts +1 -3
- package/dist/expect.mjs +1 -1
- package/dist/expect.mjs.map +1 -1
- package/dist/{index-D_aMeWqO.d.mts → index-BLIlhiWT.d.mts} +565 -565
- package/dist/{index-Bg0atWBF.d.mts → index-CIaJClcC.d.mts} +48 -48
- package/dist/index.d.mts +208 -197
- package/dist/index.mjs +148 -148
- package/dist/index.mjs.map +1 -1
- package/dist/{models-pBSRUZhY.mjs → models-CaCOUPZw.mjs} +1 -1
- package/dist/{models-pBSRUZhY.mjs.map → models-CaCOUPZw.mjs.map} +1 -1
- package/dist/plugins/chat-models/index.d.mts +279 -279
- package/dist/plugins/chat-models/index.mjs +360 -360
- package/dist/plugins/chat-models/index.mjs.map +1 -1
- package/dist/{queue-DsZQkZO_.mjs → queue-BL86z2W_.mjs} +1 -1
- package/dist/{queue-DsZQkZO_.mjs.map → queue-BL86z2W_.mjs.map} +1 -1
- package/dist/{registry-DMnwE_mY.mjs → registry-BK7k6X81.mjs} +294 -294
- package/dist/registry-BK7k6X81.mjs.map +1 -0
- package/dist/testing/expect-extensions.d.mts +27 -27
- package/dist/testing/expect-extensions.mjs +1 -1
- package/package.json +12 -12
- package/dist/cli-DTDgaqeI.mjs.map +0 -1
- package/dist/env-nV5rVErX.mjs.map +0 -1
- package/dist/expect-extensions-DCSqlneN.mjs.map +0 -1
- package/dist/registry-DMnwE_mY.mjs.map +0 -1
|
@@ -1,22 +1,14 @@
|
|
|
1
|
-
import { D as TaskRunContext, R as ModelDefinition, f as MatrixDefinition, t as ConfigHookPlugin, w as TaskExecutionPolicy } from "../../index-
|
|
1
|
+
import { D as TaskRunContext, R as ModelDefinition, f as MatrixDefinition, t as ConfigHookPlugin, w as TaskExecutionPolicy } from "../../index-BLIlhiWT.mjs";
|
|
2
2
|
|
|
3
3
|
//#region src/plugins/chat-models/runtime-config.d.ts
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Union of normalized runtime configs for supported chat-model executors.
|
|
6
6
|
*/
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* Concrete model name.
|
|
14
|
-
*/
|
|
15
|
-
model: string;
|
|
16
|
-
/**
|
|
17
|
-
* Required API key.
|
|
18
|
-
*/
|
|
19
|
-
apiKey: string;
|
|
7
|
+
type ChatModelRuntimeConfig = OllamaChatModelRuntimeConfig | OpenAIChatModelRuntimeConfig | OpenRouterChatModelRuntimeConfig;
|
|
8
|
+
/**
|
|
9
|
+
* Runtime config consumed by Ollama provider constructors.
|
|
10
|
+
*/
|
|
11
|
+
interface OllamaChatModelRuntimeConfig {
|
|
20
12
|
/**
|
|
21
13
|
* Optional base URL override.
|
|
22
14
|
*/
|
|
@@ -25,11 +17,6 @@ interface OpenAIChatModelRuntimeConfig {
|
|
|
25
17
|
* Optional request headers.
|
|
26
18
|
*/
|
|
27
19
|
headers?: ChatModelHeaders;
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Runtime config consumed by Ollama provider constructors.
|
|
31
|
-
*/
|
|
32
|
-
interface OllamaChatModelRuntimeConfig {
|
|
33
20
|
/**
|
|
34
21
|
* Resolved inference executor kind.
|
|
35
22
|
*/
|
|
@@ -38,6 +25,15 @@ interface OllamaChatModelRuntimeConfig {
|
|
|
38
25
|
* Concrete model name.
|
|
39
26
|
*/
|
|
40
27
|
model: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Runtime config consumed by OpenAI-compatible provider constructors.
|
|
31
|
+
*/
|
|
32
|
+
interface OpenAIChatModelRuntimeConfig {
|
|
33
|
+
/**
|
|
34
|
+
* Required API key.
|
|
35
|
+
*/
|
|
36
|
+
apiKey: string;
|
|
41
37
|
/**
|
|
42
38
|
* Optional base URL override.
|
|
43
39
|
*/
|
|
@@ -46,19 +42,19 @@ interface OllamaChatModelRuntimeConfig {
|
|
|
46
42
|
* Optional request headers.
|
|
47
43
|
*/
|
|
48
44
|
headers?: ChatModelHeaders;
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Runtime config consumed by OpenRouter provider constructors.
|
|
52
|
-
*/
|
|
53
|
-
interface OpenRouterChatModelRuntimeConfig {
|
|
54
45
|
/**
|
|
55
46
|
* Resolved inference executor kind.
|
|
56
47
|
*/
|
|
57
|
-
inferenceExecutor: '
|
|
48
|
+
inferenceExecutor: 'openai';
|
|
58
49
|
/**
|
|
59
50
|
* Concrete model name.
|
|
60
51
|
*/
|
|
61
52
|
model: string;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Runtime config consumed by OpenRouter provider constructors.
|
|
56
|
+
*/
|
|
57
|
+
interface OpenRouterChatModelRuntimeConfig {
|
|
62
58
|
/**
|
|
63
59
|
* Required API key.
|
|
64
60
|
*/
|
|
@@ -71,26 +67,29 @@ interface OpenRouterChatModelRuntimeConfig {
|
|
|
71
67
|
* Optional request headers.
|
|
72
68
|
*/
|
|
73
69
|
headers?: ChatModelHeaders;
|
|
70
|
+
/**
|
|
71
|
+
* Resolved inference executor kind.
|
|
72
|
+
*/
|
|
73
|
+
inferenceExecutor: 'openrouter';
|
|
74
|
+
/**
|
|
75
|
+
* Concrete model name.
|
|
76
|
+
*/
|
|
77
|
+
model: string;
|
|
74
78
|
}
|
|
75
79
|
/**
|
|
76
|
-
*
|
|
77
|
-
*/
|
|
78
|
-
type ChatModelRuntimeConfig = OpenAIChatModelRuntimeConfig | OllamaChatModelRuntimeConfig | OpenRouterChatModelRuntimeConfig;
|
|
79
|
-
/**
|
|
80
|
-
* Normalizes one configured chat model into runtime executor config.
|
|
80
|
+
* Resolves Ollama runtime config from one resolved run-context model.
|
|
81
81
|
*
|
|
82
82
|
* Use when:
|
|
83
|
-
* -
|
|
84
|
-
* -
|
|
83
|
+
* - task execution already has a model resolved through chat-model helpers
|
|
84
|
+
* - eval code wants typed Ollama provider options with a concise helper name
|
|
85
85
|
*
|
|
86
86
|
* Expects:
|
|
87
|
-
* - `model
|
|
88
|
-
* - required OpenAI fields (apiKey) to exist in `model.parameters`
|
|
87
|
+
* - `model` to resolve to an Ollama-backed chat model
|
|
89
88
|
*
|
|
90
89
|
* Returns:
|
|
91
|
-
* - validated runtime config
|
|
90
|
+
* - validated Ollama runtime config
|
|
92
91
|
*/
|
|
93
|
-
declare function
|
|
92
|
+
declare function ollamaFromRunContext(model: ModelDefinition): OllamaChatModelRuntimeConfig;
|
|
94
93
|
/**
|
|
95
94
|
* Resolves OpenAI runtime config from one resolved run-context model.
|
|
96
95
|
*
|
|
@@ -106,35 +105,49 @@ declare function toChatModelRuntimeConfig(model: ModelDefinition): ChatModelRunt
|
|
|
106
105
|
*/
|
|
107
106
|
declare function openaiFromRunContext(model: ModelDefinition): OpenAIChatModelRuntimeConfig;
|
|
108
107
|
/**
|
|
109
|
-
* Resolves
|
|
108
|
+
* Resolves OpenRouter runtime config from one resolved run-context model.
|
|
110
109
|
*
|
|
111
110
|
* Use when:
|
|
112
111
|
* - task execution already has a model resolved through chat-model helpers
|
|
113
|
-
* - eval code wants typed
|
|
112
|
+
* - eval code wants typed OpenRouter provider options with a concise helper name
|
|
114
113
|
*
|
|
115
114
|
* Expects:
|
|
116
|
-
* - `model` to resolve to an
|
|
115
|
+
* - `model` to resolve to an OpenRouter-backed chat model
|
|
117
116
|
*
|
|
118
117
|
* Returns:
|
|
119
|
-
* - validated
|
|
118
|
+
* - validated OpenRouter runtime config
|
|
120
119
|
*/
|
|
121
|
-
declare function
|
|
120
|
+
declare function openrouterFromRunContext(model: ModelDefinition): OpenRouterChatModelRuntimeConfig;
|
|
122
121
|
/**
|
|
123
|
-
*
|
|
122
|
+
* Normalizes one configured chat model into runtime executor config.
|
|
124
123
|
*
|
|
125
124
|
* Use when:
|
|
126
|
-
* -
|
|
127
|
-
* -
|
|
125
|
+
* - eval code needs typed provider constructor options from a resolved model
|
|
126
|
+
* - model parameters should be validated once with clear error messages
|
|
128
127
|
*
|
|
129
128
|
* Expects:
|
|
130
|
-
* - `model` to
|
|
129
|
+
* - `model.inferenceExecutorId` to be one of the supported executor ids
|
|
130
|
+
* - required OpenAI fields (apiKey) to exist in `model.parameters`
|
|
131
131
|
*
|
|
132
132
|
* Returns:
|
|
133
|
-
* - validated
|
|
133
|
+
* - validated runtime config union for OpenAI or Ollama
|
|
134
134
|
*/
|
|
135
|
-
declare function
|
|
135
|
+
declare function toChatModelRuntimeConfig(model: ModelDefinition): ChatModelRuntimeConfig;
|
|
136
136
|
//#endregion
|
|
137
137
|
//#region src/plugins/chat-models/telemetry.d.ts
|
|
138
|
+
/**
|
|
139
|
+
* Provider identity attached to chat-model telemetry events.
|
|
140
|
+
*/
|
|
141
|
+
interface ChatModelTelemetryProvider {
|
|
142
|
+
/**
|
|
143
|
+
* Provider id, for example `openai`.
|
|
144
|
+
*/
|
|
145
|
+
id: string;
|
|
146
|
+
/**
|
|
147
|
+
* Optional concrete model id/name.
|
|
148
|
+
*/
|
|
149
|
+
model?: string;
|
|
150
|
+
}
|
|
138
151
|
/**
|
|
139
152
|
* Represents one normalized chat-model tool call.
|
|
140
153
|
*
|
|
@@ -146,52 +159,35 @@ declare function openrouterFromRunContext(model: ModelDefinition): OpenRouterCha
|
|
|
146
159
|
* - `args` to be JSON-serializable
|
|
147
160
|
*/
|
|
148
161
|
interface ChatModelToolCall {
|
|
149
|
-
/**
|
|
150
|
-
* Optional provider-assigned tool-call identifier.
|
|
151
|
-
*/
|
|
152
|
-
id?: string;
|
|
153
|
-
/**
|
|
154
|
-
* Tool name.
|
|
155
|
-
*/
|
|
156
|
-
name: string;
|
|
157
162
|
/**
|
|
158
163
|
* Parsed tool arguments object/value.
|
|
159
164
|
*/
|
|
160
165
|
args: unknown;
|
|
161
|
-
}
|
|
162
|
-
/**
|
|
163
|
-
* Provider identity attached to chat-model telemetry events.
|
|
164
|
-
*/
|
|
165
|
-
interface ChatModelTelemetryProvider {
|
|
166
166
|
/**
|
|
167
|
-
*
|
|
167
|
+
* Optional provider-assigned tool-call identifier.
|
|
168
168
|
*/
|
|
169
|
-
id
|
|
169
|
+
id?: string;
|
|
170
170
|
/**
|
|
171
|
-
*
|
|
171
|
+
* Tool name.
|
|
172
172
|
*/
|
|
173
|
-
|
|
173
|
+
name: string;
|
|
174
174
|
}
|
|
175
175
|
/**
|
|
176
|
-
* Input options for
|
|
176
|
+
* Input options for error telemetry emission.
|
|
177
177
|
*/
|
|
178
|
-
interface
|
|
178
|
+
interface EmitChatModelErrorTelemetryOptions {
|
|
179
179
|
/**
|
|
180
180
|
* Optional case id for case-scoped telemetry events.
|
|
181
181
|
*/
|
|
182
182
|
caseId?: string;
|
|
183
183
|
/**
|
|
184
|
-
*
|
|
184
|
+
* Error payload emitted by the inference client/runtime.
|
|
185
185
|
*/
|
|
186
|
-
|
|
186
|
+
error: unknown;
|
|
187
187
|
/**
|
|
188
188
|
* Optional provider identity payload.
|
|
189
189
|
*/
|
|
190
190
|
provider?: ChatModelTelemetryProvider;
|
|
191
|
-
/**
|
|
192
|
-
* Raw chat-model response object from the inference library/provider.
|
|
193
|
-
*/
|
|
194
|
-
response: unknown;
|
|
195
191
|
}
|
|
196
192
|
/**
|
|
197
193
|
* Input options for request telemetry emission.
|
|
@@ -211,43 +207,46 @@ interface EmitChatModelRequestTelemetryOptions {
|
|
|
211
207
|
provider?: ChatModelTelemetryProvider;
|
|
212
208
|
}
|
|
213
209
|
/**
|
|
214
|
-
* Input options for
|
|
210
|
+
* Input options for response telemetry emission.
|
|
215
211
|
*/
|
|
216
|
-
interface
|
|
212
|
+
interface EmitChatModelResponseTelemetryOptions {
|
|
217
213
|
/**
|
|
218
214
|
* Optional case id for case-scoped telemetry events.
|
|
219
215
|
*/
|
|
220
216
|
caseId?: string;
|
|
221
217
|
/**
|
|
222
|
-
*
|
|
218
|
+
* Optional response latency in milliseconds.
|
|
223
219
|
*/
|
|
224
|
-
|
|
220
|
+
latencyMs?: number;
|
|
225
221
|
/**
|
|
226
222
|
* Optional provider identity payload.
|
|
227
223
|
*/
|
|
228
224
|
provider?: ChatModelTelemetryProvider;
|
|
225
|
+
/**
|
|
226
|
+
* Raw chat-model response object from the inference library/provider.
|
|
227
|
+
*/
|
|
228
|
+
response: unknown;
|
|
229
229
|
}
|
|
230
230
|
/**
|
|
231
|
-
*
|
|
231
|
+
* Emits chat-model failure telemetry as a reportable task event.
|
|
232
232
|
*
|
|
233
233
|
* Use when:
|
|
234
|
-
* -
|
|
235
|
-
* - provider payload differences should stay hidden behind one stable shape
|
|
234
|
+
* - one inference call fails and report artifacts should include normalized error context
|
|
236
235
|
*
|
|
237
|
-
*
|
|
238
|
-
* -
|
|
236
|
+
* Expects:
|
|
237
|
+
* - `context.reporterHooks?.onEvent` to be available in CLI execution paths
|
|
239
238
|
*/
|
|
240
|
-
declare function
|
|
239
|
+
declare function emitChatModelErrorTelemetry(context: TaskRunContext, options: EmitChatModelErrorTelemetryOptions): void;
|
|
241
240
|
/**
|
|
242
|
-
*
|
|
241
|
+
* Emits chat-model request telemetry as a reportable task event.
|
|
243
242
|
*
|
|
244
243
|
* Use when:
|
|
245
|
-
* -
|
|
244
|
+
* - task code submits one model request and wants request-side traceability
|
|
246
245
|
*
|
|
247
|
-
*
|
|
248
|
-
* -
|
|
246
|
+
* Expects:
|
|
247
|
+
* - `context.reporterHooks?.onEvent` to be available in CLI execution paths
|
|
249
248
|
*/
|
|
250
|
-
declare function
|
|
249
|
+
declare function emitChatModelRequestTelemetry(context: TaskRunContext, options: EmitChatModelRequestTelemetryOptions): void;
|
|
251
250
|
/**
|
|
252
251
|
* Emits chat-model response telemetry as reportable task events.
|
|
253
252
|
*
|
|
@@ -263,115 +262,80 @@ declare function extractMeteringDimensions(response: unknown): Record<string, nu
|
|
|
263
262
|
*/
|
|
264
263
|
declare function emitChatModelResponseTelemetry(context: TaskRunContext, options: EmitChatModelResponseTelemetryOptions): void;
|
|
265
264
|
/**
|
|
266
|
-
*
|
|
265
|
+
* Extracts normalized tool calls from one chat-model response shape.
|
|
267
266
|
*
|
|
268
267
|
* Use when:
|
|
269
|
-
* -
|
|
268
|
+
* - downstream scoring, reporting, or analysis should inspect tool call usage
|
|
269
|
+
* - provider payload differences should stay hidden behind one stable shape
|
|
270
270
|
*
|
|
271
|
-
*
|
|
272
|
-
* - `
|
|
271
|
+
* Returns:
|
|
272
|
+
* - normalized list of `{ id?, name, args }` tool calls
|
|
273
273
|
*/
|
|
274
|
-
declare function
|
|
274
|
+
declare function extractChatModelToolCalls(response: unknown): ChatModelToolCall[];
|
|
275
275
|
/**
|
|
276
|
-
*
|
|
276
|
+
* Extracts numeric metering dimensions from one chat-model response usage block.
|
|
277
277
|
*
|
|
278
278
|
* Use when:
|
|
279
|
-
* -
|
|
279
|
+
* - report events should capture usage dimensions in a modality-neutral map
|
|
280
280
|
*
|
|
281
|
-
*
|
|
282
|
-
* -
|
|
281
|
+
* Returns:
|
|
282
|
+
* - numeric dimensions keyed by provider usage field names
|
|
283
283
|
*/
|
|
284
|
-
declare function
|
|
284
|
+
declare function extractMeteringDimensions(response: unknown): Record<string, number>;
|
|
285
285
|
//#endregion
|
|
286
286
|
//#region src/plugins/chat-models/index.d.ts
|
|
287
287
|
/**
|
|
288
|
-
*
|
|
288
|
+
* Chat-model specific specialization of the canonical `ModelDefinition`.
|
|
289
289
|
*/
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
290
|
+
type ChatModelDefinition = Omit<ModelDefinition, 'inferenceExecutor'> & {
|
|
291
|
+
inferenceExecutor: ChatModelExecutorInput;
|
|
292
|
+
provider?: string;
|
|
293
|
+
runtimeResolvers?: {
|
|
294
|
+
apiKey?: ChatModelResolverValue<string>;
|
|
295
|
+
baseURL?: ChatModelResolverValue<string>;
|
|
296
|
+
headers?: ChatModelResolverValue<ChatModelHeaders>;
|
|
297
|
+
};
|
|
298
|
+
};
|
|
293
299
|
/**
|
|
294
300
|
* Inference-executor input accepted by `chatModelFrom`.
|
|
295
301
|
*/
|
|
296
|
-
type ChatModelExecutorInput =
|
|
297
|
-
/**
|
|
298
|
-
* Chat-model header payload accepted by executor parameters.
|
|
299
|
-
*/
|
|
300
|
-
type ChatModelHeaders = Record<string, string | string[]>;
|
|
301
|
-
/**
|
|
302
|
-
* Runtime env context passed to model callback resolvers.
|
|
303
|
-
*/
|
|
304
|
-
interface ChatModelResolverContext {
|
|
305
|
-
env: Record<string, string>;
|
|
306
|
-
}
|
|
307
|
-
/**
|
|
308
|
-
* Value-or-callback resolver used by model runtime fields.
|
|
309
|
-
*/
|
|
310
|
-
type ChatModelResolverValue<TValue> = TValue | ((config: ChatModelResolverContext) => Promise<TValue> | TValue);
|
|
311
|
-
/**
|
|
312
|
-
* OpenAI-specific inference executor config shape.
|
|
313
|
-
*/
|
|
314
|
-
interface OpenAIChatModelInferenceExecutor {
|
|
315
|
-
inferenceExecutor: 'openai';
|
|
316
|
-
apiKey?: ChatModelResolverValue<string>;
|
|
317
|
-
baseURL?: ChatModelResolverValue<string>;
|
|
318
|
-
headers?: ChatModelResolverValue<ChatModelHeaders>;
|
|
319
|
-
}
|
|
320
|
-
/**
|
|
321
|
-
* Ollama-specific inference executor config shape.
|
|
322
|
-
*/
|
|
323
|
-
interface OllamaChatModelInferenceExecutor {
|
|
324
|
-
inferenceExecutor: 'ollama';
|
|
325
|
-
baseURL?: ChatModelResolverValue<string>;
|
|
326
|
-
headers?: ChatModelResolverValue<ChatModelHeaders>;
|
|
327
|
-
}
|
|
328
|
-
/**
|
|
329
|
-
* OpenRouter-specific inference executor config shape.
|
|
330
|
-
*/
|
|
331
|
-
interface OpenRouterChatModelInferenceExecutor {
|
|
332
|
-
inferenceExecutor: 'openrouter';
|
|
333
|
-
apiKey?: ChatModelResolverValue<string>;
|
|
334
|
-
baseURL?: ChatModelResolverValue<string>;
|
|
335
|
-
headers?: ChatModelResolverValue<ChatModelHeaders>;
|
|
336
|
-
}
|
|
302
|
+
type ChatModelExecutorInput = ChatModelExecutorLike | string;
|
|
337
303
|
/**
|
|
338
|
-
*
|
|
304
|
+
* Minimal inference-executor shape expected by chat model runtime callers.
|
|
339
305
|
*/
|
|
340
|
-
interface
|
|
341
|
-
|
|
306
|
+
interface ChatModelExecutorLike {
|
|
307
|
+
chat: (model: string) => Record<string, unknown>;
|
|
342
308
|
}
|
|
343
|
-
/**
|
|
344
|
-
* Union of supported inference executor config shapes for `chatModelFrom`.
|
|
345
|
-
*/
|
|
346
|
-
type ChatModelInferenceExecutor = OpenAIChatModelInferenceExecutor | OllamaChatModelInferenceExecutor | OpenRouterChatModelInferenceExecutor | GenericChatModelInferenceExecutor;
|
|
347
309
|
/**
|
|
348
310
|
* Common builder input fields for `chatModelFrom`.
|
|
349
311
|
*/
|
|
350
312
|
interface ChatModelFromBaseOptions {
|
|
351
313
|
/**
|
|
352
|
-
*
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
314
|
+
* Alias names used by `resolveModelByName`.
|
|
315
|
+
*/
|
|
316
|
+
aliases?: string[];
|
|
317
|
+
/**
|
|
318
|
+
* Additional full task attempts allowed after the current attempt settles.
|
|
356
319
|
*
|
|
357
|
-
*
|
|
358
|
-
* - one `ChatProviders` plugin entry to expose the same id
|
|
320
|
+
* @default 0
|
|
359
321
|
*/
|
|
360
|
-
|
|
322
|
+
autoAttempt?: number;
|
|
361
323
|
/**
|
|
362
|
-
*
|
|
324
|
+
* Additional retries allowed within the current attempt.
|
|
325
|
+
*
|
|
326
|
+
* @default 0
|
|
363
327
|
*/
|
|
364
|
-
|
|
328
|
+
autoRetry?: number;
|
|
365
329
|
/**
|
|
366
|
-
*
|
|
330
|
+
* Delay in milliseconds before a retry starts.
|
|
367
331
|
*
|
|
368
|
-
* @default
|
|
332
|
+
* @default retryIndex => 500 * 2 ** (retryIndex - 1)
|
|
369
333
|
*/
|
|
370
|
-
|
|
334
|
+
autoRetryDelay?: TaskExecutionPolicy['autoRetryDelay'];
|
|
371
335
|
/**
|
|
372
|
-
*
|
|
336
|
+
* Optional execution policy hints attached to this model.
|
|
373
337
|
*/
|
|
374
|
-
|
|
338
|
+
executionPolicy?: TaskExecutionPolicy;
|
|
375
339
|
/**
|
|
376
340
|
* Optional stable model id.
|
|
377
341
|
*
|
|
@@ -379,72 +343,69 @@ interface ChatModelFromBaseOptions {
|
|
|
379
343
|
*/
|
|
380
344
|
id?: string;
|
|
381
345
|
/**
|
|
382
|
-
*
|
|
346
|
+
* Inference-executor id or inference-executor instance.
|
|
383
347
|
*/
|
|
384
|
-
|
|
348
|
+
inferenceExecutor?: ChatModelExecutorInput;
|
|
385
349
|
/**
|
|
386
|
-
* Optional
|
|
350
|
+
* Optional explicit inference-executor id for inference-executor instances.
|
|
351
|
+
*
|
|
352
|
+
* @default 'custom'
|
|
387
353
|
*/
|
|
388
|
-
|
|
354
|
+
inferenceExecutorId?: string;
|
|
389
355
|
/**
|
|
390
|
-
*
|
|
391
|
-
*
|
|
392
|
-
* @default 0
|
|
356
|
+
* Concrete model name.
|
|
393
357
|
*/
|
|
394
|
-
|
|
358
|
+
model: string;
|
|
395
359
|
/**
|
|
396
|
-
*
|
|
397
|
-
*
|
|
398
|
-
* @default retryIndex => 500 * 2 ** (retryIndex - 1)
|
|
360
|
+
* Optional model-level call parameters.
|
|
399
361
|
*/
|
|
400
|
-
|
|
362
|
+
parameters?: Record<string, unknown>;
|
|
401
363
|
/**
|
|
402
|
-
*
|
|
364
|
+
* Provider id registered through `ChatProviders`.
|
|
403
365
|
*
|
|
404
|
-
*
|
|
366
|
+
* Use when:
|
|
367
|
+
* - model runtime transport and credentials should be delegated to a named provider preset
|
|
368
|
+
*
|
|
369
|
+
* Expects:
|
|
370
|
+
* - one `ChatProviders` plugin entry to expose the same id
|
|
405
371
|
*/
|
|
406
|
-
|
|
372
|
+
provider?: string;
|
|
407
373
|
/**
|
|
408
374
|
* Timeout in milliseconds for model-backed work.
|
|
409
375
|
*/
|
|
410
376
|
timeout?: number;
|
|
411
|
-
/**
|
|
412
|
-
* Optional model-level call parameters.
|
|
413
|
-
*/
|
|
414
|
-
parameters?: Record<string, unknown>;
|
|
415
377
|
}
|
|
416
378
|
/**
|
|
417
379
|
* Builder input for `chatModelFrom`.
|
|
418
380
|
*/
|
|
419
|
-
type ChatModelFromOptions =
|
|
381
|
+
type ChatModelFromOptions = ChatModelFromBaseOptions & ChatModelInferenceExecutor;
|
|
420
382
|
/**
|
|
421
|
-
* Chat-model
|
|
383
|
+
* Chat-model header payload accepted by executor parameters.
|
|
422
384
|
*/
|
|
423
|
-
type
|
|
424
|
-
inferenceExecutor: ChatModelExecutorInput;
|
|
425
|
-
provider?: string;
|
|
426
|
-
runtimeResolvers?: {
|
|
427
|
-
apiKey?: ChatModelResolverValue<string>;
|
|
428
|
-
baseURL?: ChatModelResolverValue<string>;
|
|
429
|
-
headers?: ChatModelResolverValue<ChatModelHeaders>;
|
|
430
|
-
};
|
|
431
|
-
};
|
|
385
|
+
type ChatModelHeaders = Record<string, string | string[]>;
|
|
432
386
|
/**
|
|
433
|
-
*
|
|
434
|
-
*
|
|
435
|
-
* Use when:
|
|
436
|
-
* - provider parameter values should be read from env keys
|
|
437
|
-
* - missing keys should resolve to `undefined`
|
|
387
|
+
* Union of supported inference executor config shapes for `chatModelFrom`.
|
|
438
388
|
*/
|
|
439
|
-
type
|
|
389
|
+
type ChatModelInferenceExecutor = GenericChatModelInferenceExecutor | OllamaChatModelInferenceExecutor | OpenAIChatModelInferenceExecutor | OpenRouterChatModelInferenceExecutor;
|
|
440
390
|
/**
|
|
441
|
-
*
|
|
442
|
-
*
|
|
443
|
-
* Use when:
|
|
444
|
-
* - provider parameter values must exist before model execution
|
|
445
|
-
* - missing keys should throw with key-aware error messages
|
|
391
|
+
* Runtime env context passed to model callback resolvers.
|
|
446
392
|
*/
|
|
447
|
-
|
|
393
|
+
interface ChatModelResolverContext {
|
|
394
|
+
env: Record<string, string>;
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* Value-or-callback resolver used by model runtime fields.
|
|
398
|
+
*/
|
|
399
|
+
type ChatModelResolverValue<TValue> = ((config: ChatModelResolverContext) => Promise<TValue> | TValue) | TValue;
|
|
400
|
+
/**
|
|
401
|
+
* Options for the built-in `ChatModels` plugin.
|
|
402
|
+
*/
|
|
403
|
+
interface ChatModelsPluginOptions {
|
|
404
|
+
/**
|
|
405
|
+
* Chat model definitions to append to config.
|
|
406
|
+
*/
|
|
407
|
+
models: readonly ChatModelDefinition[];
|
|
408
|
+
}
|
|
448
409
|
/**
|
|
449
410
|
* One provider definition consumed by chat model presets.
|
|
450
411
|
*/
|
|
@@ -463,10 +424,6 @@ interface ChatProviderDefinition {
|
|
|
463
424
|
* @default 'custom'
|
|
464
425
|
*/
|
|
465
426
|
inferenceExecutorId?: string;
|
|
466
|
-
/**
|
|
467
|
-
* Optional literal provider-level parameters.
|
|
468
|
-
*/
|
|
469
|
-
parameters?: Record<string, unknown>;
|
|
470
427
|
/**
|
|
471
428
|
* Optional provider parameters resolved via `envFrom`.
|
|
472
429
|
*
|
|
@@ -475,6 +432,10 @@ interface ChatProviderDefinition {
|
|
|
475
432
|
* - map value is the env key name
|
|
476
433
|
*/
|
|
477
434
|
optionalEnv?: OptionalProviderEnvMap;
|
|
435
|
+
/**
|
|
436
|
+
* Optional literal provider-level parameters.
|
|
437
|
+
*/
|
|
438
|
+
parameters?: Record<string, unknown>;
|
|
478
439
|
/**
|
|
479
440
|
* Required provider parameters resolved via `requiredEnvFrom`.
|
|
480
441
|
*
|
|
@@ -492,97 +453,137 @@ interface ChatProviderFromOptions extends ChatProviderDefinition {}
|
|
|
492
453
|
* Options for the built-in `ChatProviders` plugin.
|
|
493
454
|
*/
|
|
494
455
|
interface ChatProvidersPluginOptions {
|
|
495
|
-
/**
|
|
496
|
-
* Provider definitions to append to config.
|
|
497
|
-
*/
|
|
498
|
-
providers: readonly ChatProviderDefinition[];
|
|
499
456
|
/**
|
|
500
457
|
* Optional explicit env source used for env-backed provider parameters.
|
|
501
458
|
*
|
|
502
459
|
* @default process.env
|
|
503
460
|
*/
|
|
504
461
|
env?: NodeJS.ProcessEnv;
|
|
462
|
+
/**
|
|
463
|
+
* Provider definitions to append to config.
|
|
464
|
+
*/
|
|
465
|
+
providers: readonly ChatProviderDefinition[];
|
|
505
466
|
}
|
|
506
467
|
/**
|
|
507
|
-
*
|
|
468
|
+
* Generic inference executor config shape.
|
|
508
469
|
*/
|
|
509
|
-
interface
|
|
510
|
-
|
|
511
|
-
chatProviders?: ChatProviderDefinition[];
|
|
512
|
-
models?: ModelDefinition[];
|
|
470
|
+
interface GenericChatModelInferenceExecutor {
|
|
471
|
+
inferenceExecutor?: ChatModelExecutorInput;
|
|
513
472
|
}
|
|
514
473
|
/**
|
|
515
|
-
*
|
|
474
|
+
* Matrix scope that can carry a chat model selector.
|
|
516
475
|
*/
|
|
517
|
-
type
|
|
476
|
+
type MatrixModelScope = 'eval' | 'run';
|
|
518
477
|
/**
|
|
519
|
-
*
|
|
520
|
-
*
|
|
521
|
-
* Use when:
|
|
522
|
-
* - registering chat models through config plugins
|
|
523
|
-
* - a single model needs aliases for matrix selection or judge lookup
|
|
478
|
+
* Options for resolving a chat model from a matrix axis.
|
|
524
479
|
*/
|
|
525
|
-
|
|
480
|
+
interface ModelFromMatrixOptions {
|
|
481
|
+
/**
|
|
482
|
+
* Matrix axis whose selected value is a model id, model name, or alias.
|
|
483
|
+
*/
|
|
484
|
+
axis: string;
|
|
485
|
+
}
|
|
526
486
|
/**
|
|
527
|
-
*
|
|
528
|
-
*
|
|
529
|
-
* Use when:
|
|
530
|
-
* - one provider preset should be reused across multiple chat models
|
|
531
|
-
* - provider configuration should support required/optional env-backed parameters
|
|
487
|
+
* Ollama-specific inference executor config shape.
|
|
532
488
|
*/
|
|
533
|
-
|
|
489
|
+
interface OllamaChatModelInferenceExecutor {
|
|
490
|
+
baseURL?: ChatModelResolverValue<string>;
|
|
491
|
+
headers?: ChatModelResolverValue<ChatModelHeaders>;
|
|
492
|
+
inferenceExecutor: 'ollama';
|
|
493
|
+
}
|
|
534
494
|
/**
|
|
535
|
-
*
|
|
495
|
+
* OpenAI-specific inference executor config shape.
|
|
536
496
|
*/
|
|
537
|
-
interface
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
497
|
+
interface OpenAIChatModelInferenceExecutor {
|
|
498
|
+
apiKey?: ChatModelResolverValue<string>;
|
|
499
|
+
baseURL?: ChatModelResolverValue<string>;
|
|
500
|
+
headers?: ChatModelResolverValue<ChatModelHeaders>;
|
|
501
|
+
inferenceExecutor: 'openai';
|
|
542
502
|
}
|
|
543
503
|
/**
|
|
544
|
-
*
|
|
504
|
+
* OpenRouter-specific inference executor config shape.
|
|
545
505
|
*/
|
|
546
|
-
|
|
506
|
+
interface OpenRouterChatModelInferenceExecutor {
|
|
507
|
+
apiKey?: ChatModelResolverValue<string>;
|
|
508
|
+
baseURL?: ChatModelResolverValue<string>;
|
|
509
|
+
headers?: ChatModelResolverValue<ChatModelHeaders>;
|
|
510
|
+
inferenceExecutor: 'openrouter';
|
|
511
|
+
}
|
|
547
512
|
/**
|
|
548
|
-
*
|
|
513
|
+
* Env-key map for optional provider parameters.
|
|
514
|
+
*
|
|
515
|
+
* Use when:
|
|
516
|
+
* - provider parameter values should be read from env keys
|
|
517
|
+
* - missing keys should resolve to `undefined`
|
|
549
518
|
*/
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
519
|
+
type OptionalProviderEnvMap = Record<string, string>;
|
|
520
|
+
/**
|
|
521
|
+
* Plugin type bound to the minimal config shape used by model plugins.
|
|
522
|
+
*/
|
|
523
|
+
type Plugin = ConfigHookPlugin<PluginConfig>;
|
|
524
|
+
/**
|
|
525
|
+
* Partial config shape needed by the chat models plugin.
|
|
526
|
+
*/
|
|
527
|
+
interface PluginConfig {
|
|
528
|
+
chatProviders?: ChatProviderDefinition[];
|
|
529
|
+
env?: NodeJS.ProcessEnv;
|
|
530
|
+
models?: ModelDefinition[];
|
|
555
531
|
}
|
|
532
|
+
/**
|
|
533
|
+
* Env-key map for required provider parameters.
|
|
534
|
+
*
|
|
535
|
+
* Use when:
|
|
536
|
+
* - provider parameter values must exist before model execution
|
|
537
|
+
* - missing keys should throw with key-aware error messages
|
|
538
|
+
*/
|
|
539
|
+
type RequiredProviderEnvMap = Record<string, string>;
|
|
556
540
|
type MatrixModelContext = Pick<TaskRunContext, 'models' | 'task'>;
|
|
557
541
|
/**
|
|
558
|
-
*
|
|
542
|
+
* Builds one normalized chat model definition.
|
|
559
543
|
*
|
|
560
544
|
* Use when:
|
|
561
|
-
* -
|
|
562
|
-
* -
|
|
545
|
+
* - registering chat models through config plugins
|
|
546
|
+
* - a single model needs aliases for matrix selection or judge lookup
|
|
547
|
+
*/
|
|
548
|
+
declare function chatModelFrom(options: ChatModelFromOptions): ChatModelDefinition;
|
|
549
|
+
/**
|
|
550
|
+
* Creates a run-matrix `model` axis from configured chat model names.
|
|
551
|
+
*
|
|
552
|
+
* Use when:
|
|
553
|
+
* - run matrix should iterate over explicit chat model ids/aliases
|
|
554
|
+
* - project configs want a concise model-axis helper
|
|
563
555
|
*
|
|
564
556
|
* Expects:
|
|
565
|
-
* -
|
|
566
|
-
* - `options.axis` to exist and contain a model id, model name, or alias
|
|
557
|
+
* - each provided name to match a configured model id or alias at runtime
|
|
567
558
|
*
|
|
568
559
|
* Returns:
|
|
569
|
-
* -
|
|
560
|
+
* - matrix axis object compatible with `runMatrix.extend/override`
|
|
570
561
|
*/
|
|
571
|
-
declare function
|
|
562
|
+
declare function chatModelMatrix(...names: string[]): MatrixDefinition;
|
|
572
563
|
/**
|
|
573
|
-
*
|
|
564
|
+
* Built-in chat models plugin that contributes model definitions to vieval config.
|
|
574
565
|
*
|
|
575
566
|
* Use when:
|
|
576
|
-
* -
|
|
577
|
-
* -
|
|
567
|
+
* - chat-model registration should stay in config-level plugin setup
|
|
568
|
+
* - tasks and assertions resolve models by name or alias at runtime
|
|
569
|
+
*/
|
|
570
|
+
declare function ChatModels(options: ChatModelsPluginOptions): Plugin;
|
|
571
|
+
/**
|
|
572
|
+
* Builds one normalized chat provider definition.
|
|
578
573
|
*
|
|
579
|
-
*
|
|
580
|
-
* -
|
|
574
|
+
* Use when:
|
|
575
|
+
* - one provider preset should be reused across multiple chat models
|
|
576
|
+
* - provider configuration should support required/optional env-backed parameters
|
|
577
|
+
*/
|
|
578
|
+
declare function chatProviderFrom(options: ChatProviderFromOptions): ChatProviderDefinition;
|
|
579
|
+
/**
|
|
580
|
+
* Built-in chat providers plugin that contributes provider presets to config.
|
|
581
581
|
*
|
|
582
|
-
*
|
|
583
|
-
* -
|
|
582
|
+
* Use when:
|
|
583
|
+
* - provider runtime config should be centralized and reusable
|
|
584
|
+
* - provider parameters should be resolved from env via `envFrom`/`requiredEnvFrom`
|
|
584
585
|
*/
|
|
585
|
-
declare function
|
|
586
|
+
declare function ChatProviders(options: ChatProvidersPluginOptions): Plugin;
|
|
586
587
|
/**
|
|
587
588
|
* Resolves a configured chat model from one eval-matrix axis.
|
|
588
589
|
*
|
|
@@ -598,35 +599,34 @@ declare function modelFromRun(context: MatrixModelContext, options: ModelFromMat
|
|
|
598
599
|
*/
|
|
599
600
|
declare function modelFromEval(context: MatrixModelContext, options: ModelFromMatrixOptions): ModelDefinition;
|
|
600
601
|
/**
|
|
601
|
-
*
|
|
602
|
+
* Resolves a configured chat model from one scoped matrix axis.
|
|
602
603
|
*
|
|
603
604
|
* Use when:
|
|
604
|
-
* -
|
|
605
|
-
* -
|
|
605
|
+
* - a matrix axis selects the agent, judge, or another chat model role
|
|
606
|
+
* - eval code should keep model lookup semantics inside the chat-models plugin
|
|
606
607
|
*
|
|
607
608
|
* Expects:
|
|
608
|
-
* -
|
|
609
|
+
* - `scope` to identify `context.task.matrix.run` or `context.task.matrix.eval`
|
|
610
|
+
* - `options.axis` to exist and contain a model id, model name, or alias
|
|
609
611
|
*
|
|
610
612
|
* Returns:
|
|
611
|
-
* -
|
|
613
|
+
* - the configured model matching the selected matrix value
|
|
612
614
|
*/
|
|
613
|
-
declare function
|
|
615
|
+
declare function modelFromMatrix(context: MatrixModelContext, scope: MatrixModelScope, options: ModelFromMatrixOptions): ModelDefinition;
|
|
614
616
|
/**
|
|
615
|
-
*
|
|
617
|
+
* Resolves a configured chat model from one run-matrix axis.
|
|
616
618
|
*
|
|
617
619
|
* Use when:
|
|
618
|
-
* -
|
|
619
|
-
* -
|
|
620
|
-
*/
|
|
621
|
-
declare function ChatProviders(options: ChatProvidersPluginOptions): Plugin;
|
|
622
|
-
/**
|
|
623
|
-
* Built-in chat models plugin that contributes model definitions to vieval config.
|
|
620
|
+
* - run matrix selects the model used by the system under evaluation
|
|
621
|
+
* - callers want the scoped helper instead of passing `scope: 'run'`
|
|
624
622
|
*
|
|
625
|
-
*
|
|
626
|
-
* -
|
|
627
|
-
*
|
|
623
|
+
* Expects:
|
|
624
|
+
* - `options.axis` to exist in `context.task.matrix.run`
|
|
625
|
+
*
|
|
626
|
+
* Returns:
|
|
627
|
+
* - the configured model matching the selected run-matrix value
|
|
628
628
|
*/
|
|
629
|
-
declare function
|
|
629
|
+
declare function modelFromRun(context: MatrixModelContext, options: ModelFromMatrixOptions): ModelDefinition;
|
|
630
630
|
//#endregion
|
|
631
631
|
export { ChatModelDefinition, ChatModelExecutorInput, ChatModelExecutorLike, ChatModelFromBaseOptions, ChatModelFromOptions, ChatModelHeaders, ChatModelInferenceExecutor, ChatModelResolverContext, ChatModelResolverValue, ChatModelRuntimeConfig, ChatModelTelemetryProvider, ChatModelToolCall, ChatModels, ChatModelsPluginOptions, ChatProviderDefinition, ChatProviderFromOptions, ChatProviders, ChatProvidersPluginOptions, EmitChatModelErrorTelemetryOptions, EmitChatModelRequestTelemetryOptions, EmitChatModelResponseTelemetryOptions, GenericChatModelInferenceExecutor, MatrixModelScope, ModelFromMatrixOptions, OllamaChatModelInferenceExecutor, OllamaChatModelRuntimeConfig, OpenAIChatModelInferenceExecutor, OpenAIChatModelRuntimeConfig, OpenRouterChatModelInferenceExecutor, OpenRouterChatModelRuntimeConfig, OptionalProviderEnvMap, Plugin, PluginConfig, RequiredProviderEnvMap, chatModelFrom, chatModelMatrix, chatProviderFrom, emitChatModelErrorTelemetry, emitChatModelRequestTelemetry, emitChatModelResponseTelemetry, extractChatModelToolCalls, extractMeteringDimensions, modelFromEval, modelFromMatrix, modelFromRun, ollamaFromRunContext, openaiFromRunContext, openrouterFromRunContext, toChatModelRuntimeConfig };
|
|
632
632
|
//# sourceMappingURL=index.d.mts.map
|